diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml new file mode 100644 index 00000000000..8094ca57467 --- /dev/null +++ b/.github/workflows/mutation-test.yml @@ -0,0 +1,131 @@ +name: "Mutation Test (manual)" + +# Manually-triggered mutation testing. Runs mutmut against the scope +# configured in [tool.mutmut] in pyproject.toml (currently the +# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly +# weekly — clicked from the Actions tab when someone wants a fresh report. +# +# Uploads a structured `mutation-report.md` (Meta ACH-style: original + +# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the +# existing tests + a task instruction) as a workflow artifact. Failures +# do not block anything because nothing depends on this workflow. + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mutation-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + mutation: + name: Run mutmut + runs-on: ubuntu-latest + # Whole-folder mutation against ~15 files / ~7.5k LOC can take hours. + # 350 minutes is just under the GitHub-hosted job cap of 360 minutes. + timeout-minutes: 350 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + # mutmut 3.x runs tests inside a `mutants/` sandbox where it injects + # mutation trampolines. uv installs the project as editable by default, + # which puts the original source dir on sys.path via a .pth file and + # shadows the sandbox copy — so tests would never exercise the mutated + # code. Reinstalling non-editable removes the .pth shadow. + - name: Reinstall litellm non-editable (so mutants/ is not shadowed) + run: | + uv pip uninstall litellm + uv pip install . --no-deps + + # pytest-retry's pytest_configure hook crashes with + # `INTERNALERROR: no option named 'filtered_exceptions'` when invoked + # via mutmut's in-process pytest.main() call. The entry-point name + # doesn't normalize cleanly with `-p no:`, so just remove the + # package outright. Reruns are wrong for mutation testing anyway — + # rerunning a "failed" mutant test would mask which mutants are killed. + - name: Remove pytest plugins that conflict with mutmut + run: | + uv pip uninstall pytest-retry || true + + - name: Run mutmut + env: + # Make the mutants/ sandbox win over site-packages on sys.path so the + # trampolined files are imported instead of the installed copy. + PYTHONPATH: ${{ github.workspace }}/mutants + run: | + set -o pipefail + mkdir -p mutants + uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log + + # Generate the structured report. The script embeds the enclosing + # function source for each survivor (via Python AST) and includes the + # existing test files, so an LLM agent has enough context to write + # killing tests without further file lookups. Modeled on Meta's ACH + # prompt template (arXiv 2501.12862). + - name: Generate detailed mutation report + if: always() + run: | + set +e + uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1 + uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1 + uv run --no-sync python scripts/mutation_report.py + # The full report can be very long for big test files; the run-page + # summary cuts off at 1 MB. Append the head of the report (summary + # + survivor list) and link out to the artifact for the full body. + { + head -c 900000 mutation-report.md + echo "" + echo "" + echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload mutmut artifacts + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: mutmut-${{ github.run_id }}-${{ github.run_attempt }} + path: | + mutation-report.md + mutmut-results.txt + mutmut-run.log + mutants/mutmut-stats.json + mutants/mutmut-cicd-stats.json + mutants/litellm/proxy/management_endpoints/**/*.py + if-no-files-found: warn + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 4bdbf26ae9d..e99bf79d783 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates: ### Running the proxy server -Start the proxy with a config file: +Create a minimal config file and start the proxy: + +```yaml +# config.yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: https://fake-api.example.com + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False +``` ```bash -uv run litellm --config dev_config.yaml --port 4000 +uv run litellm --config config.yaml --port 4000 ``` The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package. diff --git a/CLAUDE.md b/CLAUDE.md index 71e5af28ee7..938801df7c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets. - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields. - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries. -- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. +- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`. ### Setup Wizard (`litellm/setup_wizard.py`) - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI). diff --git a/deploy/Dockerfile.ghcr_base b/deploy/Dockerfile.ghcr_base deleted file mode 100644 index 66e64e5b774..00000000000 --- a/deploy/Dockerfile.ghcr_base +++ /dev/null @@ -1,18 +0,0 @@ -# Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186 - -# Set the working directory to /app -WORKDIR /app - -# Copy the configuration file into the container at /app -COPY config.yaml . - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh - -# Expose the necessary port -EXPOSE 4000/tcp - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"] diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 6aa1771b7bb..25f69080878 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -100,6 +100,16 @@ spec: - name: DATABASE_URL value: {{ .Values.db.url | quote }} {{- end }} + {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} + - name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.readReplicaUrlKey }} + {{- else if .Values.db.readReplicaUrl }} + - name: DATABASE_URL_READ_REPLICA + value: {{ .Values.db.readReplicaUrl | quote }} + {{- end }} - name: PROXY_MASTER_KEY valueFrom: secretKeyRef: diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index ba4059e0840..9c7c013341b 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -252,6 +252,26 @@ db: passwordKey: password # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint endpointKey: "" + # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this + # secret key instead of db.readReplicaUrl. Prefer this over the plain + # value: read-replica URLs typically embed credentials, and a value + # written to db.readReplicaUrl ends up visible in the rendered pod spec + # and the Helm release secret. + readReplicaUrlKey: "" + + # Optional read-replica routing. When set, the proxy sends read-only + # queries (find_*, count, group_by, query_raw/_first) to this URL while + # writes continue to go to db.url. Useful for Aurora-style clusters with + # separate reader/writer endpoints. Leave empty to keep single-DB behavior. + # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed + # alongside the writer (host/port/user/db are parsed from this URL once + # at startup; only the IAM token rotates). + # + # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over + # this field — the plain value is rendered into the pod spec and the + # Helm release secret. This field is intended for credential-less URLs + # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). + readReplicaUrl: "" # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. # The Stackgres Operator must already be installed within the target diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml deleted file mode 100644 index d5ba500d8f0..00000000000 --- a/deploy/kubernetes/kub.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: litellm-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: litellm - template: - metadata: - labels: - app: litellm - spec: - containers: - - name: litellm-container - image: ghcr.io/berriai/litellm:main-latest - imagePullPolicy: Always - env: - - name: AZURE_API_KEY - value: "d6f****" - - name: AZURE_API_BASE - value: "https://openai" - - name: LITELLM_MASTER_KEY - value: "sk-1234" - - name: DATABASE_URL - value: "postgresql://ishaan*********" - args: - - "--config" - - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml - - name: config-volume - mountPath: /app - readOnly: true - livenessProbe: - httpGet: - path: /health/liveliness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /health/readiness - port: 4000 - initialDelaySeconds: 120 - periodSeconds: 15 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 10 - volumes: # Define volume to mount proxy_config.yaml - - name: config-volume - configMap: - name: litellm-config diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml deleted file mode 100644 index 4751c837254..00000000000 --- a/deploy/kubernetes/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: litellm-service -spec: - selector: - app: litellm - ports: - - protocol: TCP - port: 4000 - targetPort: 4000 - type: LoadBalancer \ No newline at end of file diff --git a/dev_config.yaml b/dev_config.yaml deleted file mode 100644 index 64e3c14703e..00000000000 --- a/dev_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_list: - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake-model - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -general_settings: - master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/docker-compose.yml b/docker-compose.yml index 988860a7877..80e1f289aad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,11 @@ services: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" + # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) + # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for + # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL + # is auto-refreshed alongside the writer. + # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI env_file: - .env # Load local .env file diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 5de588cf4e4..00000000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,68 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -# Copy full source tree -COPY . . - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python3 - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui deleted file mode 100644 index cc44893bf92..00000000000 --- a/docker/Dockerfile.custom_ui +++ /dev/null @@ -1,86 +0,0 @@ -# Use the provided base image -# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt. -FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev - -# Set the working directory to /app -WORKDIR /app - -# Install Node.js and npm (adjust version as needed) -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 && \ - apt-get install -y --no-install-recommends nodejs npm && \ - npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ - GLOBAL="$(npm root -g)" && \ - find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done && \ - find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done && \ - find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \ - npm cache clean --force && \ - apt-get purge -y npm - -# Copy the UI source into the container -COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard - -# Set an environment variable for UI_BASE_PATH -# This can be overridden at build time -# set UI_BASE_PATH to "/ui" -ENV UI_BASE_PATH="/prod/ui" - -# Build the UI with the specified UI_BASE_PATH -WORKDIR /app/ui/litellm-dashboard -RUN npm ci -RUN UI_BASE_PATH=$UI_BASE_PATH npm run build - -# Create the destination directory -RUN mkdir -p /app/litellm/proxy/_experimental/out - -# Move the built files to the appropriate location -# Assuming the build output is in ./out directory -RUN rm -rf /app/litellm/proxy/_experimental/out/* && \ - mv ./out/* /app/litellm/proxy/_experimental/out/ - -# Switch back to the main app directory -WORKDIR /app - -# Make sure your docker/entrypoint.sh is executable -# Convert Windows line endings to Unix for entrypoint scripts -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh -RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \ - && chown -R appuser:appuser /app -USER appuser - -# Expose the necessary port -EXPOSE 4000/tcp - -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"] - -# Override the CMD instruction with your desired command and arguments -CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"] \ No newline at end of file diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index ebc92a22d50..00000000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,121 +0,0 @@ -# Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - python3-dev \ - libssl-dev \ - pkg-config \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -# Copy full source tree -COPY . . - -# Build Admin UI before final sync -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --python python - -RUN prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN apt-get update && apt-get upgrade -y \ - libxml2 \ - libexpat1 \ - openssl \ - libssl3 \ - git \ - libkrb5-3 \ - libglib2.0-0 \ - wget \ - libaom3 \ - libxslt1.1 \ - libgnutls30 \ - libc6 \ - && apt-get install -y --no-install-recommends \ - libssl3 \ - libatomic1 \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* \ - && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \ - && GLOBAL="$(npm root -g)" \ - && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \ - done \ - && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \ - rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \ - done \ - && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \ - sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \ - && npm cache clean --force \ - && apt-get purge -y npm - -WORKDIR /app -ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - XDG_CACHE_HOME=/app/.cache \ - PATH="/app/.venv/bin:${PATH}" - -COPY --from=builder /app /app - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check deleted file mode 100644 index a2e5cb9f71f..00000000000 --- a/docker/Dockerfile.health_check +++ /dev/null @@ -1,30 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -WORKDIR /app - -# Copy the uv binary and the health check script. -COPY --from=uvbin /uv /usr/local/bin/uv -COPY pyproject.toml uv.lock /app/ -COPY scripts/health_check/health_check_client.py /app/health_check_client.py - -# Resolve and install the health-check dependencies from the project lockfile -# so the runtime image stays self-contained and reproducible. -RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \ - && uv pip install --system -r /tmp/health-check-requirements.txt \ - && rm /tmp/health-check-requirements.txt \ - && rm /app/pyproject.toml /app/uv.lock \ - && chmod +x /app/health_check_client.py - -# Run as non-root user -RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser -USER appuser - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ - CMD ["python", "/app/health_check_client.py", "--help"] - -# Set entrypoint -ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/index.yaml b/index.yaml deleted file mode 100644 index 9b2461c36b5..00000000000 --- a/index.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: v1 -entries: - litellm-helm: - - apiVersion: v2 - appVersion: v1.43.18 - created: "2024-08-19T23:58:25.331689+08:00" - dependencies: - - condition: db.deployStandalone - name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=13.3.0' - - condition: redis.enabled - name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: '>=18.0.0' - description: Call all LLM APIs using the OpenAI format - digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531 - name: litellm-helm - type: application - urls: - - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz - version: 0.2.3 - postgresql: - - annotations: - category: Database - images: | - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: postgres-exporter - image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14 - - name: postgresql - image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 16.2.0 - created: "2024-08-19T23:58:25.335716+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: PostgreSQL (Postgres) is an open source object-relational database - known for reliability and data integrity. ACID-compliant, it supports foreign - keys, joins, views, triggers and stored procedures. - digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png - keywords: - - postgresql - - postgres - - database - - sql - - replication - - cluster - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: postgresql - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/postgresql - urls: - - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz - version: 14.3.1 - redis: - - annotations: - category: Database - images: | - - name: kubectl - image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3 - - name: os-shell - image: docker.io/bitnami/os-shell:12-debian-12-r16 - - name: redis - image: docker.io/bitnami/redis:7.2.4-debian-12-r9 - - name: redis-exporter - image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4 - - name: redis-sentinel - image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7 - licenses: Apache-2.0 - apiVersion: v2 - appVersion: 7.2.4 - created: "2024-08-19T23:58:25.339392+08:00" - dependencies: - - name: common - repository: oci://registry-1.docker.io/bitnamicharts - tags: - - bitnami-common - version: 2.x.x - description: Redis(R) is an open source, advanced key-value store. It is often - referred to as a data structure server since keys can contain strings, hashes, - lists, sets and sorted sets. - digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984 - home: https://bitnami.com - icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png - keywords: - - redis - - keyvalue - - database - maintainers: - - name: VMware, Inc. - url: https://github.com/bitnami/charts - name: redis - sources: - - https://github.com/bitnami/charts/tree/main/bitnami/redis - urls: - - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz - version: 18.19.1 -generated: "2024-08-19T23:58:25.322532+08:00" diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/proxy/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/proxy/README.md b/litellm-js/proxy/README.md deleted file mode 100644 index cc58e962d8f..00000000000 --- a/litellm-js/proxy/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -npm run deploy -``` diff --git a/litellm-js/proxy/package-lock.json b/litellm-js/proxy/package-lock.json deleted file mode 100644 index 0d09fa1a6c4..00000000000 --- a/litellm-js/proxy/package-lock.json +++ /dev/null @@ -1,2054 +0,0 @@ -{ - "name": "proxy", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } - }, - "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", - "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", - "dev": true, - "license": "MIT OR Apache-2.0", - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@cloudflare/unenv-preset": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", - "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", - "dev": true, - "license": "MIT OR Apache-2.0", - "peerDependencies": { - "unenv": "2.0.0-rc.24", - "workerd": ">1.20260305.0 <2.0.0-0" - }, - "peerDependenciesMeta": { - "workerd": { - "optional": true - } - } - }, - "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260430.1.tgz", - "integrity": "sha512-ADohZUHf7NBvPp2PdZig2Opxx+hDkk3ve7jrTne3JRx9kDSB73zc4LzcEeEN8LKkbAcqZmvfRJfpChSlusu0lA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260430.1.tgz", - "integrity": "sha512-/DoYC/1wHs+YRZzzqSQg1/EHB4hiv1yV5U8FnmapRRIzVaPtnt+ApeOXeMrIdKidgKOI8TqQzgBU8xbIM7Cl4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260430.1.tgz", - "integrity": "sha512-koJhBWvEVZPKCVFtMLp2iMHlYr+lFCF47wGbnlKdHVlemV0zTxJEyHI8aLlrhPLhBmOmYLp46rXw09/qJkRIhQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260430.1.tgz", - "integrity": "sha512-hMdapNAzNQZDXGGkg4Slydc3fRJP5FUZLJVVcZCW/+imhhJro9Z1rv5n/wfR+txKoSWhTYR8eOp8Pyi2bzLzlw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260430.1.tgz", - "integrity": "sha512-jS3ffixjb5USOwz4frw4WzCz0HrjVxkgyU3WiYb06N7hBAfN6eOrveAJ4QRef0+suK4V1vQFoB1oKdRBsXe9Dw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260501.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260501.1.tgz", - "integrity": "sha512-B/VX2w3my/sCqxKyWOX7SxUpFC1uD8Gh7I2zbI1d3zA8p7Tx03AFsnuEx8lYLmcd8yONAA93YsAZb1wAaLK83w==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", - "cpu": [ - "s390x" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", - "cpu": [ - "x64" - ], - "dev": true, - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.7.0" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@poppinss/colors": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", - "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^4.1.5" - } - }, - "node_modules/@poppinss/dumper": { - "version": "0.6.5", - "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", - "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@sindresorhus/is": "^7.0.2", - "supports-color": "^10.0.0" - } - }, - "node_modules/@poppinss/exception": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", - "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sindresorhus/is": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", - "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, - "node_modules/@speed-highlight/core": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.15.tgz", - "integrity": "sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==", - "dev": true, - "license": "CC0-1.0" - }, - "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, - "node_modules/blake3-wasm": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", - "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true, - "license": "MIT" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/cookie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", - "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "license": "ISC", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/error-stack-parser-es": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", - "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/formdata-node/node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "license": "MIT" - }, - "node_modules/kleur": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", - "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/miniflare": { - "version": "4.20260430.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260430.0.tgz", - "integrity": "sha512-MWvMm3Siho9Yj7lbJZidLs8hbrRvIcOrif2mnsHQZdvoKfedpea+GaN8XJxbpRcq0B2WzNI1BB1ihdnqes3/ZA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", - "undici": "7.24.8", - "workerd": "1.20260430.1", - "ws": "8.18.0", - "youch": "4.1.0-beta.10" - }, - "bin": { - "miniflare": "bootstrap.js" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/openai": { - "version": "4.29.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", - "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "web-streams-polyfill": "^3.2.1" - }, - "bin": { - "openai": "bin/cli" - } - }, - "node_modules/path-to-regexp": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", - "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.0.0", - "detect-libc": "^2.1.2", - "semver": "^7.7.3" - }, - "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" - } - }, - "node_modules/supports-color": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", - "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/undici": { - "version": "7.24.8", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.8.tgz", - "integrity": "sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/unenv": { - "version": "2.0.0-rc.24", - "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", - "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "pathe": "^2.0.3" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/workerd": { - "version": "1.20260430.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260430.1.tgz", - "integrity": "sha512-KEgIWyiw3Jmn+DCd/L3ePo5fmiiYb/UcwKvDWPf/nLLOiwShDFzDSsegU5NY/JcwgvO/QsLHVi2FYrbkcXNY5Q==", - "dev": true, - "hasInstallScript": true, - "license": "Apache-2.0", - "bin": { - "workerd": "bin/workerd" - }, - "engines": { - "node": ">=16" - }, - "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260430.1", - "@cloudflare/workerd-darwin-arm64": "1.20260430.1", - "@cloudflare/workerd-linux-64": "1.20260430.1", - "@cloudflare/workerd-linux-arm64": "1.20260430.1", - "@cloudflare/workerd-windows-64": "1.20260430.1" - } - }, - "node_modules/wrangler": { - "version": "4.87.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.87.0.tgz", - "integrity": "sha512-lfhfKwLfQlowwgV0xhlYgE9fU3n0I30d4ccGY/rTCEm/n42Mjvlr0Ng3ZPNqlsrsKBcDR531V7dsPkgELvrk/Q==", - "dev": true, - "license": "MIT OR Apache-2.0", - "dependencies": { - "@cloudflare/kv-asset-handler": "0.5.0", - "@cloudflare/unenv-preset": "2.16.1", - "blake3-wasm": "2.1.5", - "esbuild": "0.27.3", - "miniflare": "4.20260430.0", - "path-to-regexp": "6.3.0", - "unenv": "2.0.0-rc.24", - "workerd": "1.20260430.1" - }, - "bin": { - "wrangler": "bin/wrangler.js", - "wrangler2": "bin/wrangler.js" - }, - "engines": { - "node": ">=22.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - }, - "peerDependencies": { - "@cloudflare/workers-types": "^4.20260430.1" - }, - "peerDependenciesMeta": { - "@cloudflare/workers-types": { - "optional": true - } - } - }, - "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/youch": { - "version": "4.1.0-beta.10", - "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", - "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/colors": "^4.1.5", - "@poppinss/dumper": "^0.6.4", - "@speed-highlight/core": "^1.2.7", - "cookie": "^1.0.2", - "youch-core": "^0.3.3" - } - }, - "node_modules/youch-core": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", - "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@poppinss/exception": "^1.2.2", - "error-stack-parser-es": "^1.0.5" - } - } - } -} diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json deleted file mode 100644 index 9fd94cd882f..00000000000 --- a/litellm-js/proxy/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "scripts": { - "dev": "wrangler dev src/index.ts", - "deploy": "wrangler deploy --minify src/index.ts" - }, - "dependencies": { - "hono": "4.12.16", - "openai": "4.29.2" - }, - "devDependencies": { - "@cloudflare/workers-types": "4.20260501.1", - "wrangler": "4.87.0" - } -} diff --git a/litellm-js/proxy/src/index.ts b/litellm-js/proxy/src/index.ts deleted file mode 100644 index dc5dc9c689e..00000000000 --- a/litellm-js/proxy/src/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Hono } from 'hono' -import { Context } from 'hono'; -import { bearerAuth } from 'hono/bearer-auth' -import OpenAI from "openai"; - -const openai = new OpenAI({ - apiKey: "sk-1234", - baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev" -}); - -async function call_proxy() { - const completion = await openai.chat.completions.create({ - messages: [{ role: "system", content: "You are a helpful assistant." }], - model: "gpt-3.5-turbo", - }); - - return completion -} - -const app = new Hono() - -// Middleware for API Key Authentication -const apiKeyAuth = async (c: Context, next: Function) => { - const apiKey = c.req.header('Authorization'); - if (!apiKey || apiKey !== 'Bearer sk-1234') { - return c.text('Unauthorized', 401); - } - await next(); -}; - - -app.use('/*', apiKeyAuth) - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - - - - -// Handler for chat completions -const chatCompletionHandler = async (c: Context) => { - // Assuming your logic for handling chat completion goes here - // For demonstration, just returning a simple JSON response - const response = await call_proxy() - return c.json(response); -}; - -// Register the above handler for different POST routes with the apiKeyAuth middleware -app.post('/v1/chat/completions', chatCompletionHandler); -app.post('/chat/completions', chatCompletionHandler); - -// Example showing how you might handle dynamic segments within the URL -// Here, using ':model*' to capture the rest of the path as a parameter 'model' -app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler); - - -export default app diff --git a/litellm-js/proxy/tsconfig.json b/litellm-js/proxy/tsconfig.json deleted file mode 100644 index 28fcfb58246..00000000000 --- a/litellm-js/proxy/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "lib": [ - "ESNext" - ], - "types": [ - "@cloudflare/workers-types" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - "skipLibCheck": true - }, -} \ No newline at end of file diff --git a/litellm-js/proxy/wrangler.toml b/litellm-js/proxy/wrangler.toml deleted file mode 100644 index e7c323dff97..00000000000 --- a/litellm-js/proxy/wrangler.toml +++ /dev/null @@ -1,18 +0,0 @@ -name = "my-app" -compatibility_date = "2023-12-01" - -# [vars] -# MY_VAR = "my-variable" - -# [[kv_namespaces]] -# binding = "MY_KV_NAMESPACE" -# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - -# [[r2_buckets]] -# binding = "MY_BUCKET" -# bucket_name = "my-bucket" - -# [[d1_databases]] -# binding = "DB" -# database_name = "my-database" -# database_id = "" diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc deleted file mode 100644 index 7999681cc35..00000000000 --- a/litellm-js/spend-logs/.npmrc +++ /dev/null @@ -1,5 +0,0 @@ -# Supply-chain hardening -# Packages needing lifecycle scripts: npm rebuild -ignore-scripts=true -# Protects local npm install only — npm ci (used in CI) ignores this -min-release-age=3 diff --git a/litellm-js/spend-logs/Dockerfile b/litellm-js/spend-logs/Dockerfile deleted file mode 100644 index 5040dc74bf6..00000000000 --- a/litellm-js/spend-logs/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Use the specific Node.js v20.11.0 image -FROM node:20.18.1-alpine3.20 - -# Set the working directory inside the container -WORKDIR /app - -# Copy package.json and package-lock.json to the working directory -COPY ./litellm-js/spend-logs/package*.json ./ - -# Install dependencies -RUN npm ci - -# Install Prisma globally -RUN npm install -g prisma - -# Copy the rest of the application code -COPY ./litellm-js/spend-logs . - -# Generate Prisma client -RUN npx prisma generate - -# Expose the port that the Node.js server will run on -EXPOSE 3000 - -# Command to run the Node.js app with npm run dev -CMD ["npm", "run", "dev"] diff --git a/litellm-js/spend-logs/README.md b/litellm-js/spend-logs/README.md deleted file mode 100644 index e12b31db70a..00000000000 --- a/litellm-js/spend-logs/README.md +++ /dev/null @@ -1,8 +0,0 @@ -``` -npm install -npm run dev -``` - -``` -open http://localhost:3000 -``` diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json deleted file mode 100644 index e33079766c9..00000000000 --- a/litellm-js/spend-logs/package-lock.json +++ /dev/null @@ -1,597 +0,0 @@ -{ - "name": "spend-logs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.13", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz", - "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@types/node": { - "version": "20.19.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz", - "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-tsconfig": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", - "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/hono": { - "version": "4.12.16", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", - "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json deleted file mode 100644 index 5a7a95c5de1..00000000000 --- a/litellm-js/spend-logs/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "scripts": { - "dev": "tsx watch src/index.ts" - }, - "dependencies": { - "@hono/node-server": "1.19.13", - "hono": "4.12.16" - }, - "devDependencies": { - "@types/node": "20.19.25", - "tsx": "4.20.6" - } -} diff --git a/litellm-js/spend-logs/schema.prisma b/litellm-js/spend-logs/schema.prisma deleted file mode 100644 index b0403f277aa..00000000000 --- a/litellm-js/spend-logs/schema.prisma +++ /dev/null @@ -1,29 +0,0 @@ -generator client { - provider = "prisma-client-js" -} - -datasource client { - provider = "postgresql" - url = env("DATABASE_URL") -} - -model LiteLLM_SpendLogs { - request_id String @id - call_type String - api_key String @default("") - spend Float @default(0.0) - total_tokens Int @default(0) - prompt_tokens Int @default(0) - completion_tokens Int @default(0) - startTime DateTime - endTime DateTime - model String @default("") - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") - team_id String? - end_user String? -} \ No newline at end of file diff --git a/litellm-js/spend-logs/src/_types.ts b/litellm-js/spend-logs/src/_types.ts deleted file mode 100644 index 6a9b499171e..00000000000 --- a/litellm-js/spend-logs/src/_types.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type LiteLLM_IncrementSpend = { - key_transactions: Array, // [{"key": spend},..] - user_transactions: Array, - team_transactions: Array, - spend_logs_transactions: Array -} - -export type LiteLLM_IncrementObject = { - key: string, - spend: number -} - -export type LiteLLM_SpendLogs = { - request_id: string; // @id means it's a unique identifier - call_type: string; - api_key: string; // @default("") means it defaults to an empty string if not provided - spend: number; // Float in Prisma corresponds to number in TypeScript - total_tokens: number; // Int in Prisma corresponds to number in TypeScript - prompt_tokens: number; - completion_tokens: number; - startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript - endTime: Date; - model: string; // @default("") means it defaults to an empty string if not provided - api_base: string; - user: string; - metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known - cache_hit: string; - cache_key: string; - request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure - team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided - end_user?: string | null; -}; \ No newline at end of file diff --git a/litellm-js/spend-logs/src/index.ts b/litellm-js/spend-logs/src/index.ts deleted file mode 100644 index 3581d95c830..00000000000 --- a/litellm-js/spend-logs/src/index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { serve } from '@hono/node-server' -import { Hono } from 'hono' -import { PrismaClient } from '@prisma/client' -import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types' - -const app = new Hono() -const prisma = new PrismaClient() -// In-memory storage for logs -let spend_logs: LiteLLM_SpendLogs[] = []; -const key_logs: LiteLLM_IncrementObject[] = []; -const user_logs: LiteLLM_IncrementObject[] = []; -const transaction_logs: LiteLLM_IncrementObject[] = []; - - -app.get('/', (c) => { - return c.text('Hello Hono!') -}) - -const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush -const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again -const BATCH_SIZE = 100; // Preferred size of each batch to write to the database -const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval - -const flushLogsToDb = async () => { - if (spend_logs.length >= MIN_LOGS) { - // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less - const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL); - - for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) { - // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE - const batch = logsToProcess.slice(i, i + BATCH_SIZE); - - // Convert datetime strings to Date objects - const batchWithDates = batch.map(entry => ({ - ...entry, - startTime: new Date(entry.startTime), - endTime: new Date(entry.endTime), - // Repeat for any other DateTime fields you may have - })); - - await prisma.liteLLM_SpendLogs.createMany({ - data: batchWithDates, - }); - - console.log(`Flushed ${batch.length} logs to the DB.`); - } - - // Remove the processed logs from spend_logs - spend_logs = spend_logs.slice(logsToProcess.length); - - console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`); - } else { - // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold. - if(spend_logs.length > 0) { - console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`); - } else { - console.log("No logs to flush."); - } - } -}; - -// Setup interval for attempting to flush the logs -setInterval(flushLogsToDb, FLUSH_INTERVAL); - -// Route to receive log messages -app.post('/spend/update', async (c) => { - const incomingLogs = await c.req.json(); - - spend_logs.push(...incomingLogs); - - console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`); - - return c.json({ message: `Successfully stored ${incomingLogs.length} logs` }); -}); - - - -const port = 3000 -console.log(`Server is running on port ${port}`) - -serve({ - fetch: app.fetch, - port -}) diff --git a/litellm-js/spend-logs/tsconfig.json b/litellm-js/spend-logs/tsconfig.json deleted file mode 100644 index 028c03b6a81..00000000000 --- a/litellm-js/spend-logs/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "types": [ - "node" - ], - "jsx": "react-jsx", - "jsxImportSource": "hono/jsx", - } -} \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index cf05fc4c980..e1b367fb234 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = ( ) store_audit_logs = False # Enterprise feature, allow users to see audit logs skip_system_message_in_guardrail: bool = False +skip_tool_message_in_guardrail: bool = False ### end of callbacks ############# email: Optional[str] = ( @@ -1425,6 +1426,12 @@ if TYPE_CHECKING: ) from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig + from .llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig as BedrockClaudePlatformConfig, + ) + from .llms.bedrock.claude_platform.messages_transformation import ( + BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig, + ) from .llms.anthropic.completion.transformation import ( AnthropicTextConfig as AnthropicTextConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 119e62a5b38..3531e8d96b9 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -131,6 +131,7 @@ LLM_CONFIG_NAMES = ( "OpenrouterConfig", "DataRobotConfig", "AnthropicConfig", + "BedrockClaudePlatformConfig", "AnthropicTextConfig", "GroqSTTConfig", "TritonConfig", @@ -170,6 +171,7 @@ LLM_CONFIG_NAMES = ( "SagemakerNovaConfig", "CohereChatConfig", "AnthropicMessagesConfig", + "BedrockClaudePlatformMessagesConfig", "AmazonAnthropicClaudeMessagesConfig", "AmazonMantleMessagesConfig", "TogetherAIConfig", @@ -610,6 +612,10 @@ _LLM_CONFIGS_IMPORT_MAP = { "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"), "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"), "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"), + "BedrockClaudePlatformConfig": ( + ".llms.bedrock.claude_platform.transformation", + "BedrockClaudePlatformConfig", + ), "AnthropicTextConfig": ( ".llms.anthropic.completion.transformation", "AnthropicTextConfig", @@ -712,6 +718,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.anthropic.experimental_pass_through.messages.transformation", "AnthropicMessagesConfig", ), + "BedrockClaudePlatformMessagesConfig": ( + ".llms.bedrock.claude_platform.messages_transformation", + "BedrockClaudePlatformMessagesConfig", + ), "AmazonAnthropicClaudeMessagesConfig": ( ".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeMessagesConfig", diff --git a/litellm/_redis.py b/litellm/_redis.py index f12afbac297..1c11ea829ba 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -19,6 +19,7 @@ import redis.asyncio as async_redis # type: ignore from litellm import get_secret, get_secret_str from litellm._redis_credential_provider import ( + AzureADCredentialProvider, GCPIAMCredentialProvider, _generate_gcp_iam_access_token, ) @@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from ._logging import verbose_logger +AZURE_REDIS_SCOPE = "https://redis.azure.com/.default" + def _get_redis_kwargs(): arg_spec = inspect.getfullargspec(redis.Redis) @@ -43,6 +46,10 @@ def _get_redis_kwargs(): "redis_connect_func", "gcp_service_account", "gcp_ssl_ca_certs", + "azure_redis_ad_token", + "azure_client_id", + "azure_tenant_id", + "azure_client_secret", ] available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args @@ -89,6 +96,10 @@ def _get_redis_cluster_kwargs(client=None): ) # Needed for sync clusters and IAM detection available_args.append("gcp_service_account") available_args.append("gcp_ssl_ca_certs") + available_args.append("azure_redis_ad_token") + available_args.append("azure_client_id") + available_args.append("azure_tenant_id") + available_args.append("azure_client_secret") available_args.append("max_connections") return available_args @@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func( return iam_connect +def _build_azure_credential( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +): + """ + Build a long-lived Azure credential object. + + Azure SDK credentials cache tokens internally and handle expiry/refresh + transparently, so this should be called once and the result reused. + """ + try: + from azure.identity import ( + ClientSecretCredential, + DefaultAzureCredential, + ManagedIdentityCredential, + ) + except ImportError: + raise ImportError( + "azure-identity is required for Azure AD Redis authentication. " + "Install it with: pip install azure-identity" + ) + + _client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID") + _tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID") + _client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET") + + if _client_id and _tenant_id and _client_secret: + return ClientSecretCredential( + client_id=_client_id, + tenant_id=_tenant_id, + client_secret=_client_secret, + ) + elif _client_id: + return ManagedIdentityCredential(client_id=_client_id) + else: + return DefaultAzureCredential() + + +def _generate_azure_ad_redis_token( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +) -> str: + """ + One-shot helper that builds a credential and fetches a single Azure AD + access token for Redis. Each call rebuilds the credential and performs a + network round-trip, so it should not be used in steady-state Redis flows + — the sync (``create_azure_ad_redis_connect_func``) and async paths + (``AzureADCredentialProvider``) keep the credential alive across + connections so the Azure SDK's internal cache + silent refresh apply. + """ + credential = _build_azure_credential( + azure_client_id=azure_client_id, + azure_tenant_id=azure_tenant_id, + azure_client_secret=azure_client_secret, + ) + token = credential.get_token(AZURE_REDIS_SCOPE) + return token.token + + +def create_azure_ad_redis_connect_func( + azure_client_id: Optional[str] = None, + azure_tenant_id: Optional[str] = None, + azure_client_secret: Optional[str] = None, +) -> Callable: + """ + Creates a custom Redis connection function for Azure AD authentication. + + Used for sync Redis clients. The credential is created once (captured by the + closure) and reused across connections — the Azure SDK handles token caching + and silent renewal internally. Only ``get_token`` is called per connection. + """ + credential = _build_azure_credential( + azure_client_id=azure_client_id, + azure_tenant_id=azure_tenant_id, + azure_client_secret=azure_client_secret, + ) + + def ad_connect(self): + """Initialize the connection and authenticate using Azure AD""" + from redis.exceptions import ( + AuthenticationError, + AuthenticationWrongNumberOfArgsError, + ) + from redis.utils import str_if_bytes + + self._parser.on_connect(self) + + access_token = credential.get_token(AZURE_REDIS_SCOPE).token + + # Only include username when explicitly set — sending AUTH "" + # is invalid for most ACL-configured Azure Redis instances. + username = os.environ.get("REDIS_USERNAME", "") + if username: + auth_args = (username, access_token) + else: + auth_args = (access_token,) + + self.send_command("AUTH", *auth_args, check_health=False) + + try: + auth_response = self.read_response() + except AuthenticationWrongNumberOfArgsError: + # Fallback: try with just the token (Redis < 6 / no ACL) + self.send_command("AUTH", access_token, check_health=False) + auth_response = self.read_response() + + if str_if_bytes(auth_response) != "OK": + raise AuthenticationError("Azure AD authentication failed for Redis") + + # Attach the live credential object so async paths can wrap it in + # AzureADCredentialProvider for refresh-aware token retrieval. The raw + # client_id/tenant_id/secret are intentionally NOT exposed here — the + # credential closure already holds them. + ad_connect._azure_credential = credential # type: ignore[attr-defined] + return ad_connect + + def get_redis_url_from_environment(): if "REDIS_URL" in os.environ: return os.environ["REDIS_URL"] @@ -179,7 +309,7 @@ def get_redis_url_from_environment(): return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}" -def _get_redis_client_logic(**env_overrides): +def _get_redis_client_logic(**env_overrides): # noqa: PLR0915 """ Common functionality across sync + async redis client implementations """ @@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides): if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False): redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs + # Handle Azure AD authentication (after GCP IAM block) + _azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret( + "REDIS_AZURE_AD_TOKEN" + ) + + _azure_ad_enabled = ( + _azure_redis_ad_token is not None + and str(_azure_redis_ad_token).lower() == "true" + ) + + if _azure_ad_enabled and _gcp_service_account is not None: + verbose_logger.warning( + "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. " + "Using GCP IAM. Remove one to avoid misconfiguration." + ) + + if _azure_ad_enabled and _gcp_service_account is None: + _azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str( + "AZURE_CLIENT_ID" + ) + _azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str( + "AZURE_TENANT_ID" + ) + _azure_client_secret = redis_kwargs.get( + "azure_client_secret" + ) or get_secret_str("AZURE_CLIENT_SECRET") + + verbose_logger.debug("Setting up Azure AD authentication for Redis.") + redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func( + azure_client_id=_azure_client_id, + azure_tenant_id=_azure_tenant_id, + azure_client_secret=_azure_client_secret, + ) + # Marker for async paths to detect Azure AD auth. The live credential + # object is attached separately as `_azure_credential` by + # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret + # are intentionally NOT exposed on the function to avoid leaking + # credentials via inspection or logging. + redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined] + + # Always remove Azure-specific kwargs that shouldn't be passed to Redis client + redis_kwargs.pop("azure_redis_ad_token", None) + redis_kwargs.pop("azure_client_id", None) + redis_kwargs.pop("azure_tenant_id", None) + redis_kwargs.pop("azure_client_secret", None) + if "url" in redis_kwargs and redis_kwargs["url"] is not None: # Only strip host/port/db/password when not routing to a cluster. # When startup_nodes is also present the cluster path takes priority and @@ -373,7 +549,7 @@ def get_redis_client(**env_overrides): return redis.Redis(**redis_kwargs) -def get_redis_async_client( +def get_redis_async_client( # noqa: PLR0915 connection_pool: Optional[async_redis.BlockingConnectionPool] = None, **env_overrides, ) -> Union[async_redis.Redis, async_redis.RedisCluster]: @@ -398,6 +574,14 @@ def get_redis_async_client( cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider( redis_connect_func._gcp_service_account ) + # Handle Azure AD authentication for async clusters via CredentialProvider + # so the credential's internal cache + silent refresh runs per connection + # (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry). + elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + cluster_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) new_startup_nodes: List[ClusterNode] = [] @@ -431,6 +615,22 @@ def get_redis_async_client( # Check for Redis Sentinel if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: return _init_async_redis_sentinel(redis_kwargs) + + # Wrap GCP / Azure AD auth in a CredentialProvider for the standard async + # Redis client. The async client doesn't support redis_connect_func, but it + # does honour credential_provider — which is called per connection, so the + # underlying SDK can refresh tokens silently before they expire. + redis_connect_func = redis_kwargs.pop("redis_connect_func", None) + if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + redis_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) + elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"): + redis_kwargs["credential_provider"] = GCPIAMCredentialProvider( + redis_connect_func._gcp_service_account + ) + _pretty_print_redis_config(redis_kwargs=redis_kwargs) if connection_pool is not None: @@ -464,6 +664,21 @@ def get_redis_connection_pool( redis_kwargs["max_connections"], ) return async_redis.BlockingConnectionPool.from_url(**pool_kwargs) + + # Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed + # connections re-fetch tokens via the SDK's internal cache + silent refresh + # rather than reusing a single token captured at pool creation. + redis_connect_func = redis_kwargs.pop("redis_connect_func", None) + if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"): + redis_kwargs["credential_provider"] = AzureADCredentialProvider( + redis_connect_func._azure_credential, + username=os.environ.get("REDIS_USERNAME") or None, + ) + elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"): + redis_kwargs["credential_provider"] = GCPIAMCredentialProvider( + redis_connect_func._gcp_service_account + ) + connection_class = async_redis.Connection if "ssl" in redis_kwargs: connection_class = async_redis.SSLConnection diff --git a/litellm/_redis_credential_provider.py b/litellm/_redis_credential_provider.py index 70725fe12c4..586b1c7716c 100644 --- a/litellm/_redis_credential_provider.py +++ b/litellm/_redis_credential_provider.py @@ -1,10 +1,13 @@ import asyncio import threading import time -from typing import Dict, Tuple +from typing import Any, Dict, Optional, Tuple, Union from redis.credentials import CredentialProvider # type: ignore[attr-defined] +# Azure AD scope for Redis Cache for Azure. +AZURE_REDIS_SCOPE = "https://redis.azure.com/.default" + # GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry. _GCP_IAM_TOKEN_TTL_SECONDS = 3300 @@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider): _get_cached_gcp_iam_token, self._gcp_service_account ) return (token,) + + +class AzureADCredentialProvider(CredentialProvider): + """ + redis.credentials.CredentialProvider implementation that supplies Azure AD + tokens for Redis authentication. + + Wraps an azure-identity credential object so the Azure SDK's internal token + cache and silent refresh are honoured on every Redis connection. This avoids + the static-token-baked-in-pool issue where pool-managed connections would + fail authentication after the initial token expired (~1 hour TTL). + """ + + def __init__(self, credential: Any, username: Optional[str] = None) -> None: + self._credential = credential + self._username = username + + def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]: + token = self._credential.get_token(AZURE_REDIS_SCOPE).token + if self._username: + return (self._username, token) + return (token,) + + async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]: + token_obj = await asyncio.to_thread( + self._credential.get_token, AZURE_REDIS_SCOPE + ) + if self._username: + return (self._username, token_obj.token) + return (token_obj.token,) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 259439d4d09..15ee9303969 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -617,24 +617,35 @@ def retrieve_batch( _is_async = kwargs.pop("aretrieve_batch", False) is True client = kwargs.get("client", None) - # Check if this is an async invoke ARN (different from regular batch ARN) - # Async invoke ARNs have format: arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:[0-9]{12}:async-invoke/[a-z0-9]{12} - if ( - batch_id.startswith("arn:aws") - and ":bedrock:" in batch_id - and ":async-invoke/" in batch_id - ): - # Handle async invoke status check - # Remove aws_region_name from kwargs to avoid duplicate parameter - async_kwargs = kwargs.copy() - async_kwargs.pop("aws_region_name", None) + # Bedrock has two distinct ARN families that need different APIs: + # * async-invoke ARNs (Twelve Labs Marengo embeddings) -> bedrock-runtime data plane + # * model-invocation-job ARNs (CreateModelInvocationJob batch) -> bedrock control plane + # They live on different AWS service endpoints and can't share a handler. + # ARN shapes: + # arn:aws(-[^:]+)?:bedrock:::async-invoke/ + # arn:aws(-[^:]+)?:bedrock:::model-invocation-job/ + if batch_id.startswith("arn:aws") and ":bedrock:" in batch_id: + if ":async-invoke/" in batch_id: + # Remove aws_region_name from kwargs to avoid duplicate parameter + async_kwargs = kwargs.copy() + async_kwargs.pop("aws_region_name", None) - return BedrockBatchesHandler._handle_async_invoke_status( - batch_id=batch_id, - aws_region_name=kwargs.get("aws_region_name", "us-east-1"), - logging_obj=litellm_logging_obj, - **async_kwargs, - ) + return BedrockBatchesHandler._handle_async_invoke_status( + batch_id=batch_id, + aws_region_name=kwargs.get("aws_region_name", "us-east-1"), + logging_obj=litellm_logging_obj, + **async_kwargs, + ) + if ":model-invocation-job/" in batch_id: + mij_kwargs = kwargs.copy() + mij_kwargs.pop("aws_region_name", None) + + return BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=batch_id, + aws_region_name=kwargs.get("aws_region_name"), + logging_obj=litellm_logging_obj, + **mij_kwargs, + ) # Try to use provider config first (for providers like bedrock) model: Optional[str] = kwargs.get("model", None) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index da3b9184edb..32423f23314 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -119,6 +119,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def __init__(self): pass + def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any: + """Chat tool_choice uses function.name; Responses API expects top-level name.""" + if not isinstance(tool_choice, dict) or tool_choice.get("type") != "function": + return tool_choice + if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): + # Return only Responses shape so stray chat ``function`` key is not sent upstream. + return {"type": "function", "name": tool_choice["name"]} + fn = tool_choice.get("function") + if isinstance(fn, dict): + fn_name = fn.get("name") + if isinstance(fn_name, str) and fn_name: + return {"type": "function", "name": fn_name} + return tool_choice + def _handle_raw_dict_response_item( self, item: Dict[str, Any], index: int ) -> Tuple[Optional[Any], int]: @@ -309,6 +323,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): text_format = self._transform_response_format_to_text_format(value) if text_format: responses_api_request["text"] = text_format # type: ignore + elif key == "tool_choice": + responses_api_request["tool_choice"] = ( # type: ignore[assignment] + self._normalize_tool_choice_for_responses_api(value) + ) elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key == "previous_response_id": diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 77833e5de0f..abce3c3e1c7 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request" RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" +CAPTURE_MODE_NO_CONTENT = "NO_CONTENT" +CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY" +CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY" +CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT" +_VALID_CAPTURE_MODES = { + CAPTURE_MODE_NO_CONTENT, + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, +} + @dataclass class OpenTelemetryConfig: @@ -71,6 +82,9 @@ class OpenTelemetryConfig: ignore_context_propagation: Optional[bool] = None # When True, create a private TracerProvider instead of reusing or setting the global one. skip_set_global: bool = False + # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT. + # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias). + capture_message_content: Optional[str] = None def __post_init__(self) -> None: # If endpoint is specified but exporter is still the default "console", @@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger): super().__init__(**kwargs) self._init_metrics(meter_provider) self._init_logs(logger_provider) + # Sample env-var / config / message_logging at init so subsequent + # _capture_in_span / _capture_in_event calls are deterministic. + self._capture_mode_cached = self._compute_capture_mode_from_init_state() self._init_otel_logger_on_litellm_proxy() @staticmethod @@ -306,6 +323,62 @@ class OpenTelemetry(CustomLogger): hasattr(self, "callback_name") and self.callback_name == "langfuse_otel" ) + def _compute_capture_mode_from_init_state(self) -> Optional[str]: + """Sample explicit settings at init. Returns the resolved mode or + None if nothing explicit is set (in which case the legacy + ``self.message_logging`` flag is consulted dynamically per request). + + ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention. + ``"false"``/``"0"`` map to ``NO_CONTENT``. + Unknown values are ignored. + """ + explicit = self.config.capture_message_content or os.getenv( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" + ) + if not explicit: + return None + normalized = explicit.upper() + if normalized in ("TRUE", "1"): + return CAPTURE_MODE_EVENT_ONLY + if normalized in ("FALSE", "0"): + return CAPTURE_MODE_NO_CONTENT + if normalized in _VALID_CAPTURE_MODES: + return normalized + return None + + def _resolve_capture_mode(self) -> str: + """Return the active capture mode for this request. + + Precedence: + 1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT`` + (kill-switch checked dynamically). + 2. Explicit setting sampled at init from + ``OpenTelemetryConfig.capture_message_content`` or + ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``. + 3. Legacy ``self.message_logging`` (checked dynamically). + """ + if litellm.turn_off_message_logging: + return CAPTURE_MODE_NO_CONTENT + if self._capture_mode_cached is not None: + return self._capture_mode_cached + return ( + CAPTURE_MODE_SPAN_AND_EVENT + if self.message_logging + else CAPTURE_MODE_NO_CONTENT + ) + + def _capture_in_span(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_SPAN_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + + def _capture_in_event(self) -> bool: + return self._resolve_capture_mode() in ( + CAPTURE_MODE_EVENT_ONLY, + CAPTURE_MODE_SPAN_AND_EVENT, + ) + def _init_tracing(self, tracer_provider): from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -825,8 +898,7 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - # only log raw LLM request/response if message_logging is on and not globally turned off - if litellm.turn_off_message_logging or not self.message_logging: + if not self._capture_in_span(): return litellm_params = kwargs.get("litellm_params", {}) @@ -1117,9 +1189,14 @@ class OpenTelemetry(CustomLogger): } if role == "tool" and msg.get("id"): attrs["id"] = msg["id"] - if self.message_logging and msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and msg.get("content"): attrs["gen_ai.prompt"] = msg["content"] + body = msg.copy() + if not capture_event_content: + body.pop("content", None) + log_record = SdkLogRecord( timestamp=self._to_ns(datetime.now()), trace_id=parent_ctx.trace_id, @@ -1127,7 +1204,7 @@ class OpenTelemetry(CustomLogger): trace_flags=parent_ctx.trace_flags, severity_number=SeverityNumber.INFO, severity_text="INFO", - body=msg.copy(), + body=body, attributes=attrs, ) otel_logger.emit(log_record) @@ -1141,14 +1218,15 @@ class OpenTelemetry(CustomLogger): "finish_reason": choice.get("finish_reason"), } body_msg = choice.get("message", {}) - if self.message_logging and body_msg.get("content"): + capture_event_content = self._capture_in_event() + if capture_event_content and body_msg.get("content"): attrs["message.content"] = body_msg["content"] body = { "index": idx, "finish_reason": choice.get("finish_reason"), "message": {"role": body_msg.get("role", "assistant")}, } - if self.message_logging and body_msg.get("content"): + if capture_event_content and body_msg.get("content"): body["message"]["content"] = body_msg["content"] log_record = SdkLogRecord( @@ -1674,9 +1752,7 @@ class OpenTelemetry(CustomLogger): ########## LLM Request Medssages / tools / content Attributes ########### ######################################################################### - if litellm.turn_off_message_logging is True: - return - if self.message_logging is not True: + if not self._capture_in_span(): return if optional_params.get("tools"): @@ -1695,17 +1771,41 @@ class OpenTelemetry(CustomLogger): value=safe_dumps(transformed_messages), ) - if kwargs.get("system_instructions"): - transformed_system_instructions = ( - self._transform_messages_to_otel_semantic_conventions( - kwargs.get("system_instructions") + # Coalesce the different kwarg names that carry the system + # prompt depending on the call path: + # - "system_instructions" — Vertex AI Gemini chat-completion + # - "instructions" — OpenAI Responses API + # - "system" — Anthropic Messages API + # Use `is not None` rather than truthiness to avoid falsy + # values (e.g. []) falling through to the wrong kwarg. + system_instructions = ( + kwargs.get("system_instructions") + if kwargs.get("system_instructions") is not None + else ( + kwargs.get("instructions") + if kwargs.get("instructions") is not None + else kwargs.get("system") + ) + ) + if system_instructions: + if isinstance(system_instructions, str): + # Plain text system prompt — no transformation needed + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, + value=system_instructions, + ) + else: + transformed_system_instructions = ( + self._transform_messages_to_otel_semantic_conventions( + system_instructions + ) + ) + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, + value=safe_dumps(transformed_system_instructions), ) - ) - self.safe_set_attribute( - span=span, - key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value, - value=safe_dumps(transformed_system_instructions), - ) self.safe_set_attribute( span=span, @@ -1764,6 +1864,57 @@ class OpenTelemetry(CustomLogger): value=value, ) + elif response_obj.get("output"): + # Responses API: ResponsesAPIResponse has an "output" + # list instead of "choices". Each item with + # type="message" contains a "content" list of + # OutputText objects (type="output_text"). + output_items = response_obj.get("output") + output_messages = self._transform_responses_api_output_to_otel( + output_items + ) + if output_messages: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value, + value=safe_dumps(output_messages), + ) + + # Emit per-tool-call span attributes (parity with + # the choices branch that calls _tool_calls_kv_pair). + # Convert Responses API function_call items to the + # ChatCompletionMessageToolCall format expected by + # _tool_calls_kv_pair. + tool_calls = [] + for out_item in output_items: + item_d = self._to_dict(out_item) + if item_d and item_d.get("type") == "function_call": + tool_calls.append( + { + "function": { + "name": item_d.get("name", ""), + "arguments": item_d.get("arguments", ""), + } + } + ) + if tool_calls: + kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore + for key, value in kv_pairs.items(): + self.safe_set_attribute( + span=span, + key=key, + value=value, + ) + + # Extract finish reason from ResponsesAPIResponse.status + status = response_obj.get("status") + if status: + self.safe_set_attribute( + span=span, + key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value, + value=safe_dumps([status]), + ) + except Exception as e: self.handle_callback_failure( callback_name=self.callback_name or "opentelemetry" @@ -1859,6 +2010,78 @@ class OpenTelemetry(CustomLogger): transformed.append(transformed_msg) return transformed + @staticmethod + def _to_dict(obj) -> Optional[dict]: + """Normalize an object to a plain dict. + + Handles three forms that appear in practice: + + 1. Plain ``dict`` — returned as-is. + 2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a + ``.get()`` method that delegates to ``__dict__``. + 3. Raw Pydantic v2 models from the ``openai`` SDK (e.g. + ``ResponseOutputMessage``, ``ResponseOutputText``) — these do + **not** have ``.get()`` but do have ``.model_dump()``. + + Returns ``None`` for anything else so callers can skip it. + """ + if isinstance(obj, dict): + return obj + if hasattr(obj, "get"): + # BaseLiteLLMOpenAIResponseObject duck-type + return obj # type: ignore[return-value] + if hasattr(obj, "model_dump"): + # Raw Pydantic v2 model (e.g. openai SDK types) + return obj.model_dump() # type: ignore[union-attr] + return None + + def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]: + """ + Transform Responses API output items into OTEL GenAI 1.38 format. + + The Responses API returns output as a list of items, each with a + ``type`` field. Message items (``type="message"``) contain a + ``content`` list of ``OutputText`` objects with ``type="output_text"`` + and ``text`` fields. + + Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``), + or raw Pydantic v2 models from the ``openai`` SDK (with + ``.model_dump()``). We normalize each item to a dict via + ``_to_dict`` before processing. + + This method converts them to the same ``{"role": ..., "parts": [...]}`` + format used by ``_transform_choices_to_otel_semantic_conventions``. + """ + transformed = [] + for raw_item in output: + item = self._to_dict(raw_item) + if item is None: + continue + if item.get("type") == "message": + role = item.get("role", "assistant") + parts = [] + for raw_content in item.get("content", []): + content = self._to_dict(raw_content) + if content is None: + continue + if content.get("type") == "output_text": + text = content.get("text", "") + if text: + parts.append({"type": "text", "content": text}) + if parts: + transformed.append({"role": role, "parts": parts}) + elif item.get("type") == "function_call": + # Surface tool calls from Responses API output + part: dict = { + "type": "tool_call", + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), + } + if item.get("call_id"): + part["id"] = item["call_id"] + transformed.append({"role": "assistant", "parts": [part]}) + return transformed + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): try: # Only set provider-specific raw payload attributes on this span. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a815442c2f9..c73d914e6cc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1212,7 +1212,7 @@ class Logging(LiteLLMLoggingBaseClass): # Log the exact result from the LLM API, for streaming - log the type of response received litellm.error_logs["POST_CALL"] = locals() if isinstance(original_response, dict): - original_response = json.dumps(original_response) + original_response = json.dumps(original_response, default=str) try: self.model_call_details["input"] = input self.model_call_details["api_key"] = api_key diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 2bb82f227bb..74dadee5ecb 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) chat_completion_compatible_request = self._translate_to_openai(data) @@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation): ) if skip_system: structured_messages = openai_messages_without_system(structured_messages) + if skip_tool: + structured_messages = openai_messages_without_tool(structured_messages) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check=images_to_check, task_mappings=task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts in batch @@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check: List[str], task_mappings: List[Tuple[int, Optional[int]]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content and images from a message. Override this method to customize text/image extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2f11a3fccb5..1ce80207552 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1809,9 +1809,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): Translate messages to anthropic format. """ ## VALIDATE REQUEST - """ - Anthropic doesn't support tool calling without `tools=` param specified. - """ + """Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed).""" from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) @@ -1821,16 +1819,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): and messages is not None and has_tool_call_blocks(messages) ): - if litellm.modify_params: - optional_params["tools"], _ = self._map_tools( - add_dummy_tool(custom_llm_provider="anthropic") - ) - else: - raise litellm.UnsupportedParamsError( - message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", - model="", - llm_provider="anthropic", - ) + optional_params["tools"], _ = self._map_tools( + add_dummy_tool(custom_llm_provider="anthropic") + ) # Drop thinking param if thinking is enabled but thinking_blocks are missing # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index cdd2d775371..97ece6b5eab 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool return bool(getattr(litellm, "skip_system_message_in_guardrail", False)) +def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool: + per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None) + if per is not None: + return bool(per) + import litellm + + return bool(getattr(litellm, "skip_tool_message_in_guardrail", False)) + + def openai_messages_without_system( messages: List[AllMessageValues], ) -> List[AllMessageValues]: return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"] + + +def openai_messages_without_tool( + messages: List[AllMessageValues], +) -> List[AllMessageValues]: + return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"] diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 0885775932c..9dd2b055a12 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1428,7 +1428,13 @@ class BaseAWSLLM: def _sign_request( self, - service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"], + service_name: Literal[ + "bedrock", + "sagemaker", + "bedrock-agentcore", + "s3vectors", + "aws-external-anthropic", + ], headers: dict, optional_params: dict, request_data: dict, diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index f141bbd9ab4..c071f331337 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -1,8 +1,79 @@ +from datetime import datetime +from typing import Any, Optional, cast + from openai.types.batch import BatchRequestCounts from openai.types.batch import Metadata as OpenAIBatchMetadata from litellm.types.utils import LiteLLMBatch +# AWS Bedrock model-invocation-job statuses → OpenAI Batch statuses. +# Mirrors the mapping used by `BedrockBatchesConfig.transform_create_batch_response` +# so create / retrieve return consistent statuses. +_BEDROCK_MIJ_STATUS_TO_OPENAI = { + "Submitted": "validating", + "Validating": "validating", + "Scheduled": "validating", + "InProgress": "in_progress", + "Stopping": "cancelling", + "Stopped": "cancelled", + "Completed": "completed", + "PartiallyCompleted": "completed", + "Failed": "failed", + "Expired": "expired", +} + + +def _extract_region_from_bedrock_arn(arn: str) -> Optional[str]: + """ARN shape: ``arn:aws:bedrock:::/``""" + try: + parts = arn.split(":") + if len(parts) >= 4 and parts[2] == "bedrock": + return parts[3] or None + except Exception: + pass + return None + + +def _extract_job_id_from_arn(arn: str) -> Optional[str]: + """``arn:aws:bedrock:::model-invocation-job/`` -> ````.""" + if ":model-invocation-job/" not in arn: + return None + return arn.rsplit("/", 1)[-1] or None + + +def _predict_output_file_uri( + output_prefix: str, input_uri: str, job_id: Optional[str] +) -> Optional[str]: + """ + Compute the deterministic per-job result file URI Bedrock writes to. + + Bedrock lays results out as:: + + //.out + + We compute it client-side so OpenAI-style ``client.files.content(output_file_id)`` + works without an extra S3 ``ListObjectsV2`` round-trip. Returns ``None`` if we + don't have enough info; callers should fall back to the bare prefix. + """ + if not output_prefix or not input_uri or not job_id: + return None + if not output_prefix.endswith("/"): + output_prefix = output_prefix + "/" + input_basename = input_uri.rsplit("/", 1)[-1] + if not input_basename: + return None + return f"{output_prefix}{job_id}/{input_basename}.out" + + +def _to_epoch(value: Any) -> Optional[int]: + if value is None: + return None + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, datetime): + return int(value.timestamp()) + return None + class BedrockBatchesHandler: """ @@ -97,3 +168,173 @@ class BedrockBatchesHandler: with concurrent.futures.ThreadPoolExecutor() as executor: future = executor.submit(run_in_thread) return future.result() + + @staticmethod + def _handle_model_invocation_job_status( + batch_id: str, + aws_region_name: Optional[str] = None, + logging_obj=None, + **kwargs, + ) -> "LiteLLMBatch": + """ + Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch + inference jobs (the ARN type returned by ``CreateModelInvocationJob``). + + ``CreateModelInvocationJob`` lives on the Bedrock **control plane** + (``bedrock..amazonaws.com``), distinct from the data-plane + ``bedrock-runtime`` endpoint that serves Twelve Labs async-invoke ARNs. + The two ARN families therefore can't share a handler — see + ``litellm/batches/main.py`` for the dispatch. + + Args: + batch_id: A ``arn:aws:bedrock:::model-invocation-job/`` + ARN (or just the trailing job id; both are accepted by + ``GetModelInvocationJob``). + aws_region_name: Region for the boto3 ``bedrock`` client. If omitted, + we fall back to parsing the region out of ``batch_id`` itself. + logging_obj: Optional litellm logging object. + **kwargs: Optional AWS credential overrides + (``aws_access_key_id``, ``aws_secret_access_key``, + ``aws_session_token``, ``aws_profile_name``, + ``aws_role_name``, ``aws_session_name``, + ``aws_web_identity_token``, ``aws_sts_endpoint``, + ``aws_external_id``). Unknown keys are ignored. + + Returns: + ``LiteLLMBatch`` shaped like an OpenAI Batch resource. Note that + ``request_counts`` is always ``(0, 0, 0)`` because + ``GetModelInvocationJob`` does not surface per-record counts; + callers that need accurate counts should parse + ``manifest.json.out`` from the output S3 prefix. + """ + try: + import boto3 + except ImportError as exc: + raise ImportError( + "Missing boto3 to call bedrock. Run 'pip install boto3'." + ) from exc + + # Resolve region: explicit > parsed-from-ARN > us-east-1 (boto3 default). + region = ( + aws_region_name or _extract_region_from_bedrock_arn(batch_id) or "us-east-1" + ) + + # Resolve credentials through the same path the rest of the bedrock + # provider uses, so model_list / env / role-assumption configs are + # honored. We instantiate BedrockBatchesConfig (which extends + # BaseAWSLLM) lazily to avoid a circular import at module load. + from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig + + creds = BedrockBatchesConfig().get_credentials( + aws_access_key_id=kwargs.get("aws_access_key_id"), + aws_secret_access_key=kwargs.get("aws_secret_access_key"), + aws_session_token=kwargs.get("aws_session_token"), + aws_region_name=region, + aws_session_name=kwargs.get("aws_session_name"), + aws_profile_name=kwargs.get("aws_profile_name"), + aws_role_name=kwargs.get("aws_role_name"), + aws_web_identity_token=kwargs.get("aws_web_identity_token"), + aws_sts_endpoint=kwargs.get("aws_sts_endpoint"), + aws_external_id=kwargs.get("aws_external_id"), + ) + + client = boto3.client( + "bedrock", + region_name=region, + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + aws_session_token=creds.token, + ) + + if logging_obj is not None: + # Use the bare job id in the logged URL so we don't double up the + # `model-invocation-job/` segment when `batch_id` is a full ARN. + # `GetModelInvocationJob` accepts either form, but only the bare id + # produces a sensible-looking URL in logs. + url_path_id = _extract_job_id_from_arn(batch_id) or batch_id + logging_obj.pre_call( + input=batch_id, + api_key="", + additional_args={ + "complete_input_dict": {"jobIdentifier": batch_id}, + "api_base": ( + f"https://bedrock.{region}.amazonaws.com/" + f"model-invocation-job/{url_path_id}" + ), + }, + ) + + response = client.get_model_invocation_job(jobIdentifier=batch_id) + + if logging_obj is not None: + logging_obj.post_call( + input=batch_id, + api_key="", + original_response=response, + additional_args={"complete_input_dict": {"jobIdentifier": batch_id}}, + ) + + bedrock_status = str(response.get("status", "")) + openai_status = cast( + Any, + _BEDROCK_MIJ_STATUS_TO_OPENAI.get(bedrock_status, "in_progress"), + ) + + input_uri = ( + response.get("inputDataConfig", {}) + .get("s3InputDataConfig", {}) + .get("s3Uri", "") + ) + output_prefix = ( + response.get("outputDataConfig", {}) + .get("s3OutputDataConfig", {}) + .get("s3Uri", "") + ) + + # Bedrock returns the output *prefix* the user supplied at job creation. + # Actual results land at //.out — we + # surface that single-file URI as `output_file_id` so the OpenAI-style + # download flow works without an extra S3 listing call. We deliberately + # do NOT fall back to the bare prefix when prediction fails: a prefix + # is not a downloadable object, so handing it back as `output_file_id` + # would reproduce the very NoSuchKey bug this handler exists to fix. + # The bare prefix is preserved in metadata for callers that want the + # `manifest.json.out` or want to do their own listing. + job_arn = response.get("jobArn", batch_id) + job_id = _extract_job_id_from_arn(job_arn) + output_file_uri = _predict_output_file_uri(output_prefix, input_uri, job_id) + + completed_at = _to_epoch(response.get("endTime")) + + # Note: metadata uses "" (not None) for unknown URIs to satisfy the + # OpenAI Batch metadata schema, which is `dict[str, str]`. The + # `output_file_id` field on the LiteLLMBatch itself does carry None + # correctly (see below), so callers should branch on that, not on + # `metadata["output_file_uri"]`. + openai_batch_metadata: OpenAIBatchMetadata = { + "model_arn": response.get("modelId", ""), + "job_arn": job_arn, + "job_name": response.get("jobName", ""), + "failure_message": response.get("message") or "", + "input_s3_uri": input_uri, + "output_s3_uri": output_prefix, + "output_file_uri": output_file_uri or "", + } + + return LiteLLMBatch( + id=job_arn, + object="batch", + status=openai_status, + created_at=_to_epoch(response.get("submitTime")) or 0, + in_progress_at=_to_epoch(response.get("lastModifiedTime")), + completed_at=completed_at if openai_status == "completed" else None, + failed_at=completed_at if openai_status == "failed" else None, + cancelled_at=completed_at if openai_status == "cancelled" else None, + expired_at=completed_at if openai_status == "expired" else None, + request_counts=BatchRequestCounts(total=0, completed=0, failed=0), + metadata=openai_batch_metadata, + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=input_uri, + output_file_id=output_file_uri if openai_status == "completed" else None, + ) diff --git a/litellm/llms/bedrock/claude_platform/__init__.py b/litellm/llms/bedrock/claude_platform/__init__.py new file mode 100644 index 00000000000..88d4e9783c7 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/__init__.py @@ -0,0 +1,8 @@ +from .transformation import ( + BedrockClaudePlatformConfig, +) +from .messages_transformation import ( + BedrockClaudePlatformMessagesConfig, +) + +__all__ = ["BedrockClaudePlatformConfig", "BedrockClaudePlatformMessagesConfig"] diff --git a/litellm/llms/bedrock/claude_platform/common_utils.py b/litellm/llms/bedrock/claude_platform/common_utils.py new file mode 100644 index 00000000000..121221518c8 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/common_utils.py @@ -0,0 +1,107 @@ +from typing import Literal, Optional, Tuple + +import litellm +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.secret_managers.main import get_secret_str + + +CLAUDE_PLATFORM_SERVICE_NAME: Literal["aws-external-anthropic"] = ( + "aws-external-anthropic" +) +CLAUDE_PLATFORM_BEDROCK_ROUTE = "claude_platform/" + + +def strip_claude_platform_route(model: str) -> str: + if model.startswith(CLAUDE_PLATFORM_BEDROCK_ROUTE): + return model.replace(CLAUDE_PLATFORM_BEDROCK_ROUTE, "", 1) + return model + + +class BedrockClaudePlatformMixin(BaseAWSLLM): + @staticmethod + def _get_workspace_id(optional_params: dict, litellm_params: dict) -> Optional[str]: + workspace_id = ( + optional_params.get("workspace_id") + or litellm_params.get("workspace_id") + or optional_params.get("aws_workspace_id") + or litellm_params.get("aws_workspace_id") + or optional_params.get("anthropic-workspace-id") + or litellm_params.get("anthropic-workspace-id") + ) + if workspace_id is None: + workspace_id = optional_params.get( + "anthropic_workspace_id" + ) or litellm_params.get("anthropic_workspace_id") + if workspace_id is not None: + return str(workspace_id) + return get_secret_str("ANTHROPIC_AWS_WORKSPACE_ID") or get_secret_str( + "ANTHROPIC_WORKSPACE_ID" + ) + + def _get_required_aws_region_name(self, optional_params: dict) -> str: + aws_region_name = ( + optional_params.get("aws_region_name") + or get_secret_str("AWS_REGION_NAME") + or get_secret_str("AWS_REGION") + or get_secret_str("AWS_DEFAULT_REGION") + ) + if aws_region_name is None: + raise litellm.AuthenticationError( + message=( + "Missing AWS region for Claude Platform on AWS. Pass " + "`aws_region_name` or set a standard AWS region environment value." + ), + llm_provider="bedrock", + model="", + ) + self._validate_aws_region_name(str(aws_region_name)) + return str(aws_region_name) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + api_base = ( + api_base + or litellm.api_base + or get_secret_str("ANTHROPIC_AWS_BASE_URL") + or get_secret_str("ANTHROPIC_AWS_API_BASE") + ) + if api_base is None: + aws_region_name = self._get_required_aws_region_name(optional_params) + api_base = ( + f"https://{CLAUDE_PLATFORM_SERVICE_NAME}.{aws_region_name}.api.aws" + ) + if not api_base.endswith("/v1/messages"): + api_base = f"{api_base.rstrip('/')}/v1/messages" + return api_base + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + if api_key or get_secret_str("ANTHROPIC_AWS_API_KEY"): + return headers, None + + return self._sign_request( + service_name=CLAUDE_PLATFORM_SERVICE_NAME, + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + ) diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py new file mode 100644 index 00000000000..66158196322 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -0,0 +1,71 @@ +from typing import Any, Dict, List, Optional, Tuple + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +from .common_utils import BedrockClaudePlatformMixin, strip_claude_platform_route + + +class BedrockClaudePlatformMessagesConfig( + BedrockClaudePlatformMixin, AnthropicMessagesConfig +): + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + workspace_id = self._get_workspace_id(optional_params, litellm_params) + if workspace_id is None: + raise litellm.AuthenticationError( + message=( + "Missing workspace ID for Claude Platform on AWS. Pass " + "`workspace_id` or configure the provider workspace setting." + ), + llm_provider="bedrock", + model=model, + ) + + resolved_api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY") + headers = { + **headers, + "anthropic-version": headers.get( + "anthropic-version", DEFAULT_ANTHROPIC_API_VERSION + ), + "content-type": headers.get("content-type", "application/json"), + "anthropic-workspace-id": workspace_id, + } + if resolved_api_key and "x-api-key" not in headers: + headers["x-api-key"] = resolved_api_key + + headers = self._update_headers_with_anthropic_beta( + headers=headers, + optional_params=optional_params, + ) + + return headers, api_base + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + return super().transform_anthropic_messages_request( + model=strip_claude_platform_route(model), + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py new file mode 100644 index 00000000000..0167c457c96 --- /dev/null +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -0,0 +1,94 @@ +from typing import Any, Dict, List, Optional + +import litellm +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from .common_utils import BedrockClaudePlatformMixin + + +class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): + """ + Bedrock Claude Platform uses Anthropic's Messages API with AWS gateway auth. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Dict: + workspace_id = self._get_workspace_id(optional_params, litellm_params) + if workspace_id is None: + raise litellm.AuthenticationError( + message=( + "Missing workspace ID for Claude Platform on AWS. Pass " + "`workspace_id` or configure the provider workspace setting." + ), + llm_provider="bedrock", + model=model, + ) + + api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY") + anthropic_headers = self.get_anthropic_headers( + api_key=api_key, + auth_token=None, + computer_tool_used=self.is_computer_tool_used( + tools=optional_params.get("tools") + ), + prompt_caching_set=self.is_cache_control_set(messages=messages), + pdf_used=self.is_pdf_used(messages=messages), + file_id_used=self.is_file_id_used(messages=messages), + mcp_server_used=self.is_mcp_server_used( + mcp_servers=optional_params.get("mcp_servers") + ), + web_search_tool_used=self.is_web_search_tool_used( + tools=optional_params.get("tools") + ), + tool_search_used=self.is_tool_search_used( + tools=optional_params.get("tools") + ), + programmatic_tool_calling_used=self.is_programmatic_tool_calling_used( + tools=optional_params.get("tools") + ), + input_examples_used=self.is_input_examples_used( + tools=optional_params.get("tools") + ), + effort_used=self.is_effort_used( + optional_params=optional_params, model=model + ), + user_anthropic_beta_headers=self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ), + code_execution_tool_used=self.is_code_execution_tool_used( + tools=optional_params.get("tools") + ), + container_with_skills_used=self.is_container_with_skills_used( + optional_params=optional_params + ), + ) + anthropic_headers["anthropic-workspace-id"] = workspace_id + return {**headers, **anthropic_headers} + + def get_model_response_iterator( + self, + streaming_response: Any, + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + return ModelResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=bool(json_mode), + ) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 856a525f773..0256d5d4b95 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -692,6 +692,7 @@ class BedrockModelInfo(BaseLLMModelInfo): ) -> Literal[ "converse", "invoke", + "claude_platform", "converse_like", "agent", "agentcore", @@ -706,6 +707,7 @@ class BedrockModelInfo(BaseLLMModelInfo): str, Literal[ "invoke", + "claude_platform", "converse_like", "converse", "agent", @@ -716,6 +718,7 @@ class BedrockModelInfo(BaseLLMModelInfo): ], ] = { "invoke/": "invoke", + "claude_platform/": "claude_platform", "converse_like/": "converse_like", "converse/": "converse", "agent/": "agent", @@ -753,6 +756,36 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "converse/" in model + @staticmethod + def _explicit_claude_platform_route(model: str) -> bool: + """ + Check if the model is an explicit Claude Platform on AWS route. + """ + return "claude_platform/" in model + + @staticmethod + def get_claude_platform_model(model: str) -> str: + """ + Strip the Claude Platform route prefix from a Bedrock model name. + """ + return model.replace("claude_platform/", "", 1) + + @staticmethod + def map_claude_platform_auth_params( + passed_params: dict, optional_params: dict + ) -> dict: + """ + Map Claude Platform route auth params that are not OpenAI request params. + """ + for key in ( + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + ): + if key in passed_params: + optional_params[key] = passed_params[key] + return optional_params + @staticmethod def _explicit_invoke_route(model: str) -> bool: """ @@ -815,6 +848,12 @@ class BedrockModelInfo(BaseLLMModelInfo): All other routes should return None since they will go through litellm.completion """ + ######################################################### + # Claude Platform route uses Anthropic Messages API via the AWS gateway. + ######################################################### + if BedrockModelInfo._explicit_claude_platform_route(model): + return litellm.BedrockClaudePlatformMessagesConfig() + ######################################################### # Converse routes should go through litellm.completion() if BedrockModelInfo._explicit_converse_route(model): @@ -860,7 +899,9 @@ def get_bedrock_chat_config(model: str): base_model = BedrockModelInfo.get_base_model(model) # Handle explicit routes first - if bedrock_route == "converse" or bedrock_route == "converse_like": + if bedrock_route == "claude_platform": + return litellm.BedrockClaudePlatformConfig() + elif bedrock_route == "converse" or bedrock_route == "converse_like": return litellm.AmazonConverseConfig() elif bedrock_route == "openai": return litellm.AmazonBedrockOpenAIConfig() diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index aae2bc5e289..151e0e404a0 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -408,6 +408,47 @@ class AmazonAnthropicClaudeMessagesConfig( if self._supports_tool_search_on_bedrock(model): beta_set.add("tool-search-tool-2025-10-19") + @staticmethod + def _filter_context_management_for_bedrock_invoke( + anthropic_messages_request: Dict, + beta_set: set, + ) -> None: + """ + Bedrock InvokeModel accepts ``context_management`` only when it carries + ``compact_20260112`` edits paired with the ``compact-2026-01-12`` + anthropic-beta header. Other edit types (notably ``clear_thinking_20251015``, + which Claude Code sends on every request) are LiteLLM-internal and would + cause Bedrock to 400 with ``"context_management: Extra inputs are not + permitted"``. + + Filter the edits list to the supported subset, add the beta header when + compact edits remain, and drop ``context_management`` entirely when no + supported edits are left so the safety-net allowlist can pass it through. + + Ref: https://github.com/BerriAI/litellm/issues/27532 + """ + cm = anthropic_messages_request.get("context_management") + if not isinstance(cm, dict): + return + edits = cm.get("edits") + if not isinstance(edits, list): + anthropic_messages_request.pop("context_management", None) + return + + compact_edits = [ + e + for e in edits + if isinstance(e, dict) and e.get("type") == "compact_20260112" + ] + if compact_edits: + beta_set.add("compact-2026-01-12") + anthropic_messages_request["context_management"] = { + **cm, + "edits": compact_edits, + } + else: + anthropic_messages_request.pop("context_management", None) + def _convert_output_format_to_inline_schema( self, output_format: Dict, @@ -551,6 +592,11 @@ class AmazonAnthropicClaudeMessagesConfig( if injected_thinking_for_clear_thinking: beta_set.add("interleaved-thinking-2025-05-14") + self._filter_context_management_for_bedrock_invoke( + anthropic_messages_request=anthropic_messages_request, + beta_set=beta_set, + ) + self._get_tool_search_beta_header_for_bedrock( model=model, tool_search_used=tool_search_used, @@ -597,8 +643,9 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_request.pop("output_config", None) # 7. Final safety net: filter top-level fields to the Bedrock Invoke allowlist. - # Catches Anthropic-only extensions (context_management, output_config, speed, - # mcp_servers, ...) and any future additions Claude Code may start sending. + # Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...) + # and any future additions Claude Code may start sending. ``context_management`` + # has already been pre-filtered to its Bedrock-supported subset above. allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS stripped = sorted(k for k in anthropic_messages_request if k not in allowed) if stripped: diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 4e34d10b187..9ccb2e1c267 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -3,7 +3,10 @@ from typing import Optional, Union import litellm -from litellm.utils import _is_explicitly_disabled_factory, _supports_factory +from litellm.utils import ( + _is_explicitly_disabled_factory, + _supports_factory, +) from .gpt_transformation import OpenAIGPTConfig diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 86ca6625629..d413a244539 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply) + skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply) texts_to_check: List[str] = [] images_to_check: List[str] = [] @@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings=text_task_mappings, tool_call_task_mappings=tool_call_task_mappings, skip_system_message=skip_system, + skip_tool_message=skip_tool, ) # Step 2: Apply guardrail to all texts and tool calls in batch @@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["tool_calls"] = tool_calls_to_check # type: ignore structured_messages = self.get_structured_messages(data) if structured_messages: - inputs["structured_messages"] = ( - openai_messages_without_system(structured_messages) - if skip_system - else structured_messages - ) + if skip_system: + structured_messages = openai_messages_without_system( + structured_messages + ) + if skip_tool: + structured_messages = openai_messages_without_tool( + structured_messages + ) + inputs["structured_messages"] = structured_messages # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: @@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): text_task_mappings: List[Tuple[int, Optional[int]]], tool_call_task_mappings: List[Tuple[int, int]], skip_system_message: bool = False, + skip_tool_message: bool = False, ) -> None: """ Extract text content, images, and tool calls from a message. Override this method to customize text/image/tool call extraction logic. """ - if skip_system_message and str(message.get("role") or "").lower() == "system": + role = str(message.get("role") or "").lower() + if skip_system_message and role == "system": + return + if skip_tool_message and role == "tool": return content = message.get("content", None) diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index 7ff6dc986be..f49f31d7ecd 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -156,5 +156,17 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): text = response_json.get("text") or response_json.get("transcript") or "" response = TranscriptionResponse(text=text) + # OVHCloud field migration (deadline: 2026-05-11): + # `duration` is replaced by `seconds` in STT responses. + # Prefer `seconds`, fall back to `duration`, normalize to `duration` + # so downstream consumers see a consistent key. + duration = ( + response_json["seconds"] + if "seconds" in response_json and response_json["seconds"] is not None + else response_json.get("duration") + ) + if duration is not None: + response_json["duration"] = duration + response._hidden_params = response_json return response diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index ae9271ddb16..62f51f1e9da 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -13,6 +13,7 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException + from litellm.types.llms.openai import AllMessageValues @@ -98,10 +99,16 @@ class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator): new_choices = [] for choice in chunk["choices"]: - if "delta" in choice and "reasoning" in choice["delta"]: - choice["delta"]["reasoning_content"] = choice["delta"].get( - "reasoning" - ) + if "delta" in choice: + delta = choice["delta"] + # OVHCloud field migration (deadline: 2026-05-11): + # `reasoning_content` is replaced by `reasoning`. + # Normalise to `reasoning_content` so downstream consumers + # see a consistent key during the transition window. + reasoning_new = delta.get("reasoning") + reasoning_legacy = delta.get("reasoning_content") + if reasoning_new is not None and reasoning_legacy is None: + delta["reasoning_content"] = reasoning_new new_choices.append(choice) return ModelResponseStream( diff --git a/litellm/main.py b/litellm/main.py index 051a82fdd19..c3d1c2e05b0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1,3 +1,5 @@ +# LiteLLM main module: public completion, embedding, streaming, and moderation entrypoints. +# # +-----------------------------------------------+ # | | # | Give Feedback / Get Help | @@ -59,7 +61,13 @@ import litellm from litellm import client # Other utils are imported directly to avoid circular imports -from litellm.utils import exception_type, get_litellm_params, get_optional_params +from litellm.utils import ( + exception_type, + get_litellm_params, + get_optional_params, + peek_reasoning_summary_aliases, + strip_reasoning_summary_aliases_from_optional_params, +) # Logging is imported lazily when needed to avoid loading litellm_logging at import time if TYPE_CHECKING: @@ -946,6 +954,7 @@ def responses_api_bridge_check( web_search_options: Optional[OpenAIWebSearchOptions] = None, tools: Optional[List[Any]] = None, reasoning_effort: Optional[Any] = None, + reasoning_summary: Optional[Any] = None, ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} @@ -982,14 +991,23 @@ def responses_api_bridge_check( mode = "responses" model_info["mode"] = mode - # OpenAI/Azure gpt-5.4+ chat-completions calls with both tools + reasoning_effort - # must be bridged to Responses API. + # OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g. + # ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects + # those keys. + # + # - gpt-5.4+: tools + reasoning_effort (original) or any reasoning-summary alias. + # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning + # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). if ( custom_llm_provider in ("openai", "azure") - and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) - and tools - and reasoning_effort is not None and model_info.get("mode") != "responses" + and OpenAIGPT5Config.is_model_gpt_5_model(model) + and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) + and reasoning_effort is not None + and ( + reasoning_summary is not None + or (OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) and tools) + ) ): model_info["mode"] = "responses" model = model.replace("responses/", "") @@ -1510,7 +1528,11 @@ def completion( # type: ignore # noqa: PLR0915 "logit_bias": logit_bias, "user": user, # params to identify the model - "model": model, + "model": ( + model_info.get("base_model") + if isinstance(model_info, dict) and model_info.get("base_model") + else model + ), "custom_llm_provider": custom_llm_provider, "response_format": response_format, "seed": seed, @@ -1634,8 +1656,10 @@ def completion( # type: ignore # noqa: PLR0915 ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map # Only run the second bridge check if the first one didn't already # detect responses mode (e.g. via the "responses/" prefix). The second - # check handles cases like gpt-5.4+ with tools+reasoning_effort that - # the first (early) check doesn't cover. + # check handles cases like gpt-5.4+ with tools+reasoning_effort or + # reasoningSummary/reasoning_summary without tools (AI SDK) that the first + # (early) check doesn't cover. + _reasoning_summary_for_bridge = peek_reasoning_summary_aliases(optional_params) if responses_api_model_info.get("mode") != "responses": responses_api_model_info, model = responses_api_bridge_check( model=model, @@ -1643,14 +1667,29 @@ def completion( # type: ignore # noqa: PLR0915 web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, + reasoning_summary=_reasoning_summary_for_bridge, ) if responses_api_model_info.get("mode") == "responses": from litellm.completion_extras import responses_api_bridge + optional_params, rs_val = ( + strip_reasoning_summary_aliases_from_optional_params(optional_params) + ) + if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: - optional_params = dict(optional_params) optional_params["reasoning_effort"] = reasoning_effort + elif rs_val is not None: + eff = optional_params.get("reasoning_effort", reasoning_effort) + if isinstance(eff, dict): + optional_params["reasoning_effort"] = {**eff, "summary": rs_val} + elif eff is not None: + optional_params["reasoning_effort"] = { + "effort": eff, + "summary": rs_val, + } + else: + optional_params["reasoning_effort"] = {"summary": rs_val} return responses_api_bridge.completion( model=model, @@ -1669,6 +1708,16 @@ def completion( # type: ignore # noqa: PLR0915 encoding=_get_encoding(), stream=stream, ) + elif ( + custom_llm_provider == "openai" + and OpenAIGPT5Config.is_model_gpt_5_model(model) + ) or ( + custom_llm_provider == "azure" + and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model) + ): + optional_params, _ = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) if custom_llm_provider == "azure": # azure configs @@ -3813,7 +3862,33 @@ def completion( # type: ignore # noqa: PLR0915 ) bedrock_route = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "converse": + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + return response + elif bedrock_route == "converse": model = model.replace("converse/", "") response = bedrock_converse_chat_completion.completion( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4fba1980103..8df25d2c9b5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21104,6 +21104,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -27187,6 +27219,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 1794cd14381..62691641234 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -12,7 +12,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( TOKEN_NO_CACHE_HEADERS, - validate_loopback_redirect_uri, + get_request_base_url, + validate_trusted_redirect_uri, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -29,51 +30,6 @@ router = APIRouter( ) -def get_request_base_url(request: Request) -> str: - """ - Get the base URL for the request, considering X-Forwarded-* headers. - - X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured - when the request comes from a configured trusted proxy - (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``). - Otherwise the request's literal ``base_url`` is returned, so an - untrusted caller cannot poison OAuth-discovery / redirect_uri values - by injecting headers. - - Args: - request: FastAPI Request object - - Returns: - The reconstructed base URL (e.g., "https://proxy.example.com") - """ - base_url = str(request.base_url).rstrip("/") - parsed = urlparse(base_url) - - if not IPAddressUtils.is_request_from_trusted_proxy(request): - return base_url - - x_forwarded_proto = request.headers.get("X-Forwarded-Proto") - x_forwarded_host = request.headers.get("X-Forwarded-Host") - x_forwarded_port = request.headers.get("X-Forwarded-Port") - - scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme - - if x_forwarded_host: - # X-Forwarded-Host may already include port (e.g., "example.com:8080") - if ":" in x_forwarded_host and not x_forwarded_host.startswith("["): - netloc = x_forwarded_host - elif x_forwarded_port: - netloc = f"{x_forwarded_host}:{x_forwarded_port}" - else: - netloc = x_forwarded_host - else: - netloc = parsed.netloc - if x_forwarded_port and ":" not in netloc: - netloc = f"{netloc}:{x_forwarded_port}" - - return urlunparse((scheme, netloc, parsed.path, "", "", "")) - - def encode_state_with_base_url( base_url: str, original_state: str, @@ -127,12 +83,14 @@ def decode_state_hash(encrypted_state: str) -> dict: return state_data -def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str: - """Return a loopback client redirect URI from OAuth state.""" +def _get_validated_client_redirect_uri( + request: Request, state_data: Dict[str, Any] +) -> str: + """Return a trusted (same-origin or loopback) client redirect URI from OAuth state.""" redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url") if not redirect_uri or not isinstance(redirect_uri, str): raise HTTPException(status_code=400, detail="Invalid redirect URI") - validate_loopback_redirect_uri(redirect_uri) + validate_trusted_redirect_uri(request, redirect_uri) return redirect_uri @@ -338,12 +296,12 @@ async def authorize_with_server( status_code=400, detail="MCP server authorization url is not set" ) - # Loopback-only redirect_uri. The URI is encrypted into the OAuth - # state and decoded on /callback to redirect the user back; a non- - # loopback URI would be an open-redirect + code-theft primitive - # (VERIA-57 root cause B). MCP clients are native apps — loopback is - # the spec-compliant callback pattern. - validate_loopback_redirect_uri(redirect_uri) + # Loopback OR same-origin redirect_uri. The URI is encrypted into the + # OAuth state and decoded on /callback to redirect the user back; + # restricting to trusted origins blocks the open-redirect + + # code-theft primitive (VERIA-57 root cause B). Loopback supports + # native MCP clients; same-origin supports the proxy's own UI callback. + validate_trusted_redirect_uri(request, redirect_uri) parsed = urlparse(redirect_uri) base_url = urlunparse(parsed._replace(query="")) request_base_url = get_request_base_url(request) @@ -660,17 +618,18 @@ async def token_endpoint( @router.get("/callback") -async def callback(code: str, state: str): +async def callback(request: Request, code: str, state: str): try: state_data = decode_state_hash(state) original_state = state_data["original_state"] - # Re-validate loopback at the sink. /authorize rejects non-loopback + # Re-validate at the sink. /authorize rejects untrusted # redirect_uri before encoding into state, but encrypted states # minted before that check was added have no expiry and remain - # valid indefinitely. Validating here blocks the open-redirect + - # code-theft primitive even for pre-fix states. - redirect_uri = _get_validated_client_redirect_uri(state_data) + # valid indefinitely. Validating here (same-origin OR loopback) + # blocks the open-redirect + code-theft primitive even for pre-fix + # states while allowing the UI's same-origin callback to work. + redirect_uri = _get_validated_client_redirect_uri(request, state_data) params = {"code": code, "state": original_state} complete_returned_url = _append_query_params(redirect_uri, params) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 55d5e4409e8..4901bc76d2f 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -506,7 +506,8 @@ class MCPServerManager: # Add any static headers from server config. # # Note: `extra_headers` on MCPServer is a List[str] of header names to forward - # from the client request (not available in this OpenAPI tool generation step). + # from each client MCP request; values are applied at call time via + # `_request_extra_headers` in server.py (not baked in here). # `static_headers` is a dict of concrete headers to always send. headers = ( merge_mcp_headers( @@ -598,16 +599,57 @@ class MCPServerManager: ) raise e + def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None: + """Drop OpenAPI global tools and name-mapping rows owned by ``server``. + + When a server leaves ``self.registry`` (eviction, ``remove_server``, etc.), + OpenAPI tools remain in ``global_mcp_tool_registry`` and + ``tool_name_to_mcp_server_name_mapping`` unless removed here. Stale + mappings make ``_get_mcp_server_from_tool_name`` resolve to a prefix that + no longer exists in the live registry. + """ + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + + prefix_root = normalize_server_name(get_server_prefix(server)) + if server.spec_path and prefix_root: + openapi_key_prefix = prefix_root + MCP_TOOL_PREFIX_SEPARATOR + global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix) + + owned_raw: Set[str] = set() + for p in iter_known_server_prefixes(server): + if p: + owned_raw.add(p) + if server.name: + owned_raw.add(server.name) + + owned_normalized = {normalize_server_name(x) for x in owned_raw} + + stale_mapping_keys: List[str] = [] + for tool_name, mapped_server in list( + self.tool_name_to_mcp_server_name_mapping.items() + ): + if mapped_server in owned_raw: + stale_mapping_keys.append(tool_name) + elif normalize_server_name(str(mapped_server)) in owned_normalized: + stale_mapping_keys.append(tool_name) + + for key in stale_mapping_keys: + del self.tool_name_to_mcp_server_name_mapping[key] + def remove_server(self, mcp_server: LiteLLM_MCPServerTable): """ Remove a server from the registry """ - if mcp_server.server_name in self.get_registry(): - del self.registry[mcp_server.server_name] - verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_name}") - elif mcp_server.server_id in self.get_registry(): - del self.registry[mcp_server.server_id] - verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_id}") + evicted: Optional[MCPServer] = self.registry.pop(mcp_server.server_id, None) + if evicted is None and mcp_server.server_name: + evicted = self.registry.pop(mcp_server.server_name, None) + if evicted is not None: + verbose_logger.debug( + "Removed MCP Server: %s", mcp_server.server_id or mcp_server.server_name + ) + self._cleanup_server_tool_routing_artifacts(evicted) else: verbose_logger.warning( f"Server ID {mcp_server.server_id} not found in registry" @@ -805,6 +847,13 @@ class MCPServerManager: self.initialize_tool_name_to_mcp_server_name_mapping() async def add_server(self, mcp_server: LiteLLM_MCPServerTable): + # The runtime registry is the allowlist for tool calls and health + # probes (which spawn the underlying transport, including stdio + # subprocesses). Match the eligibility set used by the bulk DB + # filter in reload_servers_from_database() — NULL is legacy and + # "approved" is a legacy alias for "active". + if mcp_server.approval_status not in (None, "active", "approved"): + return try: if mcp_server.server_id not in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) @@ -818,6 +867,16 @@ class MCPServerManager: raise e async def update_server(self, mcp_server: LiteLLM_MCPServerTable): + # If a previously-active server has been moved out of the active + # state, evict any stale registry entry so subsequent tool calls and + # health probes can't reach it. + if mcp_server.approval_status not in (None, "active", "approved"): + evicted = self.registry.pop(mcp_server.server_id, None) + if evicted is None and mcp_server.server_name: + evicted = self.registry.pop(mcp_server.server_name, None) + if evicted is not None: + self._cleanup_server_tool_routing_artifacts(evicted) + return try: if mcp_server.server_id in self.registry: new_server = await self.build_mcp_server_from_table(mcp_server) diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index b13cf83058c..343d1bee613 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -2,15 +2,63 @@ (BYOK + discoverable / pass-through OAuth proxy).""" from ipaddress import ip_address -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse -from fastapi import HTTPException +from fastapi import HTTPException, Request + +from litellm._logging import verbose_logger +from litellm.proxy.auth.ip_address_utils import IPAddressUtils # RFC 6749 §5.1 / OAuth 2.1 draft-15 §4.1.3: token-endpoint responses # must not be cached — both success and error bodies may reveal secrets. TOKEN_NO_CACHE_HEADERS = {"Cache-Control": "no-store", "Pragma": "no-cache"} +def get_request_base_url(request: Request) -> str: + """ + Get the base URL for the request, considering X-Forwarded-* headers. + + X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured + when the request comes from a configured trusted proxy + (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``). + Otherwise the request's literal ``base_url`` is returned, so an + untrusted caller cannot poison OAuth-discovery / redirect_uri values + by injecting headers. + + Args: + request: FastAPI Request object + + Returns: + The reconstructed base URL (e.g., "https://proxy.example.com") + """ + base_url = str(request.base_url).rstrip("/") + parsed = urlparse(base_url) + + if not IPAddressUtils.is_request_from_trusted_proxy(request): + return base_url + + x_forwarded_proto = request.headers.get("X-Forwarded-Proto") + x_forwarded_host = request.headers.get("X-Forwarded-Host") + x_forwarded_port = request.headers.get("X-Forwarded-Port") + + scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme + + if x_forwarded_host: + # X-Forwarded-Host may already include port (e.g., "example.com:8080") + if ":" in x_forwarded_host and not x_forwarded_host.startswith("["): + netloc = x_forwarded_host + elif x_forwarded_port: + netloc = f"{x_forwarded_host}:{x_forwarded_port}" + else: + netloc = x_forwarded_host + else: + netloc = parsed.netloc + if x_forwarded_port and ":" not in netloc: + netloc = f"{netloc}:{x_forwarded_port}" + + return urlunparse((scheme, netloc, parsed.path, "", "", "")) + + def validate_loopback_redirect_uri(redirect_uri: str) -> None: """Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3 native-app pattern). MCP clients are native apps that listen on @@ -46,3 +94,60 @@ def validate_loopback_redirect_uri(redirect_uri: str) -> None: # don't let it bubble up as a 500. pass raise HTTPException(status_code=400, detail="invalid_request") + + +def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None: + """Accept same-origin (proxy's own origin) OR loopback ``redirect_uri``. + + Same-origin is required for the LiteLLM UI's OAuth flow: the UI + redirects to ``/ui/mcp/oauth/callback`` which is not loopback + but is on the proxy's own trusted HTTPS origin. An attacker cannot + host content on the proxy's own origin without already owning the + proxy, so the open-redirect / code-theft primitive that motivated + :func:`validate_loopback_redirect_uri` does not apply here. + + Loopback continues to be accepted for native MCP clients (per + OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3). + + Use this in the discoverable OAuth proxy endpoints that serve both + native clients and the proxy's own UI. BYOK endpoints that only + support native clients should keep + :func:`validate_loopback_redirect_uri`. + """ + try: + parsed = urlparse(redirect_uri) + except ValueError: + raise HTTPException(status_code=400, detail="invalid_request") + if parsed.scheme not in ("http", "https"): + raise HTTPException(status_code=400, detail="invalid_request") + if parsed.fragment: + raise HTTPException(status_code=400, detail="invalid_request") + + # Same-origin: scheme + netloc (host[:port]) must match the proxy's + # own base URL at this request (honouring trusted X-Forwarded-*). + try: + proxy_base = urlparse(get_request_base_url(request)) + if ( + parsed.netloc + and parsed.scheme == proxy_base.scheme + and parsed.netloc.lower() == proxy_base.netloc.lower() + ): + return + except Exception as exc: + # If we can't determine the proxy's origin, fall through to + # loopback. Log so the failure is diagnosable in production. + verbose_logger.warning( + "validate_trusted_redirect_uri: could not determine proxy origin, " + "falling back to loopback-only check. error=%s", + exc, + ) + + host = (parsed.hostname or "").lower() + if host == "localhost": + return + try: + if ip_address(host).is_loopback: + return + except ValueError: + pass + raise HTTPException(status_code=400, detail="invalid_request") diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 718435cce6f..271517bb1e6 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -55,6 +55,13 @@ _request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.Contex "_request_auth_header", default=None ) +# Per-request extra headers forwarded from the client request. +# Populated from MCPServer.extra_headers names matched against raw request +# headers in server.py before dispatching to a local/OpenAPI tool handler. +_request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = ( + contextvars.ContextVar("_request_extra_headers", default=None) +) + def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str: """Ensure path params cannot introduce directory traversal.""" @@ -297,6 +304,46 @@ def build_input_schema(operation: Dict[str, Any]) -> Dict[str, Any]: } +def _merge_openapi_tool_request_headers( + static_headers: Dict[str, str] +) -> Dict[str, str]: + """Merge static closure headers with per-request ContextVar overrides. + + Precedence (highest to lowest): + 1. ``_request_auth_header`` — BYOK override of ``Authorization`` + 2. ``static_headers`` — operator-configured headers baked into the + tool closure at registration time + 3. ``_request_extra_headers`` — per-request headers forwarded from + the MCP caller (allowlisted by ``MCPServer.extra_headers``) + + This matches the existing MCP invariant in + :func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers` + and the managed MCP path, where ``static_headers`` always wins over + caller-forwarded headers. Keeping the same precedence here prevents an + authenticated caller from overriding an operator-configured value + (e.g. a tenant id or upstream API key) by sending the same header name. + + Header names are compared case-insensitively so different casing cannot + bypass the precedence rules. + """ + request_extra = _request_extra_headers.get() or {} + static = static_headers or {} + + static_lower_names = {k.lower() for k in static} + effective_headers: Dict[str, str] = { + k: v for k, v in request_extra.items() if k.lower() not in static_lower_names + } + effective_headers.update(static) + + override_auth = _request_auth_header.get() + if override_auth: + for existing in [k for k in effective_headers if k.lower() == "authorization"]: + del effective_headers[existing] + effective_headers["Authorization"] = override_auth + + return effective_headers + + def create_tool_function( path: str, method: str, @@ -334,14 +381,7 @@ def create_tool_function( The function safely handles parameter names that aren't valid Python identifiers by using **kwargs instead of named parameters. """ - # Allow per-request auth override (e.g. BYOK credential set via ContextVar). - # The ContextVar holds the full Authorization header value, including the - # correct prefix (Bearer / ApiKey / Basic) formatted by the caller in - # server.py based on the server's configured auth_type. - effective_headers = dict(headers) - override_auth = _request_auth_header.get() - if override_auth: - effective_headers["Authorization"] = override_auth + effective_headers = _merge_openapi_tool_request_headers(headers) # Build URL from base_url and path url = base_url + path diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 54d9bbe6e28..276a6e8a3bb 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -158,6 +158,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( _request_auth_header, + _request_extra_headers, ) from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -2195,11 +2196,40 @@ if MCP_AVAILABLE: auth_header_value = f"Basic {mcp_auth_header}" else: auth_header_value = f"Bearer {mcp_auth_header}" + + # Forward named client headers to OpenAPI tool upstream requests. + # MCPServer.extra_headers lists header names to copy from raw_headers. + # OAuth2 M2M: never take Authorization from the caller (matches + # _prepare_mcp_server_headers for managed MCP). + forwarded_headers: Optional[Dict[str, str]] = None + if mcp_server and mcp_server.extra_headers and raw_headers: + normalized_raw = { + str(k).lower(): v + for k, v in raw_headers.items() + if isinstance(k, str) + } + skip_caller_authorization = bool(mcp_server.has_client_credentials) + for header_name in mcp_server.extra_headers: + if not isinstance(header_name, str): + continue + if ( + skip_caller_authorization + and header_name.lower() == "authorization" + ): + continue + value = normalized_raw.get(header_name.lower()) + if value is not None: + if forwarded_headers is None: + forwarded_headers = {} + forwarded_headers[header_name] = value + _auth_token = _request_auth_header.set(auth_header_value) + _extra_token = _request_extra_headers.set(forwarded_headers) try: local_content = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) + _request_extra_headers.reset(_extra_token) response = CallToolResult(content=cast(Any, local_content), isError=False) # Try managed MCP server tool (pass the full prefixed name) diff --git a/litellm/proxy/_experimental/mcp_server/tool_registry.py b/litellm/proxy/_experimental/mcp_server/tool_registry.py index 58570aafadf..829be5be979 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_registry.py +++ b/litellm/proxy/_experimental/mcp_server/tool_registry.py @@ -59,6 +59,22 @@ class MCPToolRegistry: ] return list(self.tools.values()) + def unregister_tools_with_prefix(self, prefix: str) -> int: + """Remove tools whose registered name starts with ``prefix``. + + Used when an OpenAPI-backed MCP server leaves the runtime registry so + stale tool handlers cannot be invoked after eviction. + """ + if not prefix: + return 0 + removed = 0 + for name in list(self.tools.keys()): + if name.startswith(prefix): + del self.tools[name] + removed += 1 + verbose_logger.debug("Unregistered MCP tool %s", name) + return removed + def convert_tools_to_mcp_sdk_tool_type( self, tools: List[MCPTool] ) -> List["MCPToolSDKTool"]: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed20fe86cdc..6128f485748 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -239,6 +239,7 @@ class KeyManagementRoutes(str, enum.Enum): KEY_BLOCK = "/key/block" KEY_UNBLOCK = "/key/unblock" KEY_BULK_UPDATE = "/key/bulk_update" + TEAM_KEY_BULK_UPDATE = "/team/key/bulk_update" KEY_RESET_SPEND = "/key/{key_id}/reset_spend" # info and health routes @@ -353,8 +354,10 @@ class LiteLLMRoutes(enum.Enum): # realtime "/realtime", "/v1/realtime", + "/openai/v1/realtime", "/realtime?{model}", "/v1/realtime?{model}", + "/openai/v1/realtime?{model}", # responses API "/responses", "/v1/responses", @@ -538,6 +541,7 @@ class LiteLLMRoutes(enum.Enum): KeyManagementRoutes.KEY_BLOCK.value, KeyManagementRoutes.KEY_UNBLOCK.value, KeyManagementRoutes.KEY_BULK_UPDATE.value, + KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value, KeyManagementRoutes.SPEND_LOGS.value, KeyManagementRoutes.KEY_RESET_SPEND.value, @@ -673,6 +677,10 @@ class LiteLLMRoutes(enum.Enum): "/global/activity", "/global/activity/model", "/global/activity/cache_hits", + # Tag usage endpoints scope internal users to tags produced by + # their own keys in tag_management_endpoints.py. + "/tag/daily/activity", + "/tag/list", "/v1/models/{model_id}", "/models/{model_id}", "/guardrails/list", @@ -685,7 +693,16 @@ class LiteLLMRoutes(enum.Enum): + compliance_check_routes ) - internal_user_view_only_routes = spend_tracking_routes + internal_user_view_only_routes = ( + spend_tracking_routes + + compliance_check_routes + + [ + # Tag usage endpoints scope internal viewers to tags produced by + # their own keys in tag_management_endpoints.py. + "/tag/daily/activity", + "/tag/list", + ] + ) self_managed_routes = [ "/team/member_add", @@ -707,6 +724,8 @@ class LiteLLMRoutes(enum.Enum): # Project read routes - endpoint scopes results to caller's teams (non-admin) "/project/list", "/project/info", + # Endpoint enforces proxy-admin vs team-admin model access itself. + "/health/test_connection", # Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges "/invitation/new", "/invitation/delete", @@ -4342,10 +4361,16 @@ class JWTRoutingOverride(BaseModel): A rule matches when all provided selectors match token claims. If matched, request is routed to the configured auth path. + + Wildcard selectors use shell-style patterns (* and ?) and are matched with + case-sensitive semantics; use the same casing your IdP emits in JWT claims. + Space-delimited tokenization applies only to the ``scope`` claim (OAuth/OIDC + scope strings), not to ``iss``, ``aud``, or ``client_id``. """ iss: Union[str, List[str]] client_id: Optional[Union[str, List[str]]] = None + scope: Optional[Union[str, List[str]]] = None aud: Optional[Union[str, List[str]]] = None path: Literal["oauth2"] = "oauth2" diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f6f99eb62c8..0b30999aa21 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2849,7 +2849,7 @@ def _can_object_call_model( object_type=object_type ), param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) @@ -3082,7 +3082,7 @@ async def can_user_call_model( message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", type=ProxyErrorTypes.key_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) return _can_object_call_model( @@ -3625,7 +3625,7 @@ async def _check_team_member_model_access( message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}", type=ProxyErrorTypes.team_model_access_denied, param="model", - code=status.HTTP_401_UNAUTHORIZED, + code=status.HTTP_403_FORBIDDEN, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 5ded8136ef3..431db4254eb 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -123,7 +123,7 @@ class UserAPIKeyAuthExceptionHandler: message=e.message, type=ProxyErrorTypes.budget_exceeded, param=None, - code=400, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), ) if isinstance(e, HTTPException): raise ProxyException( diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index d1fd5818f35..5b116c142b2 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -224,6 +224,41 @@ class JWTHandler: return [] + def get_all_jwt_team_ids(self, token: dict) -> List[str]: + """ + Return team IDs from both the plural ``team_ids_jwt_field`` and the + singular ``team_id_jwt_field`` claim (string or list of strings), as a + deduplicated list preserving plural-first order. + + Membership-reconciliation paths (SSO callback, JWT-bearer sync) need + to consider both claim shapes. Reading only the plural field — as + callers historically did — silently dropped users whose IdP populates + the singular field, which is what Okta and Auth0 default to when a + user has a single primary team. + + This intentionally does NOT consult ``team_id_default``: that fallback + is a property of how the JWT-bearer auth flow resolves a single + request-bound team, not of the token's claims. Callers that want the + default-team behavior should still go through ``get_team_id``. + """ + team_ids: List[str] = list(self.get_team_ids_from_jwt(token)) + if self.litellm_jwtauth.team_id_jwt_field is not None: + singular = get_nested_value( + data=token, + key_path=self.litellm_jwtauth.team_id_jwt_field, + default=None, + ) + if isinstance(singular, list): + for item in singular: + if item is None: + continue + sid = str(item) + if sid and sid not in team_ids: + team_ids.append(sid) + elif singular and str(singular) not in team_ids: + team_ids.append(str(singular)) + return team_ids + def get_end_user_id( self, token: dict, default_value: Optional[str] ) -> Optional[str]: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 8bcfbb67539..a9c36dfc512 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -50,6 +50,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES = frozenset( KeyManagementRoutes.KEY_BLOCK.value, KeyManagementRoutes.KEY_UNBLOCK.value, KeyManagementRoutes.KEY_BULK_UPDATE.value, + KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value, ] ) @@ -671,6 +672,7 @@ class RouteChecks: "/key/service-account/generate", "/key/block", "/key/unblock", + "/team/key/bulk_update", ] ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9d3c06e641f..39f0af19484 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -8,6 +8,7 @@ Returns a UserAPIKeyAuth object if the API key is valid """ import asyncio +import fnmatch import re import secrets from datetime import datetime, timezone @@ -183,22 +184,54 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str: def _routing_selector_matches_claim( - selector_value: Optional[Any], claim_value: Optional[Any] + selector_value: Optional[Any], + claim_value: Optional[Any], + *, + split_space_delimited: bool = False, ) -> bool: if selector_value is None: return True - selector_list = ( + selector_list: List[str] = ( [str(v) for v in selector_value] if isinstance(selector_value, list) else [str(selector_value)] ) + if claim_value is None: + return False + if isinstance(claim_value, list): claim_list = [str(v) for v in claim_value] - return any(v in claim_list for v in selector_list) + elif ( + split_space_delimited + and isinstance(claim_value, str) + and " " in claim_value.strip() + ): + # OAuth/OIDC often sends scope as a single space-delimited string. Only split + # for the scope selector: iss/aud/client_id must stay exact full-string match + # on unverified claims (see routing override security review). The elif guard + # (`" " in claim_value.strip()`) ensures at least two non-empty tokens survive. + claim_list = [v for v in claim_value.strip().split(" ") if v] + else: + claim_list = [str(claim_value)] - return str(claim_value) in selector_list if claim_value is not None else False + def _selector_matches_claim(selector: str, claim: str) -> bool: + # NOTE: wildcard matching is case-sensitive (fnmatch.fnmatchcase). + if "*" in selector or "?" in selector: + # Without scope splitting, do not let `*` span whitespace: a malformed + # iss like "trusted.example.com evil.com" must not match "trusted.*". + # Scope uses split_space_delimited so each claim token is checked separately. + if not split_space_delimited and any(ch.isspace() for ch in claim): + return False + return fnmatch.fnmatchcase(claim, selector) + return selector == claim + + return any( + _selector_matches_claim(selector=s, claim=c) + for s in selector_list + for c in claim_list + ) def _matches_routing_override( @@ -209,6 +242,11 @@ def _matches_routing_override( and _routing_selector_matches_claim( override.client_id, token_claims.get("client_id") ) + and _routing_selector_matches_claim( + override.scope, + token_claims.get("scope"), + split_space_delimited=True, + ) and _routing_selector_matches_claim(override.aud, token_claims.get("aud")) ) @@ -1107,7 +1145,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) valid_token = update_valid_token_with_end_user_params( @@ -1432,7 +1470,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=abbreviate_api_key(api_key=api_key), ) @@ -2417,7 +2455,7 @@ async def _run_post_custom_auth_checks( raise ProxyException( message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}", type=ProxyErrorTypes.expired_key, - code=400, + code=status.HTTP_401_UNAUTHORIZED, param=( abbreviate_api_key(api_key=valid_token.token) if valid_token.token diff --git a/litellm/proxy/common_utils/html_forms/jwt_display_template.py b/litellm/proxy/common_utils/html_forms/jwt_display_template.py index 03dff78dba0..ea65fd0e28f 100644 --- a/litellm/proxy/common_utils/html_forms/jwt_display_template.py +++ b/litellm/proxy/common_utils/html_forms/jwt_display_template.py @@ -14,7 +14,7 @@ jwt_display_template = """ padding: 20px; display: flex; justify-content: center; - align-items: center; + align-items: flex-start; min-height: 100vh; color: #333; } @@ -27,18 +27,18 @@ jwt_display_template = """ width: 800px; max-width: 100%; } - + .logo-container { text-align: center; margin-bottom: 30px; } - + .logo { font-size: 24px; font-weight: 600; color: #1e293b; } - + h2 { margin: 0 0 10px; color: #1e293b; @@ -46,7 +46,14 @@ jwt_display_template = """ font-weight: 600; text-align: center; } - + + h3 { + margin: 0 0 12px; + color: #1e293b; + font-size: 18px; + font-weight: 600; + } + .subtitle { color: #64748b; margin: 0 0 20px; @@ -58,15 +65,15 @@ jwt_display_template = """ background-color: #f1f5f9; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #2563eb; } - + .success-box { background-color: #f0fdf4; border-radius: 6px; padding: 20px; - margin-bottom: 30px; + margin-bottom: 20px; border-left: 4px solid #16a34a; } @@ -78,7 +85,7 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .success-header { display: flex; align-items: center; @@ -87,46 +94,53 @@ jwt_display_template = """ font-weight: 600; font-size: 16px; } - + .info-header svg, .success-header svg { margin-right: 8px; } - + .data-container { margin-top: 20px; } - + .data-row { display: flex; border-bottom: 1px solid #e2e8f0; padding: 12px 0; } - + .data-row:last-child { border-bottom: none; } - + .data-label { font-weight: 500; color: #334155; - width: 180px; + width: 220px; flex-shrink: 0; } - + .data-value { color: #475569; word-break: break-all; } - + + .empty-note { + color: #64748b; + font-style: italic; + margin: 0; + font-size: 14px; + } + .jwt-container { background-color: #f8fafc; border-radius: 6px; padding: 15px; - margin-top: 20px; + margin-top: 12px; overflow-x: auto; border: 1px solid #e2e8f0; } - + .jwt-text { font-family: monospace; white-space: pre-wrap; @@ -134,7 +148,7 @@ jwt_display_template = """ margin: 0; color: #334155; } - + .back-button { display: inline-block; background-color: #6466E9; @@ -146,18 +160,18 @@ jwt_display_template = """ margin-top: 20px; text-align: center; } - + .back-button:hover { background-color: #4138C2; text-decoration: none; } - + .buttons { display: flex; gap: 10px; - margin-top: 20px; + margin-top: 12px; } - + .copy-button { background-color: #e2e8f0; color: #334155; @@ -169,11 +183,11 @@ jwt_display_template = """ display: flex; align-items: center; } - + .copy-button:hover { background-color: #cbd5e1; } - + .copy-button svg { margin-right: 6px; } @@ -188,7 +202,7 @@ jwt_display_template = """

SSO Debug Information

Results from the SSO authentication process.

- +
@@ -199,11 +213,7 @@ jwt_display_template = """

The SSO authentication completed successfully. Below is the information returned by the provider.

- -
- -
- +
@@ -211,22 +221,62 @@ jwt_display_template = """ - JSON Representation + Parsed by Proxy
+

Fields the proxy extracted into its internal user model.

+
+ +
+
+ +
+
+ + + + + + Raw Claims (userinfo) +
+

Complete set of claims returned by the IdP's userinfo endpoint.

- 
Loading...
+ 
Loading...
-
- + +
+
+ + + + + + Access Token Claims +
+

Decoded payload of the access token JWT (when the IdP issues one).

+
+ 
Loading...
+
+
+ +
+
+ Try Another SSO Login @@ -234,39 +284,58 @@ jwt_display_template = """ diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 0928ce914da..52bbeaf2ad3 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -2,7 +2,7 @@ import asyncio import json import time from datetime import datetime, timezone -from typing import Any, List, Literal, Optional, Union +from typing import Any, Callable, List, Literal, Optional, Union import litellm from litellm._logging import verbose_proxy_logger @@ -83,93 +83,146 @@ class ResetBudgetJob: "Failed to reset spend counter %s: %s", counter_key, e ) + @staticmethod + async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None: + """Drop a stale management-cache entry so the next read fetches from DB. + + Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``; + for those, when the spend counter expires the budget check falls back + to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded + from the DB, but auth still may consult ``user_api_key_cache`` objects + whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache + entry forces the next auth-time fetch to reload the zeroed row from + Postgres. + """ + try: + from litellm.proxy.proxy_server import user_api_key_cache + + await user_api_key_cache.async_delete_cache(key=cache_key) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to invalidate user_api_key_cache entry %s: %s", + cache_key, + e, + ) + + async def _cascade_reset_spend_for_budget_link( + self, + budgets_to_reset: List[LiteLLM_BudgetTableFull], + table: Any, + counter_key_fn: Callable[[Any], str], + log_subject: str, + extra_where: Optional[dict] = None, + cache_key_fn: Optional[Callable[[Any], Union[str, List[str]]]] = None, + ): + """ + Generic cascade: zero spend on rows whose budget_id is in the reset set. + + ``cache_key_fn`` is optional: when provided, after the DB update each + matching row's entry or entries in ``user_api_key_cache`` are dropped so + cached spend cannot stay pinned above the zeroed DB row after a reset. + """ + budget_ids = [b.budget_id for b in budgets_to_reset if b.budget_id is not None] + if not budget_ids: + return + + where: dict = {"budget_id": {"in": budget_ids}} + if extra_where: + where.update(extra_where) + + try: + rows = await table.find_many(where=where) + except Exception as e: + rows = [] + verbose_proxy_logger.warning( + "Failed to fetch %s for counter invalidation: %s", log_subject, e + ) + + update_result = await table.update_many(where=where, data={"spend": 0}) + + for row in rows: + await self._invalidate_spend_counter(counter_key_fn(row)) + if cache_key_fn is not None: + cache_keys = cache_key_fn(row) + if isinstance(cache_keys, str): + cache_keys = [cache_keys] + for cache_key in cache_keys: + await self._invalidate_user_api_key_cache_entry(cache_key) + + return update_result + async def reset_budget_for_litellm_team_members( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the budget for all LiteLLM Team Members if their budget has expired """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - - try: - memberships = await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) - except Exception as e: - memberships = [] - verbose_proxy_logger.warning( - "Failed to fetch team memberships for counter invalidation: %s", e - ) - - update_result = await self.prisma_client.db.litellm_teammembership.update_many( - where={"budget_id": {"in": budget_ids}}, - data={ - "spend": 0, - }, + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_teammembership, + counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}", + log_subject="team memberships", + cache_key_fn=lambda m: f"{m.team_id}_{m.user_id}", ) - for m in memberships: - await self._invalidate_spend_counter( - f"spend:team_member:{m.user_id}:{m.team_id}" - ) - - return update_result - async def reset_budget_for_keys_linked_to_budgets( self, budgets_to_reset: List[LiteLLM_BudgetTableFull] ): """ Resets the spend for keys linked to budget tiers that are being reset. - This handles keys that have budget_id but no budget_duration set on the key - itself. Keys with budget_id rely on their linked budget tier's reset schedule - rather than having their own budget_duration. - - Keys that have their own budget_duration are already handled by - reset_budget_for_litellm_keys() and are excluded here to avoid - double-resetting. + Excludes keys with their own budget_duration; those are reset by + reset_budget_for_litellm_keys() to avoid double-resetting. """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - if not budget_ids: - return - - where_clause: dict = { - "budget_id": {"in": budget_ids}, - "budget_duration": None, # only keys without their own reset schedule - "spend": {"gt": 0}, # only reset keys that have accumulated spend - } - - try: - keys = await self.prisma_client.db.litellm_verificationtoken.find_many( - where=where_clause - ) - except Exception as e: - keys = [] - verbose_proxy_logger.warning( - "Failed to fetch keys for counter invalidation: %s", e - ) - - update_result = ( - await self.prisma_client.db.litellm_verificationtoken.update_many( - where=where_clause, - data={ - "spend": 0, - }, - ) + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_verificationtoken, + counter_key_fn=lambda k: f"spend:key:{k.token}", + log_subject="keys", + extra_where={"budget_duration": None, "spend": {"gt": 0}}, + cache_key_fn=lambda k: k.token, ) - for k in keys: - await self._invalidate_spend_counter(f"spend:key:{k.token}") + async def reset_budget_for_orgs_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for orgs linked to budget tiers that are being reset. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_organizationtable, + counter_key_fn=lambda o: f"spend:org:{o.organization_id}", + log_subject="orgs", + extra_where={"spend": {"gt": 0}}, + cache_key_fn=lambda o: [ + f"org_id:{o.organization_id}", + f"org_id:{o.organization_id}:with_budget", + ], + ) - return update_result + async def reset_budget_for_tags_linked_to_budgets( + self, budgets_to_reset: List[LiteLLM_BudgetTableFull] + ): + """ + Resets the spend for tags linked to budget tiers that are being reset. + + Also drops each tag's ``user_api_key_cache`` entry so the next + ``_tag_max_budget_check`` reloads the zeroed row from the DB. + ``SpendCounterReseed.from_db`` intentionally returns ``None`` for + tags, so the budget check falls back to the cached + ``LiteLLM_TagTable.spend`` once the spend counter expires; without + this invalidation, that stale ``.spend`` keeps the tag over-budget + indefinitely. + """ + return await self._cascade_reset_spend_for_budget_link( + budgets_to_reset=budgets_to_reset, + table=self.prisma_client.db.litellm_tagtable, + counter_key_fn=lambda t: f"spend:tag:{t.tag_name}", + log_subject="tags", + extra_where={"spend": {"gt": 0}}, + cache_key_fn=lambda t: f"tag:{t.tag_name}", + ) async def reset_budget_for_litellm_budget_table(self): """ @@ -237,6 +290,14 @@ class ResetBudgetJob: budgets_to_reset=budgets_to_reset ) + await self.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + + await self.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=budgets_to_reset + ) + if endusers_to_reset is not None and len(endusers_to_reset) > 0: for enduser in endusers_to_reset: try: diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 3697e498bb4..7611c9c9692 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1131,10 +1131,12 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - user_id, - response_cost, - ) in user_list_transactions.items(): + # Sort by ID for consistent lock ordering across pods to prevent deadlocks. + # batch_() issues statements sequentially within the tx, so iteration + # order = lock acquisition order. + for user_id, response_cost in sorted( + user_list_transactions.items() + ): batcher.litellm_usertable.update_many( where={"user_id": user_id}, data={"spend": {"increment": response_cost}}, @@ -1186,10 +1188,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - token, - response_cost, - ) in key_list_transactions.items(): + # Sort by token for consistent lock ordering across pods to prevent deadlocks. + for token, response_cost in sorted( + key_list_transactions.items() + ): batcher.litellm_verificationtoken.update_many( # 'update_many' prevents error from being raised if no row exists where={"token": token}, data={ @@ -1230,10 +1232,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - team_id, - response_cost, - ) in team_list_transactions.items(): + # Sort by team_id for consistent lock ordering across pods to prevent deadlocks. + for team_id, response_cost in sorted( + team_list_transactions.items() + ): verbose_proxy_logger.debug( "Updating spend for team id={} by {}".format( team_id, response_cost @@ -1288,10 +1290,11 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - key, - response_cost, - ) in team_member_list_transactions.items(): + # Sort by composite key for consistent lock ordering across pods to prevent deadlocks. + # Key format "team_id::::user_id::" makes the string sort equivalent to sorting by (team_id, user_id). + for key, response_cost in sorted( + team_member_list_transactions.items() + ): # key is "team_id::::user_id::" team_id = key.split("::")[1] user_id = key.split("::")[3] @@ -1348,10 +1351,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - org_id, - response_cost, - ) in org_list_transactions.items(): + # Sort by org_id for consistent lock ordering across pods to prevent deadlocks. + for org_id, response_cost in sorted( + org_list_transactions.items() + ): batcher.litellm_organizationtable.update_many( # 'update_many' prevents error from being raised if no row exists where={"organization_id": org_id}, data={"spend": {"increment": response_cost}}, @@ -1439,7 +1442,10 @@ class DBSpendUpdateWriter: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for entity_id, response_cost in transactions.items(): + # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks. + for entity_id, response_cost in sorted( + transactions.items() + ): verbose_proxy_logger.debug( f"Updating spend for {entity_name} {where_field}={entity_id} by {response_cost}" ) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index d112e222307..af5a58802bb 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -10,13 +10,64 @@ import subprocess import time import urllib import urllib.parse +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any, Optional, Union +from typing import Any, Dict, Optional, Union from litellm._logging import verbose_proxy_logger from litellm.secret_managers.main import str_to_bool +@dataclass(frozen=True) +class IAMEndpoint: + """Static parts of an RDS IAM-authenticated Postgres connection. + + The IAM token rotates every ~15 minutes; everything else (host, port, user, + database name, schema) stays fixed. We capture the static fields once so + refresh just regenerates the token and reassembles the URL. + """ + + host: str + port: str + user: str + name: str + schema: Optional[str] = None + + def build_url(self, token: str) -> str: + url = f"postgresql://{self.user}:{token}@{self.host}:{self.port}/{self.name}" + if self.schema: + url += f"?schema={self.schema}" + return url + + +def parse_iam_endpoint_from_url(url: str) -> IAMEndpoint: + """Parse an IAMEndpoint from a Postgres URL. + + Used so a reader URL can drive its own IAM refresh without requiring + callers to set parallel DATABASE_HOST_READ_REPLICA / etc. env vars. + """ + parsed = urllib.parse.urlparse(url) + if not parsed.hostname or not parsed.username: + raise ValueError("Cannot parse IAM endpoint from URL: missing host or username") + name = (parsed.path or "/").lstrip("/") + if not name: + raise ValueError("Cannot parse IAM endpoint from URL: missing database name") + port = str(parsed.port) if parsed.port else "5432" + schema: Optional[str] = None + if parsed.query: + qs = urllib.parse.parse_qs(parsed.query) + schema_vals = qs.get("schema") + if schema_vals: + schema = schema_vals[0] + return IAMEndpoint( + host=parsed.hostname, + port=port, + user=parsed.username, + name=name, + schema=schema, + ) + + class PrismaWrapper: """ Wrapper around Prisma client that handles RDS IAM token authentication. @@ -37,10 +88,33 @@ class PrismaWrapper: # Fallback refresh interval if token parsing fails (10 minutes) FALLBACK_REFRESH_INTERVAL_SECONDS = 600 - def __init__(self, original_prisma: Any, iam_token_db_auth: bool): + def __init__( + self, + original_prisma: Any, + iam_token_db_auth: bool, + *, + db_url_env_var: str = "DATABASE_URL", + iam_endpoint: Optional[IAMEndpoint] = None, + recreate_uses_datasource: bool = False, + log_prefix: str = "", + ): self._original_prisma = original_prisma self.iam_token_db_auth = iam_token_db_auth + # Per-connection knobs so the same wrapper can be used for the writer + # (defaults: DATABASE_URL env, IAM endpoint from DATABASE_HOST/etc., + # recreate via env reload) or for a reader (DATABASE_URL_READ_REPLICA + # env, IAM endpoint parsed from that URL, recreate via datasource + # override since Prisma only auto-reads DATABASE_URL). + self._db_url_env_var = db_url_env_var + self._iam_endpoint = iam_endpoint + self._recreate_uses_datasource = recreate_uses_datasource + # Tag every log line emitted by this wrapper instance so writer and + # reader can be told apart in interleaved output (e.g. "[writer] RDS + # IAM token refresh scheduled in 720 seconds"). Empty string (default) + # keeps backward-compatible logs for the single-DB case. + self._log_prefix = f"{log_prefix} " if log_prefix else "" + # Background token refresh task management self._token_refresh_task: Optional[asyncio.Task] = None self._reconnection_lock = asyncio.Lock() @@ -157,7 +231,7 @@ class PrismaWrapper: Returns 0 if token should be refreshed immediately. Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails. """ - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) token = self._extract_token_from_db_url(db_url) expiration_time = self._parse_token_expiration(token) @@ -199,12 +273,30 @@ class PrismaWrapper: return datetime.utcnow() > expiration_time def get_rds_iam_token(self) -> Optional[str]: - """Generate a new RDS IAM token and update DATABASE_URL.""" - if self.iam_token_db_auth: - from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + """Generate a new RDS IAM token and update the configured DB URL env var. + When the wrapper was constructed with an explicit `iam_endpoint` + (typical for a reader wrapper whose host/port/user came from a parsed + URL), use that. Otherwise fall back to the legacy DATABASE_HOST/PORT/ + USER/NAME/SCHEMA env vars (writer behavior). + """ + if not self.iam_token_db_auth: + return None + + from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token + + if self._iam_endpoint is not None: + endpoint = self._iam_endpoint + token = generate_iam_auth_token( + db_host=endpoint.host, db_port=endpoint.port, db_user=endpoint.user + ) + _db_url = endpoint.build_url(token) + else: db_host = os.getenv("DATABASE_HOST") - db_port = os.getenv("DATABASE_PORT") + # Default to the Postgres standard port; passing None to + # `generate_iam_auth_token` makes botocore embed the literal + # string "None" in the presigned URL, which then fails to parse. + db_port = os.getenv("DATABASE_PORT", "5432") db_user = os.getenv("DATABASE_USER") db_name = os.getenv("DATABASE_NAME") db_schema = os.getenv("DATABASE_SCHEMA") @@ -217,9 +309,8 @@ class PrismaWrapper: if db_schema: _db_url += f"?schema={db_schema}" - os.environ["DATABASE_URL"] = _db_url - return _db_url - return None + os.environ[self._db_url_env_var] = _db_url + return _db_url async def recreate_prisma_client( self, new_db_url: str, http_client: Optional[Any] = None @@ -231,6 +322,11 @@ class PrismaWrapper: synchronous `subprocess.Popen.wait()` that can freeze the asyncio event loop for 30-120+ seconds when the engine is stuck on TCP close, breaking `/health/liveliness` and causing Kubernetes pod restarts. + + The writer wrapper relies on Prisma re-reading `DATABASE_URL` from env; + the reader wrapper opts into `recreate_uses_datasource=True` so the + new URL is passed explicitly via `datasource={"url": ...}` (Prisma + does not auto-read alternate env vars like DATABASE_URL_READ_REPLICA). """ from prisma import Prisma # type: ignore @@ -238,10 +334,12 @@ class PrismaWrapper: if old_engine_pid > 0: await self._kill_engine_process(old_engine_pid) + kwargs: Dict[str, Any] = {} if http_client is not None: - self._original_prisma = Prisma(http=http_client) - else: - self._original_prisma = Prisma() + kwargs["http"] = http_client + if self._recreate_uses_datasource: + kwargs["datasource"] = {"url": new_db_url} + self._original_prisma = Prisma(**kwargs) await self._original_prisma.connect() @@ -265,7 +363,8 @@ class PrismaWrapper: self._token_refresh_task = asyncio.create_task(self._token_refresh_loop()) verbose_proxy_logger.info( - "Started RDS IAM token proactive refresh background task" + "%sStarted RDS IAM token proactive refresh background task", + self._log_prefix, ) async def stop_token_refresh_task(self) -> None: @@ -283,7 +382,9 @@ class PrismaWrapper: except asyncio.CancelledError: pass self._token_refresh_task = None - verbose_proxy_logger.info("Stopped RDS IAM token refresh background task") + verbose_proxy_logger.info( + "%sStopped RDS IAM token refresh background task", self._log_prefix + ) async def _token_refresh_loop(self) -> None: """ @@ -294,7 +395,7 @@ class PrismaWrapper: This is more efficient than polling, requiring only 1 wake-up per token cycle. """ verbose_proxy_logger.info( - f"RDS IAM token refresh loop started. " + f"{self._log_prefix}RDS IAM token refresh loop started. " f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration." ) @@ -305,21 +406,25 @@ class PrismaWrapper: if sleep_seconds > 0: verbose_proxy_logger.info( - f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds " - f"({sleep_seconds / 60:.1f} minutes)" + f"{self._log_prefix}RDS IAM token refresh scheduled in " + f"{sleep_seconds:.0f} seconds ({sleep_seconds / 60:.1f} minutes)" ) await asyncio.sleep(sleep_seconds) # Refresh the token - verbose_proxy_logger.info("Proactively refreshing RDS IAM token...") + verbose_proxy_logger.info( + "%sProactively refreshing RDS IAM token...", self._log_prefix + ) await self._safe_refresh_token() except asyncio.CancelledError: - verbose_proxy_logger.info("RDS IAM token refresh loop cancelled") + verbose_proxy_logger.info( + "%sRDS IAM token refresh loop cancelled", self._log_prefix + ) break except Exception as e: verbose_proxy_logger.error( - f"Error in RDS IAM token refresh loop: {e}. " + f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. " f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..." ) # On error, wait before retrying to avoid tight error loops @@ -341,65 +446,75 @@ class PrismaWrapper: await self.recreate_prisma_client(new_db_url) self._last_refresh_time = datetime.utcnow() verbose_proxy_logger.info( - "RDS IAM token refreshed successfully. New token valid for ~15 minutes." + "%sRDS IAM token refreshed successfully. New token valid for ~15 minutes.", + self._log_prefix, ) else: verbose_proxy_logger.error( - "Failed to generate new RDS IAM token during proactive refresh" + "%sFailed to generate new RDS IAM token during proactive refresh", + self._log_prefix, ) def __getattr__(self, name: str): """ Proxy attribute access to the underlying Prisma client. - If IAM token auth is enabled and the token is expired, this method - provides a synchronous fallback to refresh the token. However, this - should rarely be needed since the background task proactively refreshes - tokens before they expire. + If IAM token auth is enabled and the token is found expired here, the + proactive refresh task has missed its window. Behavior depends on + whether we're called from inside a running event loop: - FIXED: Now properly waits for reconnection to complete before returning, - instead of the previous fire-and-forget pattern that caused the bug. + - Inside the loop (typical: from a coroutine): schedule a refresh as a + background task and return the (stale) attribute. The caller's await + will likely fail with a connection error and be retried by upper + layers (`call_with_db_reconnect_retry`); by that time the refresh + has either completed or escalated to the proactive loop's error + path. We CANNOT block here — `run_coroutine_threadsafe(...)` + + `future.result()` from inside the same loop deadlocks the loop + (loop thread is blocked, scheduled coroutine never runs, 30s timeout). + + - No running loop (sync caller, mostly tests): run the refresh in a + fresh loop and re-fetch the attribute. """ original_attr = getattr(self._original_prisma, name) if self.iam_token_db_auth: - db_url = os.getenv("DATABASE_URL") + db_url = os.getenv(self._db_url_env_var) # Check if token is expired (should be rare if background task is running) if self.is_token_expired(db_url): - verbose_proxy_logger.warning( - "RDS IAM token expired in __getattr__ - proactive refresh may have failed. " - "Triggering synchronous fallback refresh..." - ) + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None - new_db_url = self.get_rds_iam_token() - if new_db_url: - loop = asyncio.get_event_loop() - - if loop.is_running(): - # FIXED: Actually wait for the reconnection to complete! - # The previous code used fire-and-forget which caused the bug. - future = asyncio.run_coroutine_threadsafe( - self.recreate_prisma_client(new_db_url), loop - ) - try: - # Wait up to 30 seconds for reconnection - future.result(timeout=30) - verbose_proxy_logger.info( - "Synchronous token refresh completed successfully" - ) - except Exception as e: - verbose_proxy_logger.error( - f"Failed to refresh token synchronously: {e}" - ) - raise - else: - asyncio.run(self.recreate_prisma_client(new_db_url)) - - # Get the NEW attribute after reconnection - original_attr = getattr(self._original_prisma, name) + if running_loop is not None: + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Scheduling async refresh; the current " + "request may fail and be retried with the fresh token.", + self._log_prefix, + ) + # Non-blocking: schedule the locked refresh on the + # running loop. The reconnection lock inside + # `_safe_refresh_token` coalesces concurrent triggers. + running_loop.create_task(self._safe_refresh_token()) else: - raise ValueError("Failed to get RDS IAM token") + verbose_proxy_logger.warning( + "%sRDS IAM token expired in __getattr__ — proactive refresh " + "may have failed. Triggering synchronous fallback refresh...", + self._log_prefix, + ) + new_db_url = self.get_rds_iam_token() + if new_db_url: + asyncio.run(self.recreate_prisma_client(new_db_url)) + # Re-fetch attribute against the recreated Prisma instance. + original_attr = getattr(self._original_prisma, name) + verbose_proxy_logger.info( + "%sSynchronous token refresh completed successfully", + self._log_prefix, + ) + else: + raise ValueError("Failed to get RDS IAM token") return original_attr diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py new file mode 100644 index 00000000000..0a976e9f1ea --- /dev/null +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -0,0 +1,213 @@ +""" +RoutingPrismaWrapper: routes Prisma reads to a read-replica client and writes +to a writer client. Used when DATABASE_URL_READ_REPLICA is configured; +otherwise PrismaClient uses the writer-only PrismaWrapper directly. +""" + +import os +from typing import Any, Callable, Optional + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.db.prisma_client import PrismaWrapper + +# Per-model action methods that read from the database. These are routed to +# the read replica when one is configured. +_MODEL_READ_METHODS = frozenset( + { + "find_first", + "find_first_or_raise", + "find_many", + "find_unique", + "find_unique_or_raise", + "count", + "group_by", + "query_first", + "query_raw", + } +) + +# Top-level Prisma client methods that read from the database. +_TOP_LEVEL_READ_METHODS = frozenset({"query_first", "query_raw"}) + + +class _RoutedActions: + """Per-model accessor that sends reads to the reader and writes to the writer. + + `should_use_reader` is consulted on every read dispatch so a mid-call flip + of the routing wrapper's reader-availability flag (e.g. after the reader + fails a recreate) is observed without re-fetching the actions accessor. + """ + + __slots__ = ("_writer_actions", "_reader_actions", "_should_use_reader") + + def __init__( + self, + writer_actions: Any, + reader_actions: Any, + should_use_reader: Callable[[], bool], + ): + self._writer_actions = writer_actions + self._reader_actions = reader_actions + self._should_use_reader = should_use_reader + + def __getattr__(self, name: str) -> Any: + if name in _MODEL_READ_METHODS and self._should_use_reader(): + return getattr(self._reader_actions, name) + return getattr(self._writer_actions, name) + + +class RoutingPrismaWrapper: + """ + Routes Prisma operations between a writer and a reader Prisma client. + + Reads (find_*, count, group_by, query_raw, query_first) go to the reader; + everything else (writes, transactions, raw execute) goes to the writer. + Lifecycle methods (connect, disconnect, IAM token refresh) act on both + clients so callers do not need to know about the split. When + IAM_TOKEN_DB_AUTH is enabled, both writer and reader refresh their tokens + independently on their own ~12-minute cadence. + + Reader degradation: a reader-side failure (failed connect, failed + recreate) is non-fatal — the wrapper sets `_reader_unavailable=True`, logs + a warning, and routes subsequent reads to the writer. The next successful + `connect()` or `recreate_prisma_client()` clears the flag. This keeps the + proxy serving traffic during transient reader outages instead of failing + startup or returning errors for read-heavy endpoints. + """ + + def __init__(self, writer: PrismaWrapper, reader: PrismaWrapper): + self._writer = writer + self._reader = reader + # When True, reads fall back to the writer. Flipped on by reader + # connect/recreate failures and flipped off on the next reader recovery. + self._reader_unavailable: bool = False + + @property + def writer(self) -> PrismaWrapper: + return self._writer + + @property + def reader(self) -> PrismaWrapper: + return self._reader + + @property + def reader_unavailable(self) -> bool: + return self._reader_unavailable + + def _should_use_reader(self) -> bool: + return not self._reader_unavailable + + async def connect(self, *args: Any, **kwargs: Any) -> None: + await self._writer.connect(*args, **kwargs) + verbose_proxy_logger.info("[writer] DB connected") + try: + await self._reader.connect(*args, **kwargs) + self._reader_unavailable = False + verbose_proxy_logger.info("[reader] DB connected") + except Exception as e: + # Degrade gracefully: the proxy keeps serving traffic with reads + # routed to the writer until the reader endpoint is reachable. + # Aborting startup here would tie proxy availability to an + # opt-in, best-effort reader endpoint. + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to connect to read replica DB: %s. " + "Falling back to the writer for reads until the reader is reachable.", + e, + ) + + async def disconnect(self, *args: Any, **kwargs: Any) -> None: + first_error: Optional[BaseException] = None + for client in (self._writer, self._reader): + try: + await client.disconnect(*args, **kwargs) + except Exception as e: + if first_error is None: + first_error = e + verbose_proxy_logger.warning("Error disconnecting Prisma client: %s", e) + if first_error is not None: + raise first_error + + def is_connected(self) -> bool: + # Reflects writer health only. The reader is best-effort; its + # availability is tracked via `_reader_unavailable` and a degraded + # reader must NOT cause a writer reconnect (would loop indefinitely + # since recreate_prisma_client only fixes writer-side problems). + return bool(self._writer.is_connected()) + + async def start_token_refresh_task(self) -> None: + await self._writer.start_token_refresh_task() + await self._reader.start_token_refresh_task() + + async def stop_token_refresh_task(self) -> None: + await self._writer.stop_token_refresh_task() + await self._reader.stop_token_refresh_task() + + async def recreate_prisma_client( + self, new_db_url: str, http_client: Optional[Any] = None + ) -> None: + """Recreate both writer and reader Prisma clients. + + The writer reconnect path in PrismaClient calls + `self.db.recreate_prisma_client(...)`. Without this method, a DB-wide + connectivity event would only re-create the writer; the reader engine + would stay broken and every routed read would fail. We always recreate + the writer first (its URL is the one passed in), then best-effort + recreate the reader. A reader failure flips `_reader_unavailable=True` + so reads transparently fall through to the writer. + """ + await self._writer.recreate_prisma_client(new_db_url, http_client=http_client) + try: + await self._recreate_reader(http_client=http_client) + self._reader_unavailable = False + except Exception as e: + self._reader_unavailable = True + verbose_proxy_logger.warning( + "Failed to recreate reader Prisma client: %s. " + "Reads will fall back to the writer until the reader recovers.", + e, + ) + + async def _recreate_reader(self, http_client: Optional[Any] = None) -> None: + """Resolve the reader URL and recreate its Prisma client. + + IAM-enabled readers regenerate their token (host/port/user came from + the parsed reader URL at construction time). Non-IAM readers reuse + the URL stored in `DATABASE_URL_READ_REPLICA`. + """ + if self._reader.iam_token_db_auth: + new_reader_url = self._reader.get_rds_iam_token() + if not new_reader_url: + raise RuntimeError( + "Failed to generate fresh IAM token for read replica" + ) + await self._reader.recreate_prisma_client( + new_reader_url, http_client=http_client + ) + return + reader_url = os.getenv("DATABASE_URL_READ_REPLICA", "") + if not reader_url: + raise RuntimeError( + "DATABASE_URL_READ_REPLICA not set; cannot recreate read replica client" + ) + await self._reader.recreate_prisma_client(reader_url, http_client=http_client) + + def __getattr__(self, name: str) -> Any: + if name in _TOP_LEVEL_READ_METHODS: + target = self._writer if self._reader_unavailable else self._reader + return getattr(target, name) + writer_attr = getattr(self._writer, name) + # Per-model action accessors are non-callable instances that expose + # both `find_many` and `create`. Methods like execute_raw / batch_ / + # tx are callables and stay on the writer untouched. + if ( + not callable(writer_attr) + and hasattr(writer_attr, "find_many") + and hasattr(writer_attr, "create") + ): + try: + reader_attr = getattr(self._reader, name) + except AttributeError: + return writer_attr + return _RoutedActions(writer_attr, reader_attr, self._should_use_reader) + return writer_attr diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 5351391e5e1..e55f3b6e16b 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -242,6 +242,10 @@ async def list_guardrails_v2( gid = guardrail.get("guardrail_id") if gid in seen_guardrail_ids: continue + # Skip stale DB-backed entries — the DB row was deleted (likely by + # another pod) and reconciliation hasn't fired yet on this pod. + if gid is not None and IN_MEMORY_GUARDRAIL_HANDLER.get_source(gid) == "db": + continue if not is_admin: g_team_id = guardrail.get("team_id") if g_team_id is not None and g_team_id not in caller_team_ids: @@ -360,7 +364,7 @@ async def create_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail( - guardrail=cast(Guardrail, result) + guardrail=cast(Guardrail, result), source="db" ) verbose_proxy_logger.info( f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})" @@ -1017,7 +1021,7 @@ async def approve_guardrail_submission( } try: IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail( - guardrail=cast(Guardrail, guardrail_dict) + guardrail=cast(Guardrail, guardrail_dict), source="db" ) verbose_proxy_logger.info( "Approved guardrail %s (ID: %s) and initialized in memory", @@ -1295,10 +1299,18 @@ async def get_guardrail_info(guardrail_id: str): guardrail_id=guardrail_id, prisma_client=prisma_client ) if result is None: - result = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id( + in_memory = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id( guardrail_id=guardrail_id ) - guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG + # Only return config-loaded entries here. A DB-backed entry that's + # missing from the DB is stale (deleted on another pod, awaiting + # reconciliation on this one) and must surface as 404. + if ( + in_memory is not None + and IN_MEMORY_GUARDRAIL_HANDLER.get_source(guardrail_id) == "config" + ): + result = in_memory + guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG if result is None: raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 868b23756d2..aafcc5f1819 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -3,7 +3,7 @@ import importlib import os from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Type, cast +from typing import Any, Dict, List, Literal, Optional, Set, Type, cast import litellm from litellm import Router @@ -403,11 +403,19 @@ class InMemoryGuardrailHandler: Guardrail id to CustomGuardrail object mapping """ + self._sources: Dict[str, Literal["db", "config"]] = {} + """ + Guardrail id to provenance marker. "db" entries are reconciled against + the DB on each polling tick; "config" entries are owned by proxy_config.yaml + and never deleted by reconciliation. + """ + def initialize_guardrail( self, guardrail: Guardrail, config_file_path: Optional[str] = None, llm_router: Optional["Router"] = None, + source: Literal["db", "config"] = "config", ) -> Optional[Guardrail]: """ Initialize a guardrail from a dictionary and add it to the litellm callback manager @@ -420,6 +428,10 @@ class InMemoryGuardrailHandler: verbose_proxy_logger.debug( "guardrail_id already exists in IN_MEMORY_GUARDRAILS" ) + # Honor the caller's source even on the early-return path so a + # racing polling tick or a hot-reload of config can correct an + # entry's provenance. + self._sources[guardrail_id] = source return self.IN_MEMORY_GUARDRAILS[guardrail_id] custom_guardrail_callback: Optional[CustomGuardrail] = None @@ -482,6 +494,11 @@ class InMemoryGuardrailHandler: "skip_system_message_in_guardrail", getattr(litellm_params, "skip_system_message_in_guardrail", None), ) + setattr( + custom_guardrail_callback, + "skip_tool_message_in_guardrail", + getattr(litellm_params, "skip_tool_message_in_guardrail", None), + ) parsed_guardrail = Guardrail( guardrail_id=guardrail.get("guardrail_id"), @@ -492,6 +509,7 @@ class InMemoryGuardrailHandler: # store references to the guardrail in memory self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback + self._sources[guardrail_id] = source return parsed_guardrail @@ -552,7 +570,10 @@ class InMemoryGuardrailHandler: return _guardrail_callback def update_in_memory_guardrail( - self, guardrail_id: str, guardrail: Guardrail + self, + guardrail_id: str, + guardrail: Guardrail, + source: Literal["db", "config"] = "db", ) -> None: """ Update a guardrail in memory @@ -561,6 +582,7 @@ class InMemoryGuardrailHandler: - updates the guardrail params in litellm.callback_manager """ self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail + self._sources[guardrail_id] = source custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.get( guardrail_id @@ -579,6 +601,7 @@ class InMemoryGuardrailHandler: """ # Remove from in-memory storage self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None) + self._sources.pop(guardrail_id, None) # Remove the callback from litellm.callbacks custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.pop( @@ -603,6 +626,34 @@ class InMemoryGuardrailHandler: """ return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) + def get_source(self, guardrail_id: str) -> Optional[Literal["db", "config"]]: + """ + Return the provenance of an in-memory guardrail. + """ + return self._sources.get(guardrail_id) + + def reconcile_db_guardrails(self, db_guardrail_ids: Set[str]) -> List[str]: + """ + Drop in-memory entries that originated from the DB but are no longer + present in db_guardrail_ids. Config-loaded guardrails are never touched. + + Called by the periodic DB polling tick so that a guardrail deleted + on another pod is eventually purged from this pod's memory + callbacks. + """ + stale_ids = [ + guardrail_id + for guardrail_id, source in self._sources.items() + if source == "db" and guardrail_id not in db_guardrail_ids + ] + for guardrail_id in stale_ids: + verbose_proxy_logger.info( + "Reconcile: removing stale DB-backed guardrail '%s' from memory " + "(deleted in DB by another pod)", + guardrail_id, + ) + self.delete_in_memory_guardrail(guardrail_id) + return stale_ids + def _has_guardrail_params_changed( self, guardrail_id: str, new_guardrail: Guardrail ) -> bool: @@ -656,7 +707,10 @@ class InMemoryGuardrailHandler: return len(changed_fields) > 0 def reinitialize_guardrail( - self, guardrail: Guardrail, config_file_path: Optional[str] = None + self, + guardrail: Guardrail, + config_file_path: Optional[str] = None, + source: Literal["db", "config"] = "config", ) -> Optional[Guardrail]: """ Force re-initialization of a guardrail even if it exists in memory. @@ -675,7 +729,7 @@ class InMemoryGuardrailHandler: # Initialize fresh (will add new callback to litellm.callbacks) return self.initialize_guardrail( - guardrail=guardrail, config_file_path=config_file_path + guardrail=guardrail, config_file_path=config_file_path, source=source ) def sync_guardrail_from_db( @@ -696,9 +750,15 @@ class InMemoryGuardrailHandler: f"Guardrail '{guardrail_name}' (ID: {guardrail_id}) params changed, re-initializing..." ) return self.reinitialize_guardrail( - guardrail=guardrail, config_file_path=config_file_path + guardrail=guardrail, + config_file_path=config_file_path, + source="db", ) + # Params unchanged but the entry is still DB-backed; make sure the + # source marker reflects that even if it was previously set differently + # (e.g. a config entry whose UUID later collided with a DB row). + self._sources[guardrail_id] = "db" return self.IN_MEMORY_GUARDRAILS.get(guardrail_id) diff --git a/litellm/proxy/guardrails/init_guardrails.py b/litellm/proxy/guardrails/init_guardrails.py index d742cc223b4..83f1281dc02 100644 --- a/litellm/proxy/guardrails/init_guardrails.py +++ b/litellm/proxy/guardrails/init_guardrails.py @@ -30,6 +30,7 @@ def init_guardrails_v2( guardrail=cast(Guardrail, guardrail), config_file_path=config_file_path, llm_router=llm_router, + source="config", ) if initialized_guardrail: guardrail_list.append(initialized_guardrail) diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index 5b8370fece8..ad58bc7e286 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -253,27 +253,63 @@ class SharedHealthCheckManager: # Always release the lock await self.release_health_check_lock() else: - # Lock not acquired, wait briefly and try to get cached results + # If Redis is not configured, skip polling — there is no cache + # to wait for. + if self.redis_cache is None: + return await perform_health_check( + model_list=model_list, + details=details, + max_concurrency=max_concurrency, + ) + + # Lock not acquired — poll for cached results until the lock + # holder finishes or the lock expires, rather than falling back + # to a redundant local health check after only 2 seconds. verbose_proxy_logger.debug( "Pod %s waiting for other pod to complete health check", self.pod_id ) - # Wait a bit for the other pod to complete - await asyncio.sleep(2) + poll_interval = 5 # seconds between cache checks + max_wait = self.lock_ttl # wait at most as long as the lock can live + elapsed = 0 - # Try to get cached results again - cached_results = await self.get_cached_health_check_results() - if cached_results is not None: - return ( - cached_results.get("healthy_endpoints", []), - cached_results.get("unhealthy_endpoints", []), - {}, - ) + while elapsed < max_wait: + await asyncio.sleep(poll_interval) + elapsed += poll_interval - # Still no cache, fall back to local health check + cached_results = await self.get_cached_health_check_results() + if cached_results is not None: + verbose_proxy_logger.info( + "Pod %s using cached health check results after waiting %ds", + self.pod_id, + elapsed, + ) + return ( + cached_results.get("healthy_endpoints", []), + cached_results.get("unhealthy_endpoints", []), + {}, + ) + + # Check if the lock is still held — if it was released without + # caching (e.g. the holder crashed), stop waiting early. + try: + lock_key = self.get_health_check_lock_key() + current_owner = await self.redis_cache.async_get_cache(lock_key) + if current_owner is None: + verbose_proxy_logger.debug( + "Pod %s detected lock released without cache, stopping wait", + self.pod_id, + ) + break + except Exception: + # Redis hiccup — continue polling rather than crashing out + pass + + # Exhausted wait — fall back to local health check verbose_proxy_logger.warning( - "Pod %s falling back to local health check (no cache available)", + "Pod %s falling back to local health check after waiting %ds (no cache available)", self.pod_id, + elapsed, ) return await perform_health_check( diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 096e23e673d..2c857c0fde8 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1742,29 +1742,54 @@ async def test_model_connection( # Look up model configuration from router if model name is provided # This gets the litellm_params from proxy config (with resolved env vars) config_litellm_params: dict = {} - if model_name and llm_router is not None: + if llm_router is not None: + # Prefer disambiguation by deployment id (`model_info.id`) when + # the caller supplies it. This is required when multiple + # deployments share a `model_name` (e.g. wildcard `openai/*` + # with multiple `api_base` values for failover): the UI's + # "Test Connection" button targets a specific row, and that + # row's id is the only thing that uniquely identifies which + # deployment to probe. Without this, all duplicates collapse + # onto `deployments[0]`. + request_model_info = model_info or {} + request_model_id = request_model_info.get("id") try: - # First try to find by proxy model_name (e.g., "gpt-4o") - deployments = llm_router.get_model_list(model_name=model_name) - - # If not found, try to find by litellm model name (e.g., "azure/gpt-4o") - if not deployments or len(deployments) == 0: - all_deployments = llm_router.get_model_list(model_name=None) - if all_deployments: - for deployment in all_deployments: - if ( - deployment.get("litellm_params", {}).get("model") - == model_name - ): - deployments = [deployment] - break - - if deployments and len(deployments) > 0: - # Use the first deployment's litellm_params as base config - # These already have resolved environment variables from proxy config - config_litellm_params = dict( - deployments[0].get("litellm_params", {}) + deployment_by_id = None + if request_model_id: + deployment_by_id = llm_router.get_deployment( + model_id=request_model_id ) + + if deployment_by_id is not None: + config_litellm_params = deployment_by_id.litellm_params.model_dump( + exclude_none=True + ) + elif model_name: + # Fall back to model_name lookup for callers (e.g. the + # "Add Model" wizard, or curl) that don't supply an id. + # First try to find by proxy model_name (e.g., "gpt-4o") + deployments = llm_router.get_model_list(model_name=model_name) + + # If not found, try to find by litellm model name + # (e.g., "azure/gpt-4o") + if not deployments or len(deployments) == 0: + all_deployments = llm_router.get_model_list(model_name=None) + if all_deployments: + for deployment in all_deployments: + if ( + deployment.get("litellm_params", {}).get("model") + == model_name + ): + deployments = [deployment] + break + + if deployments and len(deployments) > 0: + # Use the first deployment's litellm_params as base + # config. These already have resolved environment + # variables from proxy config. + config_litellm_params = dict( + deployments[0].get("litellm_params", {}) + ) except Exception as e: verbose_proxy_logger.debug( f"Could not find model {model_name} in router: {e}. " diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 95ffafb7bad..9286424878c 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -319,6 +319,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): response_cost=response_cost, ) + if self.dual_cache.redis_cache is not None: + await self._push_in_memory_increments_to_redis() + verbose_proxy_logger.debug( "current state of in memory cache %s", json.dumps( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a63613c5836..b97e7c5e693 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1,5 +1,6 @@ import asyncio import copy +import json import re import time from collections import OrderedDict @@ -794,8 +795,17 @@ class LiteLLMProxyRequestSetup: ) ) for k, v in litellm_logging_metadata_headers.items(): - if v is not None: + if v is None: + continue + # httpx requires header values to be str or bytes; coerce numbers/bools + # to str and JSON-encode dict/list (e.g. user_api_key_spend is float, + # user_api_key_auth_metadata is dict). See #27458. + if isinstance(v, (dict, list)): + returned_headers["x-litellm-{}".format(k)] = json.dumps(v) + elif isinstance(v, (str, bytes)): returned_headers["x-litellm-{}".format(k)] = v + else: + returned_headers["x-litellm-{}".format(k)] = str(v) return returned_headers @@ -1731,6 +1741,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 data=data, user_api_key_dict=user_api_key_dict, pre_alias_model_name=_pre_alias_model, + llm_router=llm_router, ) ## ENFORCED PARAMS CHECK @@ -1864,6 +1875,7 @@ def _apply_credential_overrides_from_model_config( data: dict, user_api_key_dict: UserAPIKeyAuth, pre_alias_model_name: Optional[str] = None, + llm_router: Optional[Router] = None, ) -> None: """ Walk the model_config precedence chain in team/project metadata. @@ -1899,10 +1911,19 @@ def _apply_credential_overrides_from_model_config( if not project_model_config and not team_model_config: return - # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure") + # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure"). + # When the user-facing name has no provider prefix, fall back to the + # deployment's litellm_params so multi-provider defaultconfig entries + # don't silently match the first dict key (#27516). provider: Optional[str] = None if "/" in model_name: provider = model_name.split("/", 1)[0] + elif llm_router is not None: + provider = _resolve_provider_from_deployment( + llm_router=llm_router, + model_name=model_name, + pre_alias_model_name=pre_alias_model_name, + ) credential_name = _resolve_credential_from_model_config( model_name=model_name, @@ -1938,6 +1959,48 @@ def _apply_credential_overrides_from_model_config( ) +def _resolve_provider_from_deployment( + llm_router: Router, + model_name: str, + pre_alias_model_name: Optional[str] = None, +) -> Optional[str]: + """ + Resolve a provider hint from the deployment's litellm_params when the + user-facing model name has no provider prefix. + + Tries the post-alias name first (the resolved model group), then the + pre-alias name. Returns None if no deployment is found or the deployment + has no usable provider info. + """ + candidates = [model_name] + if pre_alias_model_name and pre_alias_model_name != model_name: + candidates.append(pre_alias_model_name) + + for name in candidates: + try: + deployment = llm_router.get_deployment_by_model_group_name( + model_group_name=name + ) + except Exception: + deployment = None + if deployment is None: + continue + + litellm_params = getattr(deployment, "litellm_params", None) + if litellm_params is None: + continue + + custom_provider = getattr(litellm_params, "custom_llm_provider", None) + if custom_provider: + return custom_provider + + deployment_model = getattr(litellm_params, "model", "") or "" + if "/" in deployment_model: + return deployment_model.split("/", 1)[0] + + return None + + def _resolve_credential_from_model_config( model_name: str, project_model_config: Optional[dict], diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index b112af1fe20..4439a55c1c6 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -88,8 +88,8 @@ from litellm.router import Router from litellm.secret_managers.main import get_secret from litellm.types.proxy.management_endpoints.key_management_endpoints import ( BulkUpdateKeyRequest, - BulkUpdateKeyRequestItem, BulkUpdateKeyResponse, + BulkUpdateTeamKeysRequest, FailedKeyUpdate, SuccessfulKeyUpdate, ) @@ -1881,7 +1881,7 @@ async def _get_and_validate_existing_key( async def _process_single_key_update( - key_update_item: BulkUpdateKeyRequestItem, + update_key_request: UpdateKeyRequest, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: Optional[str], prisma_client: Optional[PrismaClient], @@ -1889,6 +1889,7 @@ async def _process_single_key_update( proxy_logging_obj: Any, llm_router: Optional[Router], user_custom_key_update: Optional[Callable] = None, + existing_key_row: Optional[LiteLLM_VerificationToken] = None, ) -> Dict[str, Any]: """ Process a single key update with all validations and checks. @@ -1897,13 +1898,14 @@ async def _process_single_key_update( including validation, permission checks, team checks, and database updates. Args: - key_update_item: The key update request item + update_key_request: Fully-constructed UpdateKeyRequest for the target key user_api_key_dict: The authenticated user's API key info litellm_changed_by: Optional header for tracking who made the change prisma_client: Prisma client instance user_api_key_cache: User API key cache proxy_logging_obj: Proxy logging object llm_router: LLM router instance + existing_key_row: Optional pre-fetched key row to avoid redundant lookups Returns: Dict containing the updated key information @@ -1912,13 +1914,14 @@ async def _process_single_key_update( HTTPException: For various validation and permission errors """ # Validate max_budget - _validate_max_budget(key_update_item.max_budget) + _validate_max_budget(update_key_request.max_budget) # Get and validate existing key - existing_key_row = await _get_and_validate_existing_key( - token=key_update_item.key, - prisma_client=prisma_client, - ) + if existing_key_row is None: + existing_key_row = await _get_and_validate_existing_key( + token=update_key_request.key, + prisma_client=prisma_client, + ) # Check team member permissions if prisma_client is not None: @@ -1930,15 +1933,6 @@ async def _process_single_key_update( user_api_key_cache=user_api_key_cache, ) - # Create UpdateKeyRequest from BulkUpdateKeyRequestItem - update_key_request = UpdateKeyRequest( - key=key_update_item.key, - budget_id=key_update_item.budget_id, - max_budget=key_update_item.max_budget, - team_id=key_update_item.team_id, - tags=key_update_item.tags, - ) - # Custom key update hook if user_custom_key_update is not None: if inspect.iscoroutinefunction(user_custom_key_update): @@ -2003,12 +1997,12 @@ async def _process_single_key_update( detail={"error": "Database not connected"}, ) - _data = {**non_default_values, "token": key_update_item.key} - response = await prisma_client.update_data(token=key_update_item.key, data=_data) + _data = {**non_default_values, "token": update_key_request.key} + response = await prisma_client.update_data(token=update_key_request.key, data=_data) # Delete cache await _delete_cache_key_object( - hashed_token=_hash_token_if_needed(key_update_item.key), + hashed_token=_hash_token_if_needed(update_key_request.key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -2598,9 +2592,15 @@ async def bulk_update_keys( for key_update_item in data.keys: try: - # Process single key update using reusable function + update_key_request = UpdateKeyRequest( + key=key_update_item.key, + budget_id=key_update_item.budget_id, + max_budget=key_update_item.max_budget, + team_id=key_update_item.team_id, + tags=key_update_item.tags, + ) updated_key_info = await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, prisma_client=prisma_client, @@ -2665,6 +2665,223 @@ async def bulk_update_keys( ) +def _build_failed_team_key_update( + token: str, + exception: Exception, + existing_key_row: Optional[LiteLLM_VerificationToken], +) -> FailedKeyUpdate: + """Normalize an exception from the per-key update loop into a FailedKeyUpdate.""" + if isinstance(exception, HTTPException): + detail = exception.detail + if isinstance(detail, dict): + error_message = detail.get("error", str(exception)) + else: + error_message = str(detail) + elif isinstance(exception, ProxyException): + error_message = exception.message + else: + error_message = str(exception) + + key_info: Optional[Dict[str, Any]] = None + if existing_key_row is not None: + if hasattr(existing_key_row, "model_dump"): + key_info = existing_key_row.model_dump() + elif hasattr(existing_key_row, "dict"): + key_info = existing_key_row.dict() + if key_info: + key_info.pop("token", None) + + return FailedKeyUpdate(key=token, key_info=key_info, failed_reason=error_message) + + +@router.post( + "/team/key/bulk_update", + tags=["key management"], + dependencies=[Depends(user_api_key_auth)], + response_model=BulkUpdateKeyResponse, +) +@management_endpoint_wrapper +async def bulk_update_team_keys( + data: BulkUpdateTeamKeysRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +): + """ + Apply one update payload to many keys inside a single team. + + Pass `team_id` plus either `key_ids` or `all_keys_in_team=True`. The + `update_fields` payload is broadcast to every selected key. Per-key + failures are returned in `failed_updates` rather than aborting the batch. + + Callable by proxy admins, or by team admins with `KEY_UPDATE` permission. + """ + from litellm.proxy.proxy_server import ( + llm_router, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + user_custom_key_update, + ) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": "Database not connected"}, + ) + + if not data.team_id: + raise HTTPException( + status_code=400, + detail={"error": "team_id is required"}, + ) + + MAX_BATCH_SIZE = 500 + if data.key_ids is not None and len(data.key_ids) > MAX_BATCH_SIZE: + raise HTTPException( + status_code=400, + detail={ + "error": f"Maximum {MAX_BATCH_SIZE} keys can be updated at once. Found {len(data.key_ids)} key_ids." + }, + ) + + if data.all_keys_in_team: + # "all" excludes blocked/expired — bulk refresh shouldn't revive a key an admin disabled. + # `blocked` is Boolean? with no default; `/key/generate` writes NULL. Prisma's `NOT` + # excludes NULLs, so explicitly OR `false` with `null` to include them. + now = datetime.now(timezone.utc) + existing_keys = await prisma_client.db.litellm_verificationtoken.find_many( + where={ + "team_id": data.team_id, + "AND": [ + {"OR": [{"blocked": False}, {"blocked": None}]}, + {"OR": [{"expires": None}, {"expires": {"gt": now}}]}, + ], + }, + order={"token": "asc"}, + take=MAX_BATCH_SIZE + 1, + ) + if len(existing_keys) > MAX_BATCH_SIZE: + raise HTTPException( + status_code=400, + detail={ + "error": f"Team {data.team_id} has more than {MAX_BATCH_SIZE} keys. Use `key_ids` to update in batches of {MAX_BATCH_SIZE}." + }, + ) + requested_tokens = [row.token for row in existing_keys] + else: + if data.key_ids is None or len(data.key_ids) == 0: + raise HTTPException( + status_code=400, + detail={ + "error": "key_ids must be provided when all_keys_in_team is False" + }, + ) + # Dedupe by hashed form — duplicates collapse to one update. + requested_tokens = [] + hashed_key_ids = [] + seen_hashes = set() + for k in data.key_ids: + h = _hash_token_if_needed(k) + if h in seen_hashes: + continue + seen_hashes.add(h) + requested_tokens.append(k) + hashed_key_ids.append(h) + existing_keys = await prisma_client.db.litellm_verificationtoken.find_many( + where={"team_id": data.team_id, "token": {"in": hashed_key_ids}} + ) + + # Anchor membership check on data.team_id (not existing_keys[0]); empty result must still gate non-admins. + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: + auth_anchor = ( + existing_keys[0] + if existing_keys + else LiteLLM_VerificationToken( + token="__team_scope_auth_check__", + team_id=data.team_id, + models=[], + ) + ) + await TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint( + user_api_key_dict=user_api_key_dict, + route=KeyManagementRoutes.KEY_UPDATE, + prisma_client=prisma_client, + existing_key_row=auth_anchor, + user_api_key_cache=user_api_key_cache, + ) + + # Block metadata.allowed_passthrough_routes for non-admins — the runtime + # route checker reads it from key/team metadata to grant passthrough. + _check_passthrough_routes_caller_permission( + data=data.update_fields, user_api_key_dict=user_api_key_dict + ) + + if not requested_tokens: + raise HTTPException( + status_code=404, + detail={"error": f"No keys found for team {data.team_id}"}, + ) + + existing_by_token = {row.token: row for row in existing_keys} + update_field_dict = data.update_fields.model_dump(exclude_unset=True) + + successful_updates: List[SuccessfulKeyUpdate] = [] + failed_updates: List[FailedKeyUpdate] = [] + + for token in requested_tokens: + db_token = _hash_token_if_needed(token) + try: + if db_token not in existing_by_token: + raise HTTPException( + status_code=404, + detail={"error": f"Key not found in team {data.team_id}"}, + ) + + # team_id from validated scope, never user payload — drives _check_team_key_limits. + update_key_request = UpdateKeyRequest( + key=token, + team_id=data.team_id, + **update_field_dict, + ) + updated_key_info = await _process_single_key_update( + update_key_request=update_key_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + user_custom_key_update=user_custom_key_update, + existing_key_row=existing_by_token[db_token], + ) + + successful_updates.append( + SuccessfulKeyUpdate(key=token, key_info=updated_key_info) + ) + + except Exception as e: + # Log the hashed prefix — `token` may be a raw sk-... and ERROR logs persist. + verbose_proxy_logger.exception( + f"Failed to update key {db_token[:12]}... in team {data.team_id}: {e}" + ) + failed_updates.append( + _build_failed_team_key_update( + token=token, + exception=e, + existing_key_row=existing_by_token.get(db_token), + ) + ) + + return BulkUpdateKeyResponse( + total_requested=len(requested_tokens), + successful_updates=successful_updates, + failed_updates=failed_updates, + ) + + async def validate_key_team_change( key: LiteLLM_VerificationToken, team: LiteLLM_TeamTable, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 7bda0f87ccd..f2e64fdde83 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -142,6 +142,7 @@ if MCP_AVAILABLE: MCPOAuthUserCredentialRequest, MCPOAuthUserCredentialStatus, MCPSubmissionsSummary, + MCPTransport, MCPUserCredentialListItem, MCPUserCredentialRequest, MCPUserCredentialResponse, @@ -1070,6 +1071,24 @@ if MCP_AVAILABLE: }, ) + # stdio servers spawn a local subprocess on the proxy host with the + # configured command + args, so accepting them from non-admin callers + # would let a team member propose a server config that an admin could + # rubber-stamp into local code execution. Restrict stdio submission to + # the admin POST /v1/mcp/server path or to config.yaml. + if payload.transport == MCPTransport.stdio: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": ( + "stdio MCP servers cannot be submitted via the user " + "registration workflow. Ask a proxy admin to add this " + "server via POST /v1/mcp/server or to declare it in " + "config.yaml." + ) + }, + ) + prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 0e60820aab1..49d9b67a28a 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -12,12 +12,13 @@ All /tag management endpoints import asyncio import json -from typing import TYPE_CHECKING, Dict, List, Optional +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import UserAPIKeyAuth, user_api_key_has_admin_view from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, @@ -39,6 +40,72 @@ if TYPE_CHECKING: router = APIRouter() +async def _get_internal_user_api_keys( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, +) -> List[str]: + user_role = user_api_key_dict.user_role + if user_role is None or not user_role.is_internal_user_role: + return [] + + user_api_keys = set() + if user_api_key_dict.api_key: + user_api_keys.add(user_api_key_dict.api_key) + + user_id = user_api_key_dict.user_id + if user_id is None: + return sorted(user_api_keys) + + key_records = await prisma_client.db.litellm_verificationtoken.find_many( + where={"user_id": user_id}, + select={"token": True}, + ) + user_api_keys.update( + key_record.token + for key_record in key_records + if getattr(key_record, "token", None) + ) + + return sorted(user_api_keys) + + +async def _get_tag_list_scope( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, +) -> Optional[Dict[str, dict]]: + user_role = user_api_key_dict.user_role + if user_api_key_has_admin_view(user_api_key_dict) or ( + user_role is None or not user_role.is_internal_user_role + ): + return None + + scoped_api_keys = await _get_internal_user_api_keys( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + return {"api_key": {"in": scoped_api_keys}} + + +async def _get_tag_daily_activity_api_key_filter( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, + requested_api_key: Optional[str], +) -> Optional[Union[str, List[str]]]: + user_role = user_api_key_dict.user_role + if user_api_key_has_admin_view(user_api_key_dict) or ( + user_role is None or not user_role.is_internal_user_role + ): + return requested_api_key + + scoped_api_keys = await _get_internal_user_api_keys( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + if requested_api_key is not None: + return requested_api_key if requested_api_key in scoped_api_keys else [] + return scoped_api_keys + + async def _get_model_names(prisma_client, model_ids: list) -> Dict[str, str]: """Helper function to get model names from model IDs""" try: @@ -395,6 +462,32 @@ async def info_tag( raise HTTPException(status_code=500, detail=str(e)) +def _validate_tag_list_date_range( + start_date: Optional[str], end_date: Optional[str] +) -> None: + """Require both dates together, and enforce YYYY-MM-DD format with start <= end.""" + if (start_date is None) != (end_date is None): + raise HTTPException( + status_code=400, + detail="start_date and end_date must be provided together", + ) + if start_date is None: + return + try: + start = datetime.strptime(start_date, "%Y-%m-%d") + end = datetime.strptime(end_date, "%Y-%m-%d") # type: ignore[arg-type] + except ValueError as e: + raise HTTPException( + status_code=400, + detail=f"Invalid date format, expected YYYY-MM-DD: {e}", + ) + if start > end: + raise HTTPException( + status_code=400, + detail="start_date must be on or before end_date", + ) + + @router.get( "/tag/list", tags=["tag management"], @@ -402,6 +495,18 @@ async def info_tag( ) async def list_tags( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + start_date: Optional[str] = Query( + None, + description=( + "Optional start date (YYYY-MM-DD). When provided together with " + "end_date, dynamic tags are limited to those active in the window. " + "Stored tags are always returned." + ), + ), + end_date: Optional[str] = Query( + None, + description="Optional end date (YYYY-MM-DD). Must be given with start_date.", + ), ): """ List all available tags with their budget information. @@ -411,10 +516,44 @@ async def list_tags( if prisma_client is None: raise HTTPException(status_code=500, detail="Database not connected") + _validate_tag_list_date_range(start_date, end_date) + try: + tag_scope = await _get_tag_list_scope( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + ) + + ## QUERY DYNAMIC TAGS ## + # Use group_by instead of find_many(distinct=["tag"]). + # Prisma's distinct fetches all columns for all rows and deduplicates + # in application code, which is extremely slow on large tables. + # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood + dynamic_tag_where: Dict[str, Any] = {"tag": {"not": None}} + if tag_scope: + dynamic_tag_where = {**dynamic_tag_where, **tag_scope} + if start_date is not None and end_date is not None: + dynamic_tag_where["date"] = {"gte": start_date, "lte": end_date} + + dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by( + by=["tag"], + where=dynamic_tag_where, + min={"created_at": True}, + max={"updated_at": True}, + ) + + used_tag_names = [row["tag"] for row in dynamic_tag_rows if row["tag"]] + if tag_scope is not None and not used_tag_names: + return [] + + stored_tag_where = ( + {"tag_name": {"in": used_tag_names}} if tag_scope is not None else None + ) + ## QUERY STORED TAGS ## tag_records = await prisma_client.db.litellm_tagtable.find_many( - include={"litellm_budget_table": True} + where=stored_tag_where, + include={"litellm_budget_table": True}, ) stored_tag_names = set() @@ -448,18 +587,6 @@ async def list_tags( list_of_tags.append(tag_dict) - ## QUERY DYNAMIC TAGS ## - # Use group_by instead of find_many(distinct=["tag"]). - # Prisma's distinct fetches all columns for all rows and deduplicates - # in application code, which is extremely slow on large tables. - # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood - dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by( - by=["tag"], - where={"tag": {"not": None}}, - min={"created_at": True}, - max={"updated_at": True}, - ) - dynamic_tag_config = [ { "name": row["tag"], @@ -527,6 +654,7 @@ async def get_tag_daily_activity( api_key: Optional[str] = None, page: int = 1, page_size: int = 10, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Get daily activity for specific tags or all tags. @@ -545,8 +673,18 @@ async def get_tag_daily_activity( """ from litellm.proxy.proxy_server import prisma_client + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected") + # Convert comma-separated tags string to list if provided tag_list = tags.split(",") if tags else None + scoped_api_key_filter = await _get_tag_daily_activity_api_key_filter( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + requested_api_key=api_key, + ) + if scoped_api_key_filter == []: + return SpendAnalyticsPaginatedResponse(results=[]) return await get_daily_activity( prisma_client=prisma_client, @@ -557,7 +695,7 @@ async def get_tag_daily_activity( start_date=start_date, end_date=end_date, model=model, - api_key=api_key, + api_key=scoped_api_key_filter, page=page, page_size=page_size, # metadata_metrics_func=None because litellm_dailytagspend rows are diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 259624f1e18..65bcca23c30 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3805,6 +3805,7 @@ async def _build_team_list_where_conditions( organization_id: Optional[str], user_id: Optional[str], use_deleted_table: bool, + search: Optional[str] = None, org_admin_org_ids: Optional[List[str]] = None, user_api_key_cache: Optional[Any] = None, proxy_logging_obj: Optional[Any] = None, @@ -3826,6 +3827,12 @@ async def _build_team_list_where_conditions( "mode": "insensitive", # Case-insensitive search } + if search: + where_conditions["OR"] = [ + {"team_id": search}, + {"team_alias": {"contains": search, "mode": "insensitive"}}, + ] + if organization_id: where_conditions["organization_id"] = organization_id elif org_admin_org_ids is not None: @@ -4019,6 +4026,10 @@ async def list_team_v2( default=None, description="Only return teams which this 'team_alias' belongs to. Supports partial matching.", ), + search: Optional[str] = fastapi.Query( + default=None, + description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).", + ), page: int = fastapi.Query( default=1, description="Page number for pagination", ge=1 ), @@ -4104,6 +4115,7 @@ async def list_team_v2( organization_id=organization_id, user_id=user_id, use_deleted_table=use_deleted_table, + search=search, org_admin_org_ids=org_admin_org_ids, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 74ee7c7220d..6e2e2bedac1 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -740,7 +740,7 @@ def generic_response_convertor( all_teams = [] if sso_jwt_handler is not None: - team_ids = sso_jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + team_ids = sso_jwt_handler.get_all_jwt_team_ids(cast(dict, response)) all_teams.extend(team_ids) if team_mappings is not None and team_mappings.team_ids_jwt_field is not None: @@ -755,7 +755,7 @@ def generic_response_convertor( f"Loaded team_ids from DB team_mappings.team_ids_jwt_field='{team_mappings.team_ids_jwt_field}': {team_ids_from_db_mapping}" ) else: - team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) + team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response)) all_teams.extend(team_ids) # Determine user role based on role_mappings if available @@ -4078,6 +4078,8 @@ async def debug_sso_callback(request: Request): redirect_url += "/sso/debug/callback" result = None + received_response: Optional[dict] = None + access_token_payload: Optional[dict] = None if google_client_id is not None: result = await GoogleSSOHandler.get_google_callback_response( request=request, @@ -4094,12 +4096,14 @@ async def debug_sso_callback(request: Request): ) elif generic_client_id is not None: - result, _, _ = await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, + result, received_response, access_token_payload = ( + await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, + ) ) # If result is None, return a basic error message @@ -4128,10 +4132,32 @@ async def debug_sso_callback(request: Request): except Exception as e: filtered_result[key] = f"Complex value (not displayable): {str(e)}" + # Defense-in-depth: ensure no bearer tokens leak into the rendered HTML even if + # a non-conforming IdP places them in its userinfo response. + safe_raw_claims = { + k: v + for k, v in (received_response or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + safe_access_token_claims = { + k: v + for k, v in (access_token_payload or {}).items() + if k not in _OAUTH_TOKEN_FIELDS + } + + sso_payload = { + "parsed_by_proxy": filtered_result, + "raw_claims": safe_raw_claims, + "access_token_claims": safe_access_token_claims, + } + # Replace the placeholder in the template with the actual data + sso_payload_json = json.dumps(sso_payload, indent=2, default=str).replace( + " dict: + """ + Add a synthetic GET stub for each WebSocket route so it appears in Swagger UI. + + Merges into any existing path entry rather than replacing it — a WebSocket route + that shares its path with an HTTP route must not erase the HTTP operation. If + a "get" operation is already documented on the path, the WebSocket stub is + skipped to preserve the real GET. + """ + for route in websocket_routes: + base_path = route.path.split("{")[0].rstrip("?") + + parameters = [] + try: + if hasattr(route, "dependant") and route.dependant is not None: + # Handle both FastAPI <0.120 and >=0.120 + query_params = getattr(route.dependant, "query_params", []) + if query_params: + for param in query_params: + parameters.append( + { + "name": param.name, + "in": "query", + "required": param.required, + "schema": {"type": "string"}, + } + ) + except (AttributeError, TypeError): + pass + + path_entry = openapi_schema["paths"].setdefault(base_path, {}) + if "get" not in path_entry: + path_entry["get"] = { + "summary": f"WebSocket: {route.name or base_path}", + "description": "WebSocket connection endpoint", + "operationId": f"websocket_{route.name or base_path.replace('/', '_')}", + "parameters": parameters, + "responses": {"101": {"description": "WebSocket Protocol Switched"}}, + "tags": ["WebSocket"], + } + + return openapi_schema + + def get_openapi_schema(): if app.openapi_schema: return app.openapi_schema @@ -1082,43 +1129,11 @@ def get_openapi_schema(): route for route in app.routes if isinstance(route, APIWebSocketRoute) ] - # Add each WebSocket route to the schema - for route in websocket_routes: - # Get the base path without query parameters - base_path = route.path.split("{")[0].rstrip("?") - - # Extract parameters from the route - parameters = [] - try: - if hasattr(route, "dependant") and route.dependant is not None: - # Handle both FastAPI <0.120 and >=0.120 - query_params = getattr(route.dependant, "query_params", []) - if query_params: - for param in query_params: - parameters.append( - { - "name": param.name, - "in": "query", - "required": param.required, - "schema": { - "type": "string" - }, # You can make this more specific if needed - } - ) - except (AttributeError, TypeError): - # If we can't access query_params, continue without them - pass - - openapi_schema["paths"][base_path] = { - "get": { - "summary": f"WebSocket: {route.name or base_path}", - "description": "WebSocket connection endpoint", - "operationId": f"websocket_{route.name or base_path.replace('/', '_')}", - "parameters": parameters, - "responses": {"101": {"description": "WebSocket Protocol Switched"}}, - "tags": ["WebSocket"], - } - } + # Add a synthetic GET stub for each so they render in Swagger UI, + # without clobbering existing HTTP operations on the same path. + openapi_schema = _inject_websocket_stubs_into_openapi_schema( + openapi_schema, websocket_routes + ) # Add LLM API request schema bodies for documentation from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec @@ -5936,10 +5951,20 @@ class ProxyConfig: verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) ) + db_guardrail_ids: set = set() for guardrail in guardrails_in_db: + guardrail_id = guardrail.get("guardrail_id") + if guardrail_id: + db_guardrail_ids.add(guardrail_id) IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( guardrail=cast(Guardrail, guardrail), ) + + # Drop in-memory DB-backed entries whose row was deleted on another + # pod. Config-loaded entries are never touched. + IN_MEMORY_GUARDRAIL_HANDLER.reconcile_db_guardrails( + db_guardrail_ids=db_guardrail_ids + ) except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {}".format( @@ -6750,27 +6775,64 @@ class ProxyStartupEvent: "budget_duration not set on Proxy. budget_duration is required to use max_budget." ) - # add proxy budget to db in the user table asyncio.create_task( - generate_key_helper_fn( # type: ignore - request_type="user", - table_name="user", - user_id=litellm_proxy_budget_name, - duration=None, - models=[], - aliases={}, - config={}, - spend=0, - max_budget=litellm.max_budget, - budget_duration=litellm.budget_duration, - query_type="update_data", - update_key_values={ - "max_budget": litellm.max_budget, - "budget_duration": litellm.budget_duration, - }, - ) + cls._upsert_proxy_budget_with_reset_at_backfill(litellm_proxy_budget_name) ) + @classmethod + async def _upsert_proxy_budget_with_reset_at_backfill( + cls, litellm_proxy_budget_name: str + ) -> None: + """ + Upsert the proxy admin user row with the configured max_budget / + budget_duration, then backfill budget_reset_at if currently NULL. + + The backfill uses `WHERE budget_reset_at IS NULL` so it only fires + when the row pre-existed without a reset schedule (e.g. row created + via a different path before the proxy budget was configured). On + subsequent restarts it no-ops, so an active reset window is never + slid forward. + """ + await generate_key_helper_fn( # type: ignore + request_type="user", + table_name="user", + user_id=litellm_proxy_budget_name, + duration=None, + models=[], + aliases={}, + config={}, + spend=0, + max_budget=litellm.max_budget, + budget_duration=litellm.budget_duration, + query_type="update_data", + update_key_values={ + "max_budget": litellm.max_budget, + "budget_duration": litellm.budget_duration, + }, + ) + + # Without this, the upsert leaves budget_reset_at=NULL on rows that + # took the UPDATE path, and reset_budget_for_litellm_users never + # matches them (NULL < now() is unknown in SQL) — so the proxy-wide + # spend cap blocks forever once it's hit. + if prisma_client is not None and litellm.budget_duration is not None: + try: + await prisma_client.db.litellm_usertable.update_many( + where={ + "user_id": litellm_proxy_budget_name, + "budget_reset_at": None, + }, + data={ + "budget_reset_at": get_budget_reset_time( + budget_duration=litellm.budget_duration + ) + }, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to backfill budget_reset_at on proxy admin row: %s", e + ) + @classmethod async def _warm_global_spend_cache( cls, @@ -8828,6 +8890,7 @@ def _realtime_query_params_template( return tuple(params) +@app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") async def realtime_websocket_endpoint( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 1d296611bfc..200a17e2368 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -95,11 +95,9 @@ async def reserve_budget_for_request( route=route, llm_router=llm_router, ) - if reservation_cost is None: - reservation_cost = await _get_smallest_remaining_budget( - counters=counters, - current_spend_by_counter_key=current_spend_by_counter_key, - ) + # estimate_request_max_cost still returns None when the model is unknown + # to the cost map (no token-priced cost fields, e.g. image/audio routes). + # In that case we fall back to read-time enforcement only. if reservation_cost is None or reservation_cost <= 0: return None @@ -553,32 +551,6 @@ def _coerce_window(window: Any) -> dict: return {} -async def _get_smallest_remaining_budget( - counters: List[_BudgetCounter], - current_spend_by_counter_key: Dict[str, float], -) -> Optional[float]: - remaining_budget: Optional[float] = None - for counter in counters: - current_spend = await _get_current_counter_value(counter=counter) - current_spend_by_counter_key[counter.counter_key] = current_spend - remaining = counter.max_budget - current_spend - if remaining <= 0: - raise litellm.BudgetExceededError( - current_cost=current_spend, - max_budget=counter.max_budget, - message=( - "Budget has been exceeded! " - f"{counter.entity_type}={counter.entity_id} " - f"Current cost: {current_spend}, " - f"Max budget: {counter.max_budget}" - ), - ) - remaining_budget = ( - remaining if remaining_budget is None else min(remaining_budget, remaining) - ) - return remaining_budget - - async def _reserve_counter( counter: _BudgetCounter, reservation_cost: float, @@ -855,6 +827,13 @@ def _estimate_request_max_cost_for_model( if model_info is None: return None + image_cost = _estimate_image_generation_cost( + request_body=request_body, + model_info=model_info, + ) + if image_cost is not None: + return image_cost + input_cost_per_token = _to_float(model_info.get("input_cost_per_token")) output_cost_per_token = _to_float(model_info.get("output_cost_per_token")) input_tokens = _estimate_input_tokens( @@ -886,6 +865,44 @@ def _estimate_request_max_cost_for_model( return cost +def _estimate_image_generation_cost( + request_body: dict, + model_info: Dict[str, Any], +) -> Optional[float]: + """ + Reserve `n × per-image cost` for image-generation requests so concurrent + requests against a depleted budget cannot all slip past the admission gate + onto the provider. Token-based pricing (e.g. gpt-image-1) is handled by + the chat-route token path; per-pixel and size/quality-tiered pricing + (DALL-E 2 size variants, premium tiers) are not handled here and fall + through to read-time enforcement. + + The "output" vs "input" cost-per-image naming is inconsistent across + providers — OpenAI's dall-e-3 entry uses ``input_cost_per_image`` while + aiml/dall-e-3 uses ``output_cost_per_image`` — so both are summed. + """ + # Gate strictly on `mode`. Several chat and embedding models carry + # ``input_cost_per_image`` / ``output_cost_per_image`` to price multimodal + # *vision input* (e.g. ``gemini-3.1-pro-preview``, ``azure/gpt-realtime-*``, + # ``amazon.titan-embed-image-v1``). Falling back to "treat as image-gen if + # an image cost field is present" would short-circuit the token-priced + # path for those models and reserve a fraction of a cent instead of the + # true per-token cost. All real image-generation entries in + # ``model_prices_and_context_window.json`` carry ``mode: image_generation`` + # or ``mode: image_edit``, so the field-presence fallback is unnecessary. + if model_info.get("mode") not in ("image_generation", "image_edit"): + return None + + output_cost_per_image = _to_float(model_info.get("output_cost_per_image")) + input_cost_per_image = _to_float(model_info.get("input_cost_per_image")) + cost_per_image = (output_cost_per_image or 0.0) + (input_cost_per_image or 0.0) + if cost_per_image <= 0: + return None + + n = _to_int(request_body.get("n")) or 1 + return cost_per_image * max(n, 1) + + def _get_model_cost_info( model: str, llm_router: Optional[Router], @@ -946,6 +963,9 @@ def _estimate_input_tokens( return None +DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK = 16384 + + def _estimate_output_tokens( request_body: dict, route: str, @@ -954,15 +974,27 @@ def _estimate_output_tokens( if _is_input_only_route(route=route): return 0 + requested: Optional[int] = None for key in ("max_completion_tokens", "max_tokens", "max_output_tokens"): - max_tokens = _to_int(request_body.get(key)) - if max_tokens is not None: - return max_tokens + requested = _to_int(request_body.get(key)) + if requested is not None: + break - # If the caller did not cap output tokens, avoid reserving a model's - # theoretical maximum context. The caller can still admit one request by - # reserving the smallest remaining budget in reserve_budget_for_request(). - return None + # Clamp at min(requested-or-default, model_max-or-default). Two purposes: + # (1) Without an explicit cap we still need a finite reservation so the + # atomic admission counter actually bounds concurrent in-flight cost + # (mirrors parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE). + # (2) An adversarial caller cannot send max_tokens=999999999 to inflate + # the reservation up to remaining team headroom and pin the counter + # at the cap — the model can only physically emit max_output_tokens + # anyway, so reserving more is both wasteful and a DoS surface. + model_ceiling = ( + _to_int(model_info.get("max_output_tokens")) + or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + ) + if requested is None: + requested = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK + return min(requested, model_ceiling) def _count_text_tokens(model: str, text: Any) -> int: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a52dc8e55fb..2998555db1a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -113,7 +113,11 @@ from litellm.proxy.db.exception_handler import ( call_with_db_reconnect_retry, ) from litellm.proxy.db.log_db_metrics import log_db_metrics -from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.prisma_client import ( + PrismaWrapper, + parse_iam_endpoint_from_url, +) +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -2569,24 +2573,101 @@ class PrismaClient: raise Exception( "Unable to find Prisma binaries. Please run 'prisma generate' first." ) + iam_flag = ( + self.iam_token_db_auth if self.iam_token_db_auth is not None else False + ) + # When read-replica routing is on, tag log lines with [writer]/[reader] + # so the two wrappers' interleaved IAM refresh logs can be told apart. + # Single-DB deployments get an empty prefix (logs unchanged). + read_replica_url = os.getenv("DATABASE_URL_READ_REPLICA") + writer_log_prefix = "[writer]" if read_replica_url else "" if http_client is not None: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(http=http_client), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, ) else: - self.db = PrismaWrapper( + writer_wrapper = PrismaWrapper( original_prisma=Prisma(), - iam_token_db_auth=( - self.iam_token_db_auth - if self.iam_token_db_auth is not None - else False - ), - ) # Client to connect to Prisma db + iam_token_db_auth=iam_flag, + log_prefix=writer_log_prefix, + ) + + # Optional read-replica routing. When DATABASE_URL_READ_REPLICA is set, + # reads (find_*, count, group_by, query_raw/_first) are routed to the + # reader endpoint and writes stay on the writer. Falls back to the + # writer-only wrapper when the env var is unset, preserving existing + # single-DB deployments. + self.db: Union[PrismaWrapper, RoutingPrismaWrapper] + if read_replica_url: + try: + # If IAM auth is enabled, the reader refreshes its own token on + # the same cadence as the writer. We parse the static endpoint + # pieces (host/port/user/db) once from the reader URL — only + # the IAM token rotates after that. + reader_iam_endpoint = ( + parse_iam_endpoint_from_url(read_replica_url) if iam_flag else None + ) + # Mint a fresh IAM token for the reader BEFORE constructing the + # Prisma client. Mirrors what `proxy_cli.py` already does for + # the writer (proxy_cli.py:812-832) — without this, the reader + # Prisma is built with whatever placeholder URL the user + # supplied (no real token), and the first query falls through + # to the synchronous fallback path in + # `PrismaWrapper.__getattr__`, which deadlocks the event loop + # and times out after 30s. + if iam_flag and reader_iam_endpoint is not None: + from litellm.proxy.auth.rds_iam_token import ( + generate_iam_auth_token, + ) + + reader_token = generate_iam_auth_token( + db_host=reader_iam_endpoint.host, + db_port=reader_iam_endpoint.port, + db_user=reader_iam_endpoint.user, + ) + read_replica_url = reader_iam_endpoint.build_url(reader_token) + os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url + reader_kwargs: Dict[str, Any] = { + "datasource": {"url": read_replica_url} + } + if http_client is not None: + reader_prisma = Prisma(http=http_client, **reader_kwargs) + else: + reader_prisma = Prisma(**reader_kwargs) + reader_wrapper = PrismaWrapper( + original_prisma=reader_prisma, + iam_token_db_auth=iam_flag, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=reader_iam_endpoint, + recreate_uses_datasource=True, + log_prefix="[reader]", + ) + self.db = RoutingPrismaWrapper( + writer=writer_wrapper, reader=reader_wrapper + ) + verbose_proxy_logger.info( + "PrismaClient: read-replica routing enabled via DATABASE_URL_READ_REPLICA" + + (" (with IAM token auto-refresh)" if iam_flag else "") + ) + except Exception as e: + # Reader is opt-in; never let its construction fail proxy + # startup. Mirrors the runtime contract from + # `RoutingPrismaWrapper.connect`: reader-side failures are + # logged and we keep serving traffic via the writer alone. + # This recovers from transient AWS STS hiccups during the + # reader IAM token mint, malformed DATABASE_URL_READ_REPLICA, + # and Prisma construction errors. Operator restart is required + # to retry read-routing once the underlying issue is resolved. + verbose_proxy_logger.warning( + "Failed to initialize read replica Prisma client: %s. " + "Falling back to writer-only mode (no read routing) until proxy restart.", + e, + ) + self.db = writer_wrapper + else: + self.db = writer_wrapper # Client to connect to Prisma db self._db_reconnect_lock = asyncio.Lock() self._db_health_watchdog_task: Optional[asyncio.Task] = None self._db_last_reconnect_attempt_ts: float = 0.0 @@ -2624,6 +2705,13 @@ class PrismaClient: self._engine_wait_thread: Optional[threading.Thread] = None verbose_proxy_logger.debug("Success - Created Prisma Client") + @property + def writer_db(self) -> PrismaWrapper: + """Underlying writer Prisma wrapper, regardless of read-replica routing.""" + if isinstance(self.db, RoutingPrismaWrapper): + return self.db.writer + return self.db + def get_request_status( self, payload: Union[dict, SpendLogsPayload] ) -> Literal["success", "failure"]: @@ -4272,7 +4360,10 @@ class PrismaClient: self._cleanup_engine_watcher() await self.db.recreate_prisma_client(db_url) await self._start_engine_watcher() - await self.db.query_raw("SELECT 1") + # Smoke-test the writer specifically; query_raw on the routing + # wrapper sends to the reader, which would not validate the + # newly-recreated writer engine. + await self.writer_db.query_raw("SELECT 1") await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout) @@ -4886,10 +4977,10 @@ class ProxyUpdateSpend: timeout=timedelta(seconds=60) ) as transaction: async with transaction.batch_() as batcher: - for ( - end_user_id, - response_cost, - ) in end_user_list_transactions.items(): + # Sort by end_user_id for consistent lock ordering across pods to prevent deadlocks. + for end_user_id, response_cost in sorted( + end_user_list_transactions.items() + ): if litellm.max_end_user_budget is not None: pass batcher.litellm_endusertable.upsert( diff --git a/litellm/router.py b/litellm/router.py index 7512ee387dc..5e30ae618c6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1673,7 +1673,7 @@ class Router: for cb in self.optional_callbacks ) if not already_registered: - ec_callback = EncryptedContentAffinityCheck() + ec_callback = EncryptedContentAffinityCheck(router=self) self.optional_callbacks.append(ec_callback) litellm.logging_callback_manager.add_litellm_callback(ec_callback) @@ -7076,11 +7076,11 @@ class Router: _shared_model_info = { k: v for k, v in _model_info.items() if k not in _custom_pricing_fields } - litellm.register_model( - model_cost={ - _model_name: _shared_model_info, - } - ) + _backend_alias_cost = {_model_name: _shared_model_info} + if "responses/" in _model_name: + _stripped_model_name = _model_name.replace("responses/", "") + _backend_alias_cost[_stripped_model_name] = _shared_model_info + litellm.register_model(model_cost=_backend_alias_cost) ## Check if LLM Deployment is allowed for this deployment if ( @@ -7752,6 +7752,12 @@ class Router: # initialize client self._add_deployment(deployment=deployment) + _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) + for field in CustomPricingLiteLLMParams.model_fields.keys(): + field_value = deployment.litellm_params.get(field) + if field_value is not None: + _model_info_dict[field] = field_value + # Register custom pricing in litellm.model_cost. # Mirrors _create_deployment() logic to ensure dynamically-added deployments # (e.g., loaded from DB) also have their custom pricing registered. @@ -7759,13 +7765,31 @@ class Router: # zero-cost models, causing budget checks to block free models. _model_id = deployment.model_info.id if _model_id is not None: - _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) - for field in CustomPricingLiteLLMParams.model_fields.keys(): - field_value = deployment.litellm_params.get(field) - if field_value is not None: - _model_info_dict[field] = field_value litellm.register_model(model_cost={_model_id: _model_info_dict}) + ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP + ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes + _model_name = deployment.litellm_params.model + if deployment.litellm_params.custom_llm_provider is not None: + _model_name = ( + deployment.litellm_params.custom_llm_provider + "/" + _model_name + ) + + # For the shared backend key, strip custom pricing fields so that + # one deployment's pricing overrides don't pollute another + # deployment sharing the same backend model name. + # Each deployment's full pricing is already stored under its + # unique model_id above (when present). + _custom_pricing_fields = CustomPricingLiteLLMParams.model_fields.keys() + _shared_model_info = { + k: v for k, v in _model_info_dict.items() if k not in _custom_pricing_fields + } + _backend_alias_cost = {_model_name: _shared_model_info} + if "responses/" in _model_name: + _stripped_model_name = _model_name.replace("responses/", "") + _backend_alias_cost[_stripped_model_name] = _shared_model_info + litellm.register_model(model_cost=_backend_alias_cost) + # add to model names self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 3f1714ba5a5..866073c84dc 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -36,13 +36,16 @@ Safe to enable globally: - No cache required. """ -from typing import Any, List, Optional, cast +from typing import TYPE_CHECKING, Any, List, Optional, cast from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues +if TYPE_CHECKING: + from litellm.router import Router + class EncryptedContentAffinityCheck(CustomLogger): """ @@ -55,8 +58,9 @@ class EncryptedContentAffinityCheck(CustomLogger): Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])``. """ - def __init__(self) -> None: + def __init__(self, router: Optional["Router"] = None) -> None: super().__init__() + self.router = router # ------------------------------------------------------------------ # Helpers @@ -119,6 +123,58 @@ class EncryptedContentAffinityCheck(CustomLogger): return deployment return None + @staticmethod + def _encryption_boundary_key( + litellm_params: Any, + ) -> Optional[tuple]: + """ + ``(api_base, api_key)`` pair identifying an Azure resource. Two + deployments sharing both are interchangeable for ``encrypted_content`` + follow-ups; Azure rejects content produced by any other resource. + + Accepts any object exposing dict-style ``.get(key, default)``: plain + dicts (the common case in ``healthy_deployments``) as well as + ``LiteLLM_Params``-style Pydantic instances, which define a custom + ``.get()``. A stricter ``isinstance(dict)`` guard would silently drop + the latter from boundary matching and fall back to the full pool — + i.e. trigger the exact ``invalid_encrypted_content`` failure this + check exists to prevent. + """ + getter = getattr(litellm_params, "get", None) + if not callable(getter): + return None + api_base = getter("api_base") + api_key = getter("api_key") + if not api_base or not api_key: + return None + return (api_base, api_key) + + def _find_deployments_on_same_encryption_boundary( + self, + healthy_deployments: List[dict], + model_id: str, + ) -> List[dict]: + """ + Deployments in ``healthy_deployments`` sharing the originating + deployment's ``(api_base, api_key)``. Returns ``[]`` if router isn't + wired in, the originating deployment was removed, or no boundary match. + """ + if self.router is None: + return [] + originating = self.router.get_deployment(model_id=model_id) + if originating is None: + return [] + boundary = self._encryption_boundary_key( + originating.litellm_params.model_dump(exclude_none=True) + ) + if boundary is None: + return [] + return [ + d + for d in healthy_deployments + if self._encryption_boundary_key(d.get("litellm_params", {})) == boundary + ] + # ------------------------------------------------------------------ # Request routing (pre-call filter) # ------------------------------------------------------------------ @@ -172,8 +228,25 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["_encrypted_content_affinity_pinned"] = True return [deployment] + # Follow-up switched model_name (LIT-2531): pin by Azure resource instead. + boundary_matches = self._find_deployments_on_same_encryption_boundary( + healthy_deployments=typed_healthy_deployments, + model_id=model_id, + ) + if boundary_matches: + verbose_router_logger.debug( + "EncryptedContentAffinityCheck: model_id=%s not in healthy_deployments; " + "pinning to %d deployment(s) on same encryption boundary", + model_id, + len(boundary_matches), + ) + request_kwargs["_encrypted_content_affinity_pinned"] = True + return boundary_matches + verbose_router_logger.error( - "EncryptedContentAffinityCheck: decoded deployment=%s not found in healthy_deployments", + "EncryptedContentAffinityCheck: decoded deployment=%s not found in " + "healthy_deployments and no boundary match available; falling back to " + "full deployment pool (encrypted_content may be rejected upstream)", model_id, ) return typed_healthy_deployments diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 04347aebe3b..751113400d3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -633,6 +633,16 @@ class BaseLitellmParams( ), ) + skip_tool_message_in_guardrail: Optional[bool] = Field( + default=None, + description=( + "When True, unified guardrails skip tool-role messages when building " + "evaluation inputs (texts and structured_messages). When False, tool " + "messages are included even if litellm_settings sets a global skip. When " + "None, use the global litellm.skip_tool_message_in_guardrail setting." + ), + ) + # Lakera specific params category_thresholds: Optional[LakeraCategoryThresholds] = Field( default=None, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 64827db13f6..5db2a45054a 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -1042,3 +1042,10 @@ class BedrockInvokeAnthropicMessagesRequest(TypedDict, total=False): thinking: dict metadata: dict output_config: dict + + # `context_management` is allowed for Bedrock InvokeModel only when it + # carries `compact_20260112` edits paired with the `compact-2026-01-12` + # anthropic-beta header. The Invoke transformation filters edits to the + # supported subset and strips the field entirely when nothing remains, so + # other edit types (e.g. `clear_thinking_20251015`) never reach Bedrock. + context_management: dict diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index b1d25455d18..d214cdb4f5d 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -1,6 +1,7 @@ -from typing import Any, Dict, List, Optional +from datetime import datetime +from typing import Any, Dict, List, Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, model_validator class BulkUpdateKeyRequestItem(BaseModel): @@ -40,3 +41,78 @@ class BulkUpdateKeyResponse(BaseModel): total_requested: int successful_updates: List[SuccessfulKeyUpdate] failed_updates: List[FailedKeyUpdate] + + +class KeyUpdateFields(BaseModel): + """Allowlist of bulk-broadcastable fields for /team/key/bulk_update; `extra="forbid"` blocks RBAC/ownership/scope mutations even by team admins.""" + + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + + # Budgets + max_budget: Optional[float] = None + budget_id: Optional[str] = None + budget_duration: Optional[str] = None + budget_limits: Optional[List[Any]] = None + model_max_budget: Optional[Dict[str, Any]] = None + + # Rate limits + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + model_tpm_limit: Optional[Dict[str, Any]] = None + model_rpm_limit: Optional[Dict[str, Any]] = None + max_parallel_requests: Optional[int] = None + rpm_limit_type: Optional[ + Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] + ] = None + tpm_limit_type: Optional[ + Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] + ] = None + + # Temporary budget grants (auto-expire). `spend` deliberately omitted — bulk-zeroing it bypasses budget enforcement; admin-only via /key/update. + temp_budget_increase: Optional[float] = None + temp_budget_expiry: Optional[datetime] = None + + # Expiry + duration: Optional[str] = None + + # Operational metadata + tags: Optional[List[str]] = None + metadata: Optional[Dict[str, Any]] = None + + @model_validator(mode="after") + def validate_temp_budget(self) -> "KeyUpdateFields": + if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: + if self.temp_budget_increase is None or self.temp_budget_expiry is None: + raise ValueError( + "temp_budget_increase and temp_budget_expiry must be set together" + ) + return self + + @model_validator(mode="after") + def require_at_least_one_field(self) -> "KeyUpdateFields": + # Reject empty payload — would iterate every key with no-op writes. + if not self.model_fields_set: + raise ValueError("update_fields must specify at least one field to update.") + return self + + +class BulkUpdateTeamKeysRequest(BaseModel): + """Apply one update payload to many keys inside a team; provide either `key_ids` or `all_keys_in_team=True`.""" + + team_id: str + key_ids: Optional[List[str]] = None + all_keys_in_team: bool = False + update_fields: KeyUpdateFields + + @model_validator(mode="after") + def validate_selection(self) -> "BulkUpdateTeamKeysRequest": + has_key_ids = self.key_ids is not None and len(self.key_ids) > 0 + if has_key_ids and self.all_keys_in_team: + raise ValueError( + "Provide either `key_ids` or `all_keys_in_team=True`, not both." + ) + if not has_key_ids and not self.all_keys_in_team: + raise ValueError( + "Must provide either `key_ids` (non-empty) or `all_keys_in_team=True`." + ) + return self diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 00a7748309b..400edcac889 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -825,6 +825,7 @@ API_ROUTE_TO_CALL_TYPES = { # Realtime API "/realtime": [CallTypes.arealtime], "/v1/realtime": [CallTypes.arealtime], + "/openai/v1/realtime": [CallTypes.arealtime], # Provider-specific routes "/anthropic/v1/messages": [CallTypes.anthropic_messages], # Google GenAI routes diff --git a/litellm/utils.py b/litellm/utils.py index 019fbc2add8..da80e4ae164 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1,3 +1,5 @@ +"""Utility helpers for LiteLLM core request handling and provider support.""" + # from __future__ import annotations must be the first non-comment statement from __future__ import annotations @@ -4405,6 +4407,10 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + if bedrock_route == "claude_platform": + optional_params = BedrockModelInfo.map_claude_platform_auth_params( + passed_params=passed_params, optional_params=optional_params + ) elif custom_llm_provider == "cloudflare": optional_params = litellm.CloudflareChatConfig().map_openai_params( model=model, @@ -9490,6 +9496,49 @@ def get_non_default_completion_params(kwargs: dict) -> dict: return non_default_params +def peek_reasoning_summary_aliases(optional_params: dict) -> Optional[Any]: + """Read AI-SDK-style reasoning summary from optional_params or nested extra_body. + + Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. + """ + if "reasoningSummary" in optional_params: + return optional_params["reasoningSummary"] + if "reasoning_summary" in optional_params: + return optional_params["reasoning_summary"] + extra_body = optional_params.get("extra_body") + if isinstance(extra_body, dict): + if "reasoningSummary" in extra_body: + return extra_body["reasoningSummary"] + if "reasoning_summary" in extra_body: + return extra_body["reasoning_summary"] + return None + + +def strip_reasoning_summary_aliases_from_optional_params( + optional_params: dict, +) -> Tuple[dict, Optional[Any]]: + """Copy optional_params; remove reasoningSummary aliases from top-level and extra_body.""" + op = dict(optional_params) + rs_val = op.pop("reasoningSummary", None) + snake_rs_val = op.pop("reasoning_summary", None) + if rs_val is None: + rs_val = snake_rs_val + eb = op.get("extra_body") + if isinstance(eb, dict): + eb = dict(eb) + eb_rs_val = eb.pop("reasoningSummary", None) + eb_snake_rs_val = eb.pop("reasoning_summary", None) + if rs_val is None: + rs_val = eb_rs_val + if rs_val is None: + rs_val = eb_snake_rs_val + if eb: + op["extra_body"] = eb + else: + op.pop("extra_body", None) + return op, rs_val + + def get_non_default_transcription_params(kwargs: dict) -> dict: from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 92e87c00ef6..70b065aa918 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21109,6 +21109,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-2": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -27192,6 +27224,20 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.6-plus": { + "input_cost_per_token": 3.25e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.95e-06, + "source": "https://openrouter.ai/qwen/qwen3.6-plus", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/qwen/qwen3.5-35b-a3b": { "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", diff --git a/pyproject.toml b/pyproject.toml index d194d467913..65557d8a9ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "importlib-metadata>=8.0.0,<9.0", "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", - "jinja2>=3.1.0,<4.0", + "jinja2>=3.1.6,<4.0", "aiohttp>=3.10,<4.0", "pydantic>=2.10.0,<3.0.0", "jsonschema>=4.0.0,<5.0", @@ -275,6 +275,33 @@ filterwarnings = [ "ignore::DeprecationWarning:pytest_asyncio.plugin", ] +[tool.mutmut] +# Mutation-testing scope. Driven by the manually-triggered workflow at +# .github/workflows/mutation-test.yml. mutmut is not part of the project's +# default install; it is pulled in via `uv run --with mutmut==` in CI. +# `also_copy = ["litellm/"]` is required because mutmut runs in a `mutants/` +# sandbox and the test conftest imports from across the litellm package. +paths_to_mutate = [ + "litellm/proxy/management_endpoints/", +] +tests_dir = [ + "tests/test_litellm/proxy/management_endpoints/", +] +also_copy = [ + "litellm/", +] +# Disable rerun/parallel plugins for mutation runs: +# - pytest-retry triggers an `INTERNALERROR: no option named 'filtered_exceptions'` +# when invoked via mutmut's in-process `pytest.main()` call. +# - rerunning a "failed" test on a mutant would mask which mutants are killed +# vs. survive, so reruns are wrong for mutation testing regardless. +# - xdist is unnecessary inside mutmut (mutmut handles its own parallelism). +pytest_add_cli_args = [ + "-p", "no:retry", + "-p", "no:rerunfailures", + "-p", "no:xdist", +] + [tool.coverage.run] source = ["litellm"] relative_files = true diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py new file mode 100644 index 00000000000..a606e3f71cf --- /dev/null +++ b/scripts/mutation_report.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Generate an agent-actionable mutation testing report. + +Reads the mutmut sandbox state at `mutants/` and produces a single +`mutation-report.md` grouped by function. For each function with surviving +mutants, the report embeds the original function source (via AST), the +unified diff for each surviving mutation (via `mutmut show`), and the +existing test file(s) — followed by an ACH-style instruction asking the +reader to write tests that kill the survivors. + +Run after `mutmut run` and `mutmut export-cicd-stats`. Expects mutmut to be +invokable as `uv run --no-sync --with mutmut== mutmut `. +""" +from __future__ import annotations + +import ast +import json +import re +import subprocess +import sys +import tomllib +from collections import defaultdict +from difflib import SequenceMatcher +from pathlib import Path +from textwrap import dedent + +ROOT = Path(__file__).resolve().parent.parent +MUTMUT_INVOCATION = ["uv", "run", "--no-sync", "--with", "mutmut==3.5.0", "mutmut"] + + +def load_mutmut_config() -> dict: + with open(ROOT / "pyproject.toml", "rb") as f: + return tomllib.load(f)["tool"]["mutmut"] + + +def get_survivors() -> list[str]: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False + ) + survivors = [] + for line in proc.stdout.splitlines(): + m = re.match(r"\s*(\S+):\s*survived\s*$", line) + if m: + survivors.append(m.group(1)) + return survivors + + +def get_mutmut_show(mutant_name: str) -> str: + proc = subprocess.run( + [*MUTMUT_INVOCATION, "show", mutant_name], + capture_output=True, + text=True, + check=False, + ) + return proc.stdout.strip() or "(mutmut show produced no output)" + + +def parse_mutant_name(name: str) -> tuple[str, str, str]: + """Parse `.x___mutmut_` -> (module, function, N). + + mutmut prefixes mutated functions with `x_` (single underscore). For a + function named `foo`, mutants are `x_foo__mutmut_N`. For a function named + `_foo` (leading underscore), the mutant becomes `x__foo__mutmut_N` — so + the regex matches a single underscore after `x` and captures everything + (including any leading underscores) up to `__mutmut_`. + """ + m = re.match(r"^(.+)\.x_(.+)__mutmut_(\d+)$", name) + if not m: + return name, name, "?" + return m.group(1), m.group(2), m.group(3) + + +def function_anchor(module_path: str, function_name: str) -> str: + return re.sub(r"[^a-z0-9_-]+", "-", f"{module_path}-{function_name}".lower()).strip( + "-" + ) + + +def module_to_file(module_path: str) -> Path | None: + candidate = ROOT / Path(*module_path.split(".")).with_suffix(".py") + return candidate if candidate.exists() else None + + +def find_function_in_file( + file_path: Path, function_name: str +) -> tuple[int, int, str, list[int]] | None: + """Find a top-level or nested function by name; returns the first match. + + Returns ``(start_line, end_line, source, all_match_lines)`` or ``None``. + ``all_match_lines`` is the start line of every function (any nesting + level) in the file with this name. When ``len(all_match_lines) > 1`` the + file defines the same name in multiple places (e.g., a module-level + helper and a class method) — mutmut's mutant identifier does not carry + class context, so we can't determine which definition was mutated. + Callers surface a disambiguation note in that case. + """ + src = file_path.read_text() + tree = ast.parse(src) + matches = [ + node + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == function_name + ] + if not matches: + return None + first = matches[0] + lines = src.splitlines() + return ( + first.lineno, + first.end_lineno, + "\n".join(lines[first.lineno - 1 : first.end_lineno]), + [m.lineno for m in matches], + ) + + +def collect_test_files(tests_dir: list[str]) -> list[Path]: + found: list[Path] = [] + for entry in tests_dir: + p = ROOT / entry + if p.is_file(): + found.append(p) + elif p.is_dir(): + found.extend(sorted(p.rglob("test_*.py"))) + return found + + +def _indent_of(line: str) -> str: + return line[: len(line) - len(line.lstrip())] + + +def render_meta_style_mutant( + module_path: str, function_name: str, mutant_num: str +) -> str | None: + """Render the mutated function with `# MUTANT START`/`# MUTANT END` delimiters. + + Reads `mutants/.py` (the trampoline file mutmut emits), finds + `x___mutmut_orig` and `x___mutmut_`, and renders the + mutated version with the lines that differ from `__mutmut_orig` wrapped + in `# MUTANT START`/`# MUTANT END` comments — the format from Meta's + ACH paper (arXiv 2501.12862, Table 1). + + The function header is rewritten to use the original function name so + the agent sees the source as it would appear in the file (rather than + mutmut's internal `x_*__mutmut_` name). + + Returns None if the trampoline file or either function cannot be found + (the caller falls back to the unified diff). + """ + trampoline = ROOT / "mutants" / Path(*module_path.split(".")).with_suffix(".py") + if not trampoline.exists(): + return None + + src = trampoline.read_text() + try: + tree = ast.parse(src) + except SyntaxError: + return None + file_lines = src.splitlines() + + orig_def = f"x_{function_name}__mutmut_orig" + mutant_def = f"x_{function_name}__mutmut_{mutant_num}" + + orig_node = mutated_node = None + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name == orig_def: + orig_node = node + elif node.name == mutant_def: + mutated_node = node + + if orig_node is None or mutated_node is None: + return None + + orig_lines = file_lines[orig_node.lineno - 1 : orig_node.end_lineno] + mutated_lines = file_lines[mutated_node.lineno - 1 : mutated_node.end_lineno] + if not orig_lines or not mutated_lines: + return None + + # Rewrite the def line to use the original (non-trampolined) function name + # so the agent sees the function as it appears in the source file. + orig_lines[0] = orig_lines[0].replace(orig_def, function_name, 1) + mutated_lines[0] = mutated_lines[0].replace(mutant_def, function_name, 1) + + matcher = SequenceMatcher(a=orig_lines, b=mutated_lines) + out: list[str] = [] + in_diff = False + + for op, i1, i2, j1, j2 in matcher.get_opcodes(): + if op == "equal": + if in_diff: + # Close the block at the indent of the line just inside it. + indent = _indent_of(out[-1]) if out else "" + out.append(f"{indent}# MUTANT END") + in_diff = False + out.extend(mutated_lines[j1:j2]) + else: + if not in_diff: + # Open the block at the indent of the first differing line. + if j1 < len(mutated_lines): + indent = _indent_of(mutated_lines[j1]) + elif i1 < len(orig_lines): + indent = _indent_of(orig_lines[i1]) + else: + indent = "" + out.append(f"{indent}# MUTANT START") + in_diff = True + if op == "delete": + # Mutation removed lines — surface what was deleted as a + # comment so the agent can see the intent of the change. + for deleted in orig_lines[i1:i2]: + indent = _indent_of(deleted) + out.append(f"{indent}# (deleted by mutation): {deleted.lstrip()}") + else: + # replace / insert: take from mutated_lines + out.extend(mutated_lines[j1:j2]) + + if in_diff: + indent = _indent_of(out[-1]) if out else "" + out.append(f"{indent}# MUTANT END") + + return "\n".join(out) + + +def render(config: dict, survivors: list[str], stats: dict | None) -> str: + by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + for survivor in survivors: + module_path, function_name, mutant_num = parse_mutant_name(survivor) + by_function[(module_path, function_name)].append((survivor, mutant_num)) + + out: list[str] = [] + out.append("# Mutation Test Report") + out.append("") + + out.append("## Summary") + out.append("") + if stats: + total = stats.get("total", 0) or sum( + stats.get(k, 0) + for k in ( + "killed", + "survived", + "no_tests", + "skipped", + "suspicious", + "timeout", + "segfault", + ) + ) + killed = stats.get("killed", 0) + survived = stats.get("survived", 0) + score = (killed / total * 100) if total else 0.0 + out.append(f"- Total mutants: **{total}**") + out.append(f"- Killed: **{killed}**") + out.append(f"- Survived: **{survived}**") + out.append(f"- Mutation score: **{score:.1f}%**") + for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"): + v = stats.get(k, 0) + if v: + out.append(f"- {k.replace('_', ' ').title()}: {v}") + else: + out.append(f"- Survivors found: **{len(survivors)}**") + out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)") + out.append("") + + if not survivors: + out.append("**No surviving mutants — the test suite caught every mutation.**") + out.append("") + return "\n".join(out) + + out.append("## Surviving mutants by function") + out.append("") + for (module_path, function_name), items in by_function.items(): + anchor = function_anchor(module_path, function_name) + out.append( + f"- [`{function_name}`](#{anchor}) — {len(items)} mutant" + f"{'s' if len(items) != 1 else ''} ({module_path})" + ) + out.append("") + + for (module_path, function_name), items in by_function.items(): + anchor = function_anchor(module_path, function_name) + out.append(f'') + out.append(f"## `{module_path}.{function_name}`") + out.append("") + out.append(f"**Module:** `{module_path}`") + + file_path = module_to_file(module_path) + if file_path is None: + out.append("") + out.append(f"_(could not locate source file for module `{module_path}`)_") + out.append("") + else: + rel = file_path.relative_to(ROOT) + out.append(f"**File:** `{rel}`") + out.append("") + found = find_function_in_file(file_path, function_name) + if found: + start, end, fn_src, all_lines = found + out.append(f"### Original function (lines {start}-{end})") + out.append("") + if len(all_lines) > 1: + line_list = ", ".join(str(line) for line in all_lines) + out.append( + f"> **Note:** {len(all_lines)} functions named " + f"`{function_name}` are defined in this file at lines " + f"{line_list}. Showing the first match. mutmut's " + f"mutant identifier does not carry class context, so " + f"the body below may not correspond to the function " + f"that was actually mutated — verify manually before " + f"writing the killing test." + ) + out.append("") + out.append("```python") + out.append(fn_src) + out.append("```") + out.append("") + else: + out.append(f"_(could not locate `{function_name}` in {rel} via AST)_") + out.append("") + + out.append(f"### Surviving mutations ({len(items)})") + out.append("") + for i, (mutant_name, mutant_num) in enumerate(items, 1): + out.append(f"#### Mutation {i} of {len(items)} — `{mutant_name}`") + out.append("") + meta_style = render_meta_style_mutant( + module_path, function_name, mutant_num + ) + if meta_style is not None: + out.append( + "Mutated function (the bug is delimited by " + "`# MUTANT START` / `# MUTANT END`):" + ) + out.append("") + out.append("```python") + out.append(meta_style) + out.append("```") + out.append("") + out.append("
Unified diff (`mutmut show`)") + out.append("") + out.append("```diff") + out.append(get_mutmut_show(mutant_name)) + out.append("```") + out.append("") + out.append("
") + out.append("") + else: + # Fallback: trampoline file or function lookup failed. + out.append("```diff") + out.append(get_mutmut_show(mutant_name)) + out.append("```") + out.append("") + + test_files = collect_test_files(config.get("tests_dir", [])) + if test_files: + out.append("## Existing tests") + out.append("") + out.append( + "These are the test files that mutmut considered when classifying the " + "mutants above. New tests should be added here, matching existing " + "conventions, fixtures, and naming." + ) + out.append("") + for tf in test_files: + rel = tf.relative_to(ROOT) + out.append(f"### `{rel}`") + out.append("") + out.append("```python") + out.append(tf.read_text()) + out.append("```") + out.append("") + + out.append("## Task") + out.append("") + out.append( + dedent( + """\ + For each surviving mutant listed above, write a new test in the + existing test file (matching its conventions, fixtures, and naming + style) that: + + - **Fails** when the mutated version of the function is in place. + - **Passes** when the original (correct) version is in place. + + Aim for one test per surviving mutant. If multiple mutants in the + same function can be killed by a single test, that is fine — note + which mutant numbers in the test name or docstring. + + Do not modify the source file. Only add tests. + """ + ).strip() + ) + out.append("") + + return "\n".join(out) + + +def main() -> int: + config = load_mutmut_config() + + stats_file = ROOT / "mutants" / "mutmut-cicd-stats.json" + stats: dict | None = None + if stats_file.exists(): + try: + stats = json.loads(stats_file.read_text()) + except json.JSONDecodeError as exc: + print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) + + survivors = get_survivors() + report = render(config, survivors, stats) + + out_path = ROOT / "mutation-report.md" + out_path.write_text(report) + print( + f"Wrote {out_path} ({len(survivors)} survivor" + f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index da08f9673d4..5148ea4db91 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -157,16 +157,16 @@ async def test_bedrock_retrieve_batch(): """ print("Testing bedrock batch retrieval") - # Mock bedrock batch response mock_bedrock_response = { "jobArn": "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", "jobName": "test-job-123", "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "roleArn": "arn:aws:iam::123456789012:role/service-role/AmazonBedrockExecutionRoleForAgents_TEST", - "status": "InProgress", - "message": "Job is in progress", + "status": "Completed", + "message": "", "submitTime": "2024-01-01T12:00:00Z", "lastModifiedTime": "2024-01-01T12:30:00Z", + "endTime": "2024-01-01T13:00:00Z", "inputDataConfig": { "s3InputDataConfig": {"s3Uri": "s3://test-bucket/input/test-input.jsonl"} }, @@ -175,43 +175,38 @@ async def test_bedrock_retrieve_batch(): }, } - # Mock the HTTP response - mock_response = MagicMock() - mock_response.json.return_value = mock_bedrock_response - mock_response.status_code = 200 + mock_bedrock_client = MagicMock() + mock_bedrock_client.get_model_invocation_job.return_value = mock_bedrock_response + mock_creds = MagicMock(access_key="ak", secret_key="sk", token="tok") - # Print the mock response to debug - print("MOCK RESPONSE DATA:", mock_bedrock_response) - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" - ) as mock_get: - mock_response.raise_for_status.return_value = None - mock_get.return_value = mock_response - - # Test retrieve batch + with ( + patch("boto3.client", return_value=mock_bedrock_client), + patch( + "litellm.llms.bedrock.batches.transformation.BedrockBatchesConfig.get_credentials", + return_value=mock_creds, + ), + ): batch_response = await litellm.aretrieve_batch( batch_id="arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123", custom_llm_provider="bedrock", model="us.anthropic.claude-haiku-4-5-20251001-v1:0", ) - print("MOCKED BATCH RESPONSE=", batch_response) - - # Validate the response assert ( batch_response.id == "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/test-job-123" ) assert batch_response.object == "batch" - assert ( - batch_response.status == "in_progress" - ) # Bedrock "InProgress" maps to "in_progress" + assert batch_response.status == "completed" assert batch_response.endpoint == "/v1/chat/completions" - # Validate input and output file IDs in the final transformed response assert batch_response.input_file_id == "s3://test-bucket/input/test-input.jsonl" - assert batch_response.output_file_id == "s3://test-bucket/output/" + # Bedrock returns only the output *prefix*; the handler predicts the + # actual output object as //.out. + assert ( + batch_response.output_file_id + == "s3://test-bucket/output/test-job-123/test-input.jsonl.out" + ) def test_bedrock_batch_with_encryption_key_in_post_request(): diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py index 69c3b4cb59a..32685c5cbd3 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py @@ -19,7 +19,7 @@ def client(): def mock_user_api_key_auth(): """Mock the user_api_key_auth dependency""" with patch( - "enterprise.litellm_enterprise.proxy.management_endpoints.internal_user_endpoints.user_api_key_auth" + "litellm_enterprise.proxy.management_endpoints.internal_user_endpoints.user_api_key_auth" ) as mock_auth: mock_auth.return_value = {"user_id": "test_user", "api_key": "test_key"} yield mock_auth @@ -31,12 +31,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test when max_users is set and user count is within limit""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - {"max_users": 10}, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + {"max_users": 10}, + ), ): # Mock database count mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=5) @@ -66,12 +70,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test when max_users is not set (premium_user_data is None or doesn't contain max_users)""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - None, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + None, + ), ): # Mock database count mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=3) @@ -99,12 +107,16 @@ class TestAvailableEnterpriseUsers: self, client, mock_user_api_key_auth ): """Test the current bug where total_users_remaining can be negative""" - with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch( - "litellm.proxy.proxy_server.premium_user", - True, - ), patch( - "litellm.proxy.proxy_server.premium_user_data", - {"key": "value"}, + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.proxy_server.premium_user_data", + {"key": "value"}, + ), ): # Mock database count higher than max_users to trigger the bug mock_prisma.db.litellm_usertable.count = AsyncMock(return_value=8) @@ -140,12 +152,15 @@ class TestAvailableEnterpriseUsers: """Test when prisma_client is None (no database connection)""" from litellm.proxy._types import CommonProxyErrors - with patch( - "litellm.proxy.proxy_server.prisma_client", - None, - ), patch( - "litellm.proxy.proxy_server.premium_user", - True, + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + None, + ), + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), ): # Override the dependency client.app.dependency_overrides[mock_user_api_key_auth] = lambda: { diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index a64c6c7aa36..6240bedd3e6 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -233,6 +233,12 @@ async def test_reset_budget_endusers_partial_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -400,6 +406,12 @@ async def test_reset_budget_continues_other_categories_on_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -884,6 +896,12 @@ async def test_service_logger_endusers_success(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -966,6 +984,12 @@ async def test_service_logger_endusers_failure(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + # Mock db.litellm_organizationtable.update_many (used by reset_budget_for_orgs_linked_to_budgets) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + # Mock db.litellm_tagtable.update_many (used by reset_budget_for_tags_linked_to_budgets) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() @@ -1060,6 +1084,10 @@ async def test_reset_budget_for_litellm_team_members_called(): prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( return_value={"count": 0} ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 0} + ) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 0}) proxy_logging_obj = MagicMock() proxy_logging_obj.service_logging_obj = MagicMock() diff --git a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py index 884563c6e5c..170440f6b9f 100644 --- a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py +++ b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py @@ -104,7 +104,8 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): Send a text message containing the blocked phrase. Guardrail must: - Send error event (guardrail_violation) to client. - - Send response.audio_transcript.delta with the block message to client. + - Send response.output_audio_transcript.delta (or beta-protocol + response.audio_transcript.delta) with the block message to client. - NOT forward response.create to OpenAI (no AI response). """ import websockets @@ -119,7 +120,6 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) @@ -182,7 +182,11 @@ async def test_text_message_blocked_by_guardrail_no_ai_response(): transcript_deltas = [ e for e in client_events - if e.get("type") == "response.audio_transcript.delta" + if e.get("type") + in ( + "response.output_audio_transcript.delta", + "response.audio_transcript.delta", + ) ] assert ( len(transcript_deltas) >= 1 @@ -298,7 +302,6 @@ async def test_clean_text_message_passes_through_to_openai(): OPENAI_REALTIME_URL, additional_headers={ "Authorization": f"Bearer {OPENAI_API_KEY}", - "OpenAI-Beta": "realtime=v1", }, ) as backend_ws: streaming, input_queue = await _build_streaming(client_events, backend_ws) diff --git a/tests/llm_translation/test_openrouter.py b/tests/llm_translation/test_openrouter.py index 631b0770e3d..8fbb8803d11 100644 --- a/tests/llm_translation/test_openrouter.py +++ b/tests/llm_translation/test_openrouter.py @@ -11,7 +11,7 @@ import litellm def test_completion_openrouter_reasoning_content(): litellm._turn_on_debug() resp = litellm.completion( - model="openrouter/anthropic/claude-3.7-sonnet", + model="openrouter/anthropic/claude-sonnet-4", messages=[{"role": "user", "content": "Hello world"}], reasoning={"effort": "high"}, ) diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 6fd253ee294..3c7e004b62e 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -268,51 +268,63 @@ def test_aaparallel_function_call_with_anthropic_thinking(model): from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message +_PARALLEL_TOOL_HISTORY_MESSAGES = [ + { + "role": "user", + "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", + }, + Message( + content="Here are the current weather conditions for San Francisco, Tokyo, and Paris:", + role="assistant", + tool_calls=[ + ChatCompletionMessageToolCall( + index=1, + function=Function( + arguments='{"location": "San Francisco, CA", "unit": "fahrenheit"}', + name="get_current_weather", + ), + id="tooluse_Jj98qn6xQlOP_PiQr-w9iA", + type="function", + ) + ], + function_call=None, + ), + { + "tool_call_id": "tooluse_Jj98qn6xQlOP_PiQr-w9iA", + "role": "tool", + "name": "get_current_weather", + "content": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}', + }, +] + + @pytest.mark.parametrize( - "model, provider", + "model, messages, expect_unsupported_params_error", [ + # Bedrock Converse still requires modify_params to inject the dummy tool. ( "anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock", + _PARALLEL_TOOL_HISTORY_MESSAGES, + True, ), - ("claude-haiku-4-5-20251001", "anthropic"), - ], -) -@pytest.mark.parametrize( - "messages, expected_error_msg", - [ + # Anthropic Messages API: dummy tool is injected without modify_params. ( + "claude-haiku-4-5-20251001", + _PARALLEL_TOOL_HISTORY_MESSAGES, + False, + ), + ( + "anthropic.claude-3-sonnet-20240229-v1:0", [ { "role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", - }, - Message( - content="Here are the current weather conditions for San Francisco, Tokyo, and Paris:", - role="assistant", - tool_calls=[ - ChatCompletionMessageToolCall( - index=1, - function=Function( - arguments='{"location": "San Francisco, CA", "unit": "fahrenheit"}', - name="get_current_weather", - ), - id="tooluse_Jj98qn6xQlOP_PiQr-w9iA", - type="function", - ) - ], - function_call=None, - ), - { - "tool_call_id": "tooluse_Jj98qn6xQlOP_PiQr-w9iA", - "role": "tool", - "name": "get_current_weather", - "content": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}', - }, + } ], - True, + False, ), ( + "claude-haiku-4-5-20251001", [ { "role": "user", @@ -324,25 +336,26 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message ], ) def test_parallel_function_call_anthropic_error_msg( - model, provider, messages, expected_error_msg + model, messages, expect_unsupported_params_error ): """ - Anthropic doesn't support tool calling without `tools=` param specified. + Tool history without an explicit ``tools`` param: - Ensure this error is thrown when `tools=` param is not specified. But tool call requests are made. + - Bedrock **Converse** still raises ``UnsupportedParamsError`` unless + ``litellm.modify_params`` is enabled (dummy tool is only added there). + - **Anthropic** (and Bedrock Invoke via ``AnthropicConfig.transform_request``) + always get a dummy tool so CLIs work with ``modify_params`` left off. Reference Issue: https://github.com/BerriAI/litellm/issues/5747, https://github.com/BerriAI/litellm/issues/5388 """ - # Ensure modify_params is False so UnsupportedParamsError is raised + # Ensure modify_params is False so Bedrock Converse path still raises. # (other tests in this file set it to True and don't reset it) original_modify_params = litellm.modify_params litellm.modify_params = False try: litellm.set_verbose = True - messages = messages - - if expected_error_msg: + if expect_unsupported_params_error: with pytest.raises(litellm.UnsupportedParamsError) as e: second_response = litellm.completion( model=model, diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 6af07585796..409f4fad99a 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1494,6 +1494,7 @@ async def test_add_update_server_with_alias(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) @@ -1551,6 +1552,7 @@ async def test_add_update_server_without_alias(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) @@ -1609,6 +1611,7 @@ async def test_add_update_server_fallback_to_server_id(): mock_mcp_server.created_at = None mock_mcp_server.updated_at = None mock_mcp_server.instructions = None + mock_mcp_server.approval_status = "active" # Add server to manager await test_manager.add_server(mock_mcp_server) diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index 62fc8732ebd..f61befac4fb 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -25,8 +25,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "400" - ), f"Expected error code 400, got: {error_dict['code']}" + error_dict["code"] == "429" + ), f"Expected error code 429, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 7ea75a9d61d..87d85a19603 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -99,7 +99,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): # Assert error structure and values assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" assert "key not allowed to access model" in _error_body["message"] @@ -297,7 +297,7 @@ def _validate_model_access_exception( # Assert error structure and values assert _error_body["type"] == expected_type assert _error_body["param"] == "model" - assert _error_body["code"] == "401" + assert _error_body["code"] == "403" if expected_type == "key_model_access_denied": assert "key not allowed to access model" in _error_body["message"] elif expected_type == "team_model_access_denied": diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 70232f25c37..68aff36038b 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -587,12 +587,21 @@ def test_foward_litellm_user_info_to_backend_llm_call(): user_api_key_dict=user_api_key_dict, ) + # All header values must be str/bytes so httpx won't reject them when the + # downstream client builds the request (regression: #27458). + for k, v in data.items(): + assert isinstance(v, (str, bytes)), ( + f"header {k!r} has non-str value {v!r} ({type(v).__name__}); " + "httpx will raise 'Header value must be str or bytes' when the LLM " + "request is built." + ) + expected_data = { "x-litellm-user_api_key_user_id": "test_user_id", "x-litellm-user_api_key_org_id": "test_org_id", "x-litellm-user_api_key_hash": "test_api_key", - "x-litellm-user_api_key_spend": 0.0, - "x-litellm-user_api_key_auth_metadata": {}, + "x-litellm-user_api_key_spend": "0.0", + "x-litellm-user_api_key_auth_metadata": "{}", } assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True) diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index b4aac113f57..0daa5b17ffa 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -413,3 +413,71 @@ async def test_async_log_success_event_uses_end_user_model_budget_duration( f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{budget_duration}" ) assert call_kwargs["response_cost"] == 0.05 + + +@pytest.mark.asyncio +async def test_async_log_success_event_pushes_redis_increments_when_redis_configured(): + """ + Virtual-key model max budget limiter does not run RouterBudgetLimiting.__init__, + so the periodic Redis flush task never starts. After logging spend we must call + _push_in_memory_increments_to_redis when Redis is wired so other workers see spend. + """ + dual_cache = DualCache() + dual_cache.redis_cache = object() # truthy placeholder; push only checks is not None + limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=dual_cache) + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_log_success_event_skips_redis_push_without_redis(budget_limiter): + """When dual_cache has no Redis backend, do not await _push_in_memory_increments_to_redis.""" + assert budget_limiter.dual_cache.redis_cache is None + model = "gpt-4" + kwargs = { + "standard_logging_object": { + "response_cost": 0.01, + "model": model, + "metadata": {"user_api_key_hash": "vk-hash"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + model: {"budget_limit": 10.0, "time_period": "1d"}, + }, + }, + }, + } + with patch.object(budget_limiter, "_increment_spend_for_key", new_callable=AsyncMock): + with patch.object( + budget_limiter, + "_push_in_memory_increments_to_redis", + new_callable=AsyncMock, + ) as mock_push: + await budget_limiter.async_log_success_event( + kwargs, response_obj=None, start_time=None, end_time=None + ) + mock_push.assert_not_awaited() diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index e40543e01a0..697a9ebc720 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2098,6 +2098,56 @@ def test_map_optional_params_preserves_reasoning_summary(): assert responses_api_request["reasoning"]["summary"] == "detailed" +def test_map_optional_params_tool_choice_chat_nested_to_responses_api(): + """Chat tool_choice must become Responses ToolChoiceFunction (top-level name).""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + handler = LiteLLMResponsesTransformationHandler() + responses_api_request = ResponsesAPIOptionalRequestParams() + handler._map_optional_params_to_responses_api_request( + { + "stream": False, + "tool_choice": { + "type": "function", + "function": {"name": "Echo"}, + }, + }, + responses_api_request, + ) + assert responses_api_request["tool_choice"] == { + "type": "function", + "name": "Echo", + } + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + ("auto", "auto"), + ("none", "none"), + ( + {"type": "function", "name": "Echo"}, + {"type": "function", "name": "Echo"}, + ), + ( + {"type": "function", "name": "foo", "function": {"name": "bar"}}, + {"type": "function", "name": "foo"}, + ), + ({"type": "required"}, {"type": "required"}), + ], +) +def test_normalize_tool_choice_for_responses_api(tool_choice, expected): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + assert handler._normalize_tool_choice_for_responses_api(tool_choice) == expected + + def test_convert_chat_completion_file_type_to_input_file(): """ Test that Chat Completion content with type 'file' is correctly mapped diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py index af5e2341406..5cabfe5fb7f 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py @@ -7,7 +7,7 @@ from unittest.mock import patch import pytest from fastapi.testclient import TestClient -from enterprise.litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( +from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( BaseEmailLogger, ) @@ -30,6 +30,7 @@ def no_invitation_wait(monkeypatch): monkeypatch.setattr(BaseEmailLogger, "_wait_for_invitation_creation", _noop) + @pytest.fixture def base_email_logger(): return BaseEmailLogger() @@ -283,7 +284,10 @@ async def test_send_key_created_email_without_key( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert "sk-secret-key-456" not in call_args["html_body"] - assert "[Key hidden for security - retrieve from dashboard]" in call_args["html_body"] + assert ( + "[Key hidden for security - retrieve from dashboard]" + in call_args["html_body"] + ) @pytest.mark.asyncio @@ -317,7 +321,10 @@ async def test_send_key_rotated_email_without_key( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert "sk-secret-rotated-789" not in call_args["html_body"] - assert "[Key hidden for security - retrieve from dashboard]" in call_args["html_body"] + assert ( + "[Key hidden for security - retrieve from dashboard]" + in call_args["html_body"] + ) @pytest.mark.asyncio @@ -371,52 +378,52 @@ async def test_get_invitation_link_creates_new_when_none_exist(base_email_logger """Test that _get_invitation_link creates a new invitation when none exist""" # Mock prisma client with no existing invitation rows mock_prisma = mock.MagicMock() - + # Mock find_many to return empty list (no existing invitations) async def mock_find_many_empty(*args, **kwargs): return [] - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_empty - + # Mock the create_invitation_for_user function mock_created_invitation = mock.MagicMock() mock_created_invitation.id = "new-invitation-id" - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user", - return_value=mock_created_invitation + return_value=mock_created_invitation, ) as mock_create_invitation: # Execute result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was called mock_create_invitation.assert_called_once() call_args = mock_create_invitation.call_args[1] assert call_args["data"].user_id == "test-user" assert call_args["user_api_key_dict"].user_id == "test-user" - + # Verify the returned link uses the new invitation ID assert result == "http://test.com/ui?invitation_id=new-invitation-id" -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_get_invitation_link_uses_existing_when_available(base_email_logger): """Test that _get_invitation_link uses existing invitation when available""" # Mock prisma client with existing invitation row mock_invitation_row = mock.MagicMock() mock_invitation_row.id = "existing-invitation-id" - + mock_prisma = mock.MagicMock() - + # Mock find_many to return existing invitation async def mock_find_many_existing(*args, **kwargs): return [mock_invitation_row] - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_existing - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user" @@ -425,10 +432,10 @@ async def test_get_invitation_link_uses_existing_when_available(base_email_logge result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was NOT called mock_create_invitation.assert_not_called() - + # Verify the returned link uses the existing invitation ID assert result == "http://test.com/ui?invitation_id=existing-invitation-id" @@ -438,33 +445,33 @@ async def test_get_invitation_link_creates_new_when_list_is_none(base_email_logg """Test that _get_invitation_link creates a new invitation when invitation_rows is None""" # Mock prisma client to return None mock_prisma = mock.MagicMock() - + # Mock find_many to return None async def mock_find_many_none(*args, **kwargs): return None - + mock_prisma.db.litellm_invitationlink.find_many = mock_find_many_none - + # Mock the create_invitation_for_user function mock_created_invitation = mock.MagicMock() mock_created_invitation.id = "new-invitation-from-none" - + with mock.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with mock.patch( "litellm.proxy.management_helpers.user_invitation.create_invitation_for_user", - return_value=mock_created_invitation + return_value=mock_created_invitation, ) as mock_create_invitation: # Execute result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - + # Verify that create_invitation_for_user was called mock_create_invitation.assert_called_once() call_args = mock_create_invitation.call_args[1] assert call_args["data"].user_id == "test-user" assert call_args["user_api_key_dict"].user_id == "test-user" - + # Verify the returned link uses the new invitation ID assert result == "http://test.com/ui?invitation_id=new-invitation-from-none" @@ -495,13 +502,15 @@ async def test_get_email_params_user_invitation( user_email="test@example.com", ) - assert result.logo_url == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + assert ( + result.logo_url + == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + ) assert result.support_contact == "support@berri.ai" assert result.base_url == "http://test.com/ui?invitation_id=test-id" assert result.recipient_email == "test@example.com" - @pytest.fixture def mock_env_vars(monkeypatch): """Set up test environment variables""" @@ -513,69 +522,74 @@ def mock_env_vars(monkeypatch): monkeypatch.setenv("PROXY_BASE_URL", "http://test.com") monkeypatch.setenv("PROXY_API_URL", "https://test.com") + @pytest.mark.asyncio async def test_get_email_params_custom_templates_premium_user(mock_env_vars): """Test that _get_email_params returns correct values with custom templates for premium users""" # Mock premium_user as True with patch("litellm.proxy.proxy_server.premium_user", True): email_logger = BaseEmailLogger() - + # Test invitation email params invitation_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_id="testid", user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + assert invitation_params.subject == "Welcome to Test Company!" assert invitation_params.signature == "Best regards,\nTest Company Team" assert invitation_params.logo_url == "https://test-company.com/logo.png" assert invitation_params.support_contact == "support@test-company.com" assert invitation_params.base_url == "http://test.com" - + # Test key created email params key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_id="testid", user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "Your Test Company API Key" assert key_params.signature == "Best regards,\nTest Company Team" + @pytest.mark.asyncio async def test_get_email_params_non_premium_user(mock_env_vars): """Test that non-premium users get default templates even when custom ones are provided""" # Mock premium_user as False with patch("litellm.proxy.proxy_server.premium_user", False): email_logger = BaseEmailLogger() - + # Test invitation email params email_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + # Should use default values even though custom values are set in env assert email_params.subject == "LiteLLM: New User Invitation" assert email_params.signature == EMAIL_FOOTER - assert email_params.logo_url == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + assert ( + email_params.logo_url + == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" + ) assert email_params.support_contact == "support@berri.ai" - # Test key created email params key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "LiteLLM: API Key Created" assert key_params.signature == EMAIL_FOOTER + @pytest.mark.asyncio async def test_get_email_params_default_templates(monkeypatch): """Test that _get_email_params uses default templates when custom ones aren't provided""" @@ -583,28 +597,28 @@ async def test_get_email_params_default_templates(monkeypatch): monkeypatch.delenv("EMAIL_SUBJECT_INVITATION", raising=False) monkeypatch.delenv("EMAIL_SUBJECT_KEY_CREATED", raising=False) monkeypatch.delenv("EMAIL_SIGNATURE", raising=False) - + # Mock premium_user as True (shouldn't matter since no custom values are set) with patch("litellm.proxy.proxy_server.premium_user", True): email_logger = BaseEmailLogger() - + # Test invitation email params with default template invitation_params = await email_logger._get_email_params( email_event=EmailEvent.new_user_invitation, user_email="test@example.com", - event_message="New User Invitation" + event_message="New User Invitation", ) - + assert invitation_params.subject == "LiteLLM: New User Invitation" assert invitation_params.signature == EMAIL_FOOTER - + # Test key created email params with default template key_params = await email_logger._get_email_params( email_event=EmailEvent.virtual_key_created, user_email="test@example.com", - event_message="API Key Created" + event_message="API Key Created", ) - + assert key_params.subject == "LiteLLM: API Key Created" assert key_params.signature == EMAIL_FOOTER @@ -639,7 +653,10 @@ async def test_send_soft_budget_alert_email( call_args = mock_send_email.call_args[1] assert call_args["from_email"] == BaseEmailLogger.DEFAULT_LITELLM_EMAIL assert call_args["to_email"] == ["test@example.com"] - assert call_args["subject"] == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + assert ( + call_args["subject"] + == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + ) assert "$100.0" in call_args["html_body"] # soft_budget assert "$105.0" in call_args["html_body"] # spend assert "$200.0" in call_args["html_body"] # max_budget @@ -673,13 +690,13 @@ async def test_send_soft_budget_alert_email_no_max_budget( call_args = mock_send_email.call_args[1] assert "$100.0" in call_args["html_body"] # soft_budget assert "$105.0" in call_args["html_body"] # spend - assert "Maximum Budget" not in call_args["html_body"] # max_budget should not be shown + assert ( + "Maximum Budget" not in call_args["html_body"] + ) # max_budget should not be shown @pytest.mark.asyncio -async def test_budget_alerts_soft_budget_crossed( - base_email_logger, mock_send_email -): +async def test_budget_alerts_soft_budget_crossed(base_email_logger, mock_send_email): """Test that budget_alerts sends email when soft budget is crossed""" user_info = CallInfo( user_id="test_user", @@ -708,11 +725,14 @@ async def test_budget_alerts_soft_budget_crossed( mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] - + # Verify cache was set to prevent duplicate alerts mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:soft_budget_crossed:test_user" + assert ( + cache_call_args["key"] + == "email_budget_alerts:soft_budget_crossed:test_user" + ) assert cache_call_args["value"] == "SENT" assert cache_call_args["ttl"] == EMAIL_BUDGET_ALERT_TTL @@ -766,9 +786,7 @@ async def test_budget_alerts_soft_budget_duplicate_prevention( @pytest.mark.asyncio -async def test_budget_alerts_no_budgets( - base_email_logger, mock_send_email -): +async def test_budget_alerts_no_budgets(base_email_logger, mock_send_email): """Test that budget_alerts returns early when no budgets are set""" user_info = CallInfo( user_id="test_user", @@ -817,7 +835,10 @@ async def test_budget_alerts_uses_token_for_cache_key( # Verify cache key uses token instead of user_id mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:soft_budget_crossed:hashed_token_123" + assert ( + cache_call_args["key"] + == "email_budget_alerts:soft_budget_crossed:hashed_token_123" + ) @pytest.mark.asyncio @@ -838,7 +859,9 @@ async def test_get_email_params_soft_budget_crossed( ) # Should use default subject template for soft_budget_crossed - assert result.subject == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + assert ( + result.subject == "LiteLLM: Soft Budget Crossed - Total Soft Budget: $100.0" + ) assert result.recipient_email == "test@example.com" assert result.base_url == "http://test.com" @@ -867,16 +890,20 @@ async def test_budget_alerts_max_budget_alert_crossed( "PROXY_BASE_URL": "http://test.com", }, ): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] assert "Max Budget Alert" in call_args["subject"] - + mock_cache.async_set_cache.assert_called_once() cache_call_args = mock_cache.async_set_cache.call_args[1] - assert cache_call_args["key"] == "email_budget_alerts:max_budget_alert:test_user" + assert ( + cache_call_args["key"] == "email_budget_alerts:max_budget_alert:test_user" + ) assert cache_call_args["value"] == "SENT" assert cache_call_args["ttl"] == EMAIL_BUDGET_ALERT_TTL @@ -906,15 +933,15 @@ async def test_multi_threshold_sends_crossed_thresholds( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # spend=80 crosses 50% ($50) and 75% ($75), but not 100% ($100) assert mock_send_email.call_count == 2 # Check cache keys include threshold percentage - cache_keys = [ - c[1]["key"] for c in mock_cache.async_set_cache.call_args_list - ] + cache_keys = [c[1]["key"] for c in mock_cache.async_set_cache.call_args_list] assert "email_budget_alerts:max_budget_alert:50:hashed_key_1" in cache_keys assert "email_budget_alerts:max_budget_alert:75:hashed_key_1" in cache_keys @@ -949,7 +976,9 @@ async def test_multi_threshold_dedup_cache_prevents_resend( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # Only 75% should fire assert mock_send_email.call_count == 1 @@ -980,7 +1009,9 @@ async def test_multi_threshold_owner_email_auto_included( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() to_emails = mock_send_email.call_args[1]["to_email"] @@ -1002,7 +1033,7 @@ async def test_multi_threshold_malformed_keys_skipped( event_group=Litellm_EntityType.KEY, max_budget_alert_emails={ "fifty": ["finance@co.com"], # invalid - "50": ["finance@co.com"], # valid, crossed + "50": ["finance@co.com"], # valid, crossed }, ) @@ -1012,7 +1043,9 @@ async def test_multi_threshold_malformed_keys_skipped( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) # Only the valid "50" threshold should fire assert mock_send_email.call_count == 1 @@ -1041,7 +1074,9 @@ async def test_multi_threshold_empty_emails_only_owner( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() to_emails = mock_send_email.call_args[1]["to_email"] @@ -1067,11 +1102,13 @@ async def test_no_map_preserves_old_single_threshold( base_email_logger.internal_usage_cache = mock_cache with mock.patch.dict(os.environ, {"PROXY_BASE_URL": "http://test.com"}): - await base_email_logger.budget_alerts(type="max_budget_alert", user_info=user_info) + await base_email_logger.budget_alerts( + type="max_budget_alert", user_info=user_info + ) mock_send_email.assert_called_once() call_args = mock_send_email.call_args[1] assert call_args["to_email"] == ["test@example.com"] # Old path cache key has no threshold percentage cache_key = mock_cache.async_set_cache.call_args[1]["key"] - assert cache_key == "email_budget_alerts:max_budget_alert:test_user" \ No newline at end of file + assert cache_key == "email_budget_alerts:max_budget_alert:test_user" diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py index b160ca5130c..d67dc3cf6bc 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_callback_controls.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch import pytest -from enterprise.litellm_enterprise.enterprise_callbacks.callback_controls import ( +from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, ) from litellm.constants import X_LITELLM_DISABLE_CALLBACKS @@ -18,168 +18,282 @@ from litellm.types.utils import StandardCallbackDynamicParams class TestEnterpriseCallbackControls: - + @pytest.fixture def mock_premium_user(self): """Fixture to mock premium user check as True""" - with patch.object(EnterpriseCallbackControls, '_should_allow_dynamic_callback_disabling', return_value=True): + with patch.object( + EnterpriseCallbackControls, + "_should_allow_dynamic_callback_disabling", + return_value=True, + ): yield - - @pytest.fixture + + @pytest.fixture def mock_non_premium_user(self): """Fixture to mock premium user check as False""" - with patch.object(EnterpriseCallbackControls, '_should_allow_dynamic_callback_disabling', return_value=False): + with patch.object( + EnterpriseCallbackControls, + "_should_allow_dynamic_callback_disabling", + return_value=False, + ): yield @pytest.fixture def mock_request_headers(self): """Fixture to mock get_proxy_server_request_headers""" - with patch('enterprise.litellm_enterprise.enterprise_callbacks.callback_controls.get_proxy_server_request_headers') as mock_headers: + with patch( + "litellm_enterprise.enterprise_callbacks.callback_controls.get_proxy_server_request_headers" + ) as mock_headers: yield mock_headers - def test_callback_disabled_langfuse_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_langfuse_string( + self, mock_premium_user, mock_request_headers + ): """Test that 'langfuse' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_langfuse_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_langfuse_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that LangfusePromptManagement CustomLogger instance is disabled when 'langfuse' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + langfuse_logger = LangfusePromptManagement() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(langfuse_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + langfuse_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_s3_v2_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_s3_v2_string( + self, mock_premium_user, mock_request_headers + ): """Test that 's3_v2' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_s3_v2_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_s3_v2_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that S3Logger CustomLogger instance is disabled when 's3_v2' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "s3_v2"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock S3Logger to avoid async initialization issues - with patch('litellm.integrations.s3_v2.S3Logger.__init__', return_value=None): + with patch("litellm.integrations.s3_v2.S3Logger.__init__", return_value=None): s3_logger = S3Logger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(s3_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + s3_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_datadog_string(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_datadog_string( + self, mock_premium_user, mock_request_headers + ): """Test that 'datadog' string callback is disabled when specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_datadog_customlogger(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_datadog_customlogger( + self, mock_premium_user, mock_request_headers + ): """Test that DataDogLogger CustomLogger instance is disabled when 'datadog' specified in headers""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "datadog"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock DataDogLogger to avoid async initialization issues - with patch('litellm.integrations.datadog.datadog.DataDogLogger.__init__', return_value=None): + with patch( + "litellm.integrations.datadog.datadog.DataDogLogger.__init__", + return_value=None, + ): datadog_logger = DataDogLogger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(datadog_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + datadog_logger, litellm_params, standard_callback_dynamic_params + ) assert result is True def test_multiple_callbacks_disabled(self, mock_premium_user, mock_request_headers): """Test that multiple callbacks can be disabled with comma-separated list""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse,datadog,s3_v2"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "langfuse,datadog,s3_v2" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - # Test each callback is disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - - # Test non-disabled callback is not disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("prometheus", litellm_params, standard_callback_dynamic_params) is False - def test_callback_not_disabled_when_not_in_list(self, mock_premium_user, mock_request_headers): + # Test each callback is disabled + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + + # Test non-disabled callback is not disabled + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "prometheus", litellm_params, standard_callback_dynamic_params + ) + is False + ) + + def test_callback_not_disabled_when_not_in_list( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks not in the disabled list are not disabled""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_not_disabled_when_no_header(self, mock_premium_user, mock_request_headers): + def test_callback_not_disabled_when_no_header( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks are not disabled when the header is not present""" mock_request_headers.return_value = {} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_not_disabled_when_header_none(self, mock_premium_user, mock_request_headers): + def test_callback_not_disabled_when_header_none( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks are not disabled when the header value is None""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: None} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_non_premium_user_cannot_disable_callbacks(self, mock_non_premium_user, mock_request_headers): + def test_non_premium_user_cannot_disable_callbacks( + self, mock_non_premium_user, mock_request_headers + ): """Test that non-premium users cannot disable callbacks even with the header""" mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_case_insensitive_callback_matching(self, mock_premium_user, mock_request_headers): + def test_case_insensitive_callback_matching( + self, mock_premium_user, mock_request_headers + ): """Test that callback matching is case insensitive""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "LANGFUSE,DataDog"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "LANGFUSE,DataDog" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Test lowercase callbacks are disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) - def test_whitespace_handling_in_disabled_callbacks(self, mock_premium_user, mock_request_headers): + def test_whitespace_handling_in_disabled_callbacks( + self, mock_premium_user, mock_request_headers + ): """Test that whitespace around callback names is handled correctly""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: " langfuse , datadog , s3_v2 "} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: " langfuse , datadog , s3_v2 " + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - def test_custom_logger_not_in_registry(self, mock_premium_user, mock_request_headers): + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + + def test_custom_logger_not_in_registry( + self, mock_premium_user, mock_request_headers + ): """Test that CustomLogger not in registry is not disabled""" - mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "unknown_logger"} + mock_request_headers.return_value = { + X_LITELLM_DISABLE_CALLBACKS: "unknown_logger" + } litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Create a mock CustomLogger that's not in the registry class UnknownLogger(CustomLogger): pass - + unknown_logger = UnknownLogger() - result = EnterpriseCallbackControls.is_callback_disabled_dynamically(unknown_logger, litellm_params, standard_callback_dynamic_params) + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + unknown_logger, litellm_params, standard_callback_dynamic_params + ) assert result is False def test_exception_handling(self, mock_premium_user, mock_request_headers): @@ -188,32 +302,64 @@ class TestEnterpriseCallbackControls: mock_request_headers.side_effect = Exception("Test exception") litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False - def test_callback_disabled_via_request_body_langfuse(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_via_request_body_langfuse( + self, mock_premium_user, mock_request_headers + ): """Test that callbacks can be disabled via request body litellm_disabled_callbacks""" mock_request_headers.return_value = {} # No headers litellm_params = {"proxy_server_request": {"url": "test"}} - standard_callback_dynamic_params = StandardCallbackDynamicParams(litellm_disabled_callbacks=["langfuse"]) - - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + standard_callback_dynamic_params = StandardCallbackDynamicParams( + litellm_disabled_callbacks=["langfuse"] + ) + + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_callback_disabled_via_request_body_multiple(self, mock_premium_user, mock_request_headers): + def test_callback_disabled_via_request_body_multiple( + self, mock_premium_user, mock_request_headers + ): """Test that multiple callbacks can be disabled via request body""" mock_request_headers.return_value = {} # No headers litellm_params = {"proxy_server_request": {"url": "test"}} - standard_callback_dynamic_params = StandardCallbackDynamicParams(litellm_disabled_callbacks=["langfuse", "datadog", "s3_v2"]) - + standard_callback_dynamic_params = StandardCallbackDynamicParams( + litellm_disabled_callbacks=["langfuse", "datadog", "s3_v2"] + ) + # Test each callback is disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("datadog", litellm_params, standard_callback_dynamic_params) is True - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("s3_v2", litellm_params, standard_callback_dynamic_params) is True - + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "datadog", litellm_params, standard_callback_dynamic_params + ) + is True + ) + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "s3_v2", litellm_params, standard_callback_dynamic_params + ) + is True + ) + # Test non-disabled callback is not disabled - assert EnterpriseCallbackControls.is_callback_disabled_dynamically("prometheus", litellm_params, standard_callback_dynamic_params) is False + assert ( + EnterpriseCallbackControls.is_callback_disabled_dynamically( + "prometheus", litellm_params, standard_callback_dynamic_params + ) + is False + ) def test_admin_can_disable_dynamic_callback_disabling(self, mock_request_headers): """ @@ -223,11 +369,13 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock litellm.allow_dynamic_callback_disabling set to False - with patch('litellm.allow_dynamic_callback_disabling', False): - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.allow_dynamic_callback_disabling", False): + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is False def test_admin_can_enable_dynamic_callback_disabling(self, mock_request_headers): @@ -238,14 +386,18 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # Mock litellm.allow_dynamic_callback_disabling set to True - with patch('litellm.allow_dynamic_callback_disabling', True): - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.allow_dynamic_callback_disabling", True): + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True - def test_default_admin_setting_allows_dynamic_callback_disabling(self, mock_request_headers): + def test_default_admin_setting_allows_dynamic_callback_disabling( + self, mock_request_headers + ): """ Test that when allow_dynamic_callback_disabling is not set, it defaults to True and allows dynamic callback disabling for premium users @@ -253,8 +405,10 @@ class TestEnterpriseCallbackControls: mock_request_headers.return_value = {X_LITELLM_DISABLE_CALLBACKS: "langfuse"} litellm_params = {"proxy_server_request": {"url": "test"}} standard_callback_dynamic_params = StandardCallbackDynamicParams() - + # litellm.allow_dynamic_callback_disabling should default to True - with patch('litellm.proxy.proxy_server.premium_user', True): - result = EnterpriseCallbackControls.is_callback_disabled_dynamically("langfuse", litellm_params, standard_callback_dynamic_params) + with patch("litellm.proxy.proxy_server.premium_user", True): + result = EnterpriseCallbackControls.is_callback_disabled_dynamically( + "langfuse", litellm_params, standard_callback_dynamic_params + ) assert result is True diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 962806c5b52..898f42b45bf 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -442,6 +442,109 @@ class TestOpenTelemetryDualHandlerIsolation(unittest.TestCase): ) +class TestOpenTelemetryCaptureMessageContent(unittest.TestCase): + """OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT and the + OpenTelemetryConfig.capture_message_content programmatic override + drive what the handler captures in spans vs events.""" + + @staticmethod + def _make(env=None, config_value=None, message_logging=True): + env_dict = ( + {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": env} + if env is not None + else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ) + with patch.dict(os.environ, env_dict): + handler = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content=config_value + ) + ) + handler.message_logging = message_logging + return handler, handler._resolve_capture_mode() + + def test_no_explicit_setting_falls_back_to_message_logging_true(self): + _, mode = self._make() + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_no_explicit_setting_falls_back_to_message_logging_false(self): + _, mode = self._make(message_logging=False) + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_no_content(self): + _, mode = self._make(env="NO_CONTENT") + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_span_only(self): + _, mode = self._make(env="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_env_var_event_only(self): + _, mode = self._make(env="EVENT_ONLY") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_span_and_event(self): + _, mode = self._make(env="SPAN_AND_EVENT") + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_env_var_legacy_true_maps_to_event_only(self): + _, mode = self._make(env="true") + self.assertEqual(mode, "EVENT_ONLY") + + def test_env_var_legacy_false_maps_to_no_content(self): + for env in ("false", "0"): + with self.subTest(env=env): + _, mode = self._make(env=env) + self.assertEqual(mode, "NO_CONTENT") + + def test_env_var_unknown_value_falls_through_to_legacy(self): + _, mode = self._make(env="garbage", message_logging=True) + self.assertEqual(mode, "SPAN_AND_EVENT") + + def test_config_field_overrides_env(self): + _, mode = self._make(env="EVENT_ONLY", config_value="SPAN_ONLY") + self.assertEqual(mode, "SPAN_ONLY") + + def test_turn_off_message_logging_forces_no_content(self): + with patch("litellm.turn_off_message_logging", True): + _, mode = self._make(env="SPAN_AND_EVENT", message_logging=True) + self.assertEqual(mode, "NO_CONTENT") + + def test_capture_in_span_and_event_predicates(self): + cases = { + "NO_CONTENT": (False, False), + "SPAN_ONLY": (True, False), + "EVENT_ONLY": (False, True), + "SPAN_AND_EVENT": (True, True), + } + for mode, (in_span, in_event) in cases.items(): + handler, _ = self._make(env=mode) + self.assertEqual(handler._capture_in_span(), in_span, msg=mode) + self.assertEqual(handler._capture_in_event(), in_event, msg=mode) + + def test_two_handlers_can_have_different_modes(self): + # FIL's stated requirement: one handler strips content, the other keeps it. + with patch.dict( + os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""} + ): + stripped = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="NO_CONTENT" + ) + ) + kept = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", capture_message_content="SPAN_AND_EVENT" + ) + ) + self.assertEqual(stripped._resolve_capture_mode(), "NO_CONTENT") + self.assertEqual(kept._resolve_capture_mode(), "SPAN_AND_EVENT") + self.assertFalse(stripped._capture_in_span()) + self.assertFalse(stripped._capture_in_event()) + self.assertTrue(kept._capture_in_span()) + self.assertTrue(kept._capture_in_event()) + + class TestOpenTelemetry(unittest.TestCase): POLL_INTERVAL = 0.05 POLL_TIMEOUT = 2.0 @@ -1067,6 +1170,7 @@ class TestOpenTelemetry(unittest.TestCase): result = otel._get_span_name(kwargs) self.assertEqual(result, LITELLM_REQUEST_SPAN_NAME) + @patch.dict(os.environ, {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": ""}) @patch("litellm.turn_off_message_logging", False) def test_maybe_log_raw_request_creates_span(self): """Test _maybe_log_raw_request creates span when logging enabled""" @@ -2194,6 +2298,19 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase): See: https://github.com/BerriAI/litellm/issues/17794 """ + def setUp(self): + # Insulate from a shell-set OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + # so these tests exercise the legacy default path (message_logging=True). + self._prev = os.environ.pop( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", None + ) + + def tearDown(self): + if self._prev is not None: + os.environ["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = ( + self._prev + ) + def test_input_messages_uses_parts_structure(self): """ Test that gen_ai.input.messages uses the OTEL 1.38 parts array structure. @@ -3042,3 +3159,630 @@ class TestResponseIdFallback(unittest.TestCase): otel.set_attributes(mock_span, kwargs, response_obj) mock_span.set_attribute.assert_any_call("litellm.call_id", call_id) + + + +class TestOpenTelemetryResponsesAPI(unittest.TestCase): + """ + Tests for Responses API (/v1/responses) OTel span attributes. + + The Responses API uses ``output`` (list of output items) instead of + ``choices``, ``instructions`` instead of ``system_instructions``, and + ``status`` instead of per-choice ``finish_reason``. + + See: https://github.com/BerriAI/litellm/issues/25840 + """ + + def _base_kwargs(self, **overrides): + """Return minimal kwargs for set_attributes with Responses API defaults.""" + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "What is 2+2?"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "resp_abc123", + "call_type": "responses", + "metadata": {}, + }, + } + kwargs.update(overrides) + return kwargs + + def _responses_api_response_obj(self, text="The answer is 4.", status="completed"): + """Return a dict mimicking ResponsesAPIResponse with a message output.""" + return { + "id": "resp_abc123", + "model": "gpt-4o", + "status": status, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": text, + } + ], + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + }, + } + + def _get_attr(self, mock_span, attr_name): + """Extract the value set for a specific attribute name, or None.""" + calls = [ + call + for call in mock_span.set_attribute.call_args_list + if call[0][0] == attr_name + ] + if not calls: + return None + return calls[0][0][1] + + # ------------------------------------------------------------------ + # gen_ai.output.messages + # ------------------------------------------------------------------ + + def test_output_messages_populated_for_responses_api(self): + """gen_ai.output.messages must be set when response has output items.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs() + response_obj = self._responses_api_response_obj(text="The answer is 4.") + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + self.assertIsNotNone(raw, "gen_ai.output.messages should be set") + + parsed = json.loads(raw) + self.assertIsInstance(parsed, list) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertIn("parts", parsed[0]) + self.assertEqual(parsed[0]["parts"][0]["type"], "text") + self.assertEqual(parsed[0]["parts"][0]["content"], "The answer is 4.") + + def test_output_messages_with_multiple_content_items(self): + """Multiple output_text items in a single message should all appear as parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_multi", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "First paragraph."}, + {"type": "output_text", "text": "Second paragraph."}, + ], + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed[0]["parts"]), 2) + self.assertEqual(parsed[0]["parts"][0]["content"], "First paragraph.") + self.assertEqual(parsed[0]["parts"][1]["content"], "Second paragraph.") + + def test_output_messages_with_function_call(self): + """function_call output items should appear as tool_call parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_fc", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_abc", + "arguments": '{"location": "SF"}', + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed), 1) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertEqual(parsed[0]["parts"][0]["type"], "tool_call") + self.assertEqual(parsed[0]["parts"][0]["name"], "get_weather") + self.assertEqual(parsed[0]["parts"][0]["arguments"], '{"location": "SF"}') + self.assertEqual(parsed[0]["parts"][0]["id"], "call_abc") + + def test_output_messages_mixed_message_and_function_call(self): + """Mixed output with both message and function_call items.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_mixed", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Let me check the weather."}, + ], + }, + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_xyz", + "arguments": "{}", + }, + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(len(parsed), 2) + self.assertEqual(parsed[0]["role"], "assistant") + self.assertEqual(parsed[0]["parts"][0]["content"], "Let me check the weather.") + self.assertEqual(parsed[1]["parts"][0]["type"], "tool_call") + + def test_output_messages_empty_text_skipped(self): + """Output items with empty text should not produce parts.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_empty", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": ""}], + } + ], + } + + otel.set_attributes( + span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj + ) + + # No output messages should be set since the text is empty + raw = self._get_attr(mock_span, "gen_ai.output.messages") + self.assertIsNone(raw, "Empty output text should not produce gen_ai.output.messages") + + def test_choices_still_work(self): + """Existing choices-based responses must still work (no regression).""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + }, + } + + response_obj = { + "id": "chatcmpl-123", + "model": "gpt-4", + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hi there!"}, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15}, + } + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.output.messages") + parsed = json.loads(raw) + self.assertEqual(parsed[0]["parts"][0]["content"], "Hi there!") + self.assertEqual(parsed[0]["finish_reason"], "stop") + + # ------------------------------------------------------------------ + # gen_ai.response.finish_reasons + # ------------------------------------------------------------------ + + def test_finish_reasons_from_status(self): + """gen_ai.response.finish_reasons should use ResponsesAPIResponse.status.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + otel.set_attributes( + span=mock_span, + kwargs=self._base_kwargs(), + response_obj=self._responses_api_response_obj(status="completed"), + ) + + raw = self._get_attr(mock_span, "gen_ai.response.finish_reasons") + self.assertIsNotNone(raw) + parsed = json.loads(raw) + self.assertEqual(parsed, ["completed"]) + + def test_finish_reasons_incomplete_status(self): + """Non-completed status values should still be captured.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + otel.set_attributes( + span=mock_span, + kwargs=self._base_kwargs(), + response_obj=self._responses_api_response_obj(status="incomplete"), + ) + + raw = self._get_attr(mock_span, "gen_ai.response.finish_reasons") + parsed = json.loads(raw) + self.assertEqual(parsed, ["incomplete"]) + + # ------------------------------------------------------------------ + # gen_ai.system_instructions + # ------------------------------------------------------------------ + + def test_system_instructions_from_instructions_kwarg(self): + """Responses API passes system prompt as kwargs['instructions'].""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs(instructions="You are a math tutor.") + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "You are a math tutor.") + + def test_system_instructions_from_system_kwarg(self): + """Anthropic Messages API passes system prompt as kwargs['system'].""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs(system="You are a helpful assistant.") + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "You are a helpful assistant.") + + def test_system_instructions_from_system_instructions_kwarg(self): + """Vertex AI Gemini path uses kwargs['system_instructions'] (existing behavior).""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions=[{"role": "system", "content": "Be concise."}] + ) + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + raw = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertIsNotNone(raw) + parsed = json.loads(raw) + self.assertEqual(parsed[0]["role"], "system") + self.assertIn("parts", parsed[0]) + + def test_system_instructions_precedence(self): + """system_instructions takes precedence over instructions and system.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions="From Gemini", + instructions="From Responses API", + system="From Anthropic", + ) + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # system_instructions (string) should win — it's checked first + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertEqual(value, "From Gemini") + + def test_no_system_instructions_when_absent(self): + """No gen_ai.system_instructions attr when none of the kwargs are set.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs() + response_obj = self._responses_api_response_obj() + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + value = self._get_attr(mock_span, "gen_ai.system_instructions") + self.assertIsNone(value) + + +class TestTransformResponsesAPIOutput(unittest.TestCase): + """ + Unit tests for _transform_responses_api_output_to_otel. + """ + + def test_message_with_output_text(self): + otel = OpenTelemetry() + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello!"}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "assistant") + self.assertEqual(result[0]["parts"], [{"type": "text", "content": "Hello!"}]) + + def test_function_call_item(self): + otel = OpenTelemetry() + output = [ + { + "type": "function_call", + "name": "search", + "call_id": "call_1", + "arguments": '{"q": "test"}', + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "assistant") + self.assertEqual(result[0]["parts"][0]["type"], "tool_call") + self.assertEqual(result[0]["parts"][0]["name"], "search") + self.assertEqual(result[0]["parts"][0]["id"], "call_1") + + def test_function_call_without_call_id(self): + otel = OpenTelemetry() + output = [ + { + "type": "function_call", + "name": "search", + "arguments": "{}", + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertNotIn("id", result[0]["parts"][0]) + + def test_unknown_type_ignored(self): + otel = OpenTelemetry() + output = [{"type": "reasoning", "content": "thinking..."}] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_non_dict_items_ignored(self): + otel = OpenTelemetry() + output = ["not a dict", 42, None] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_empty_output(self): + otel = OpenTelemetry() + result = otel._transform_responses_api_output_to_otel([]) + self.assertEqual(result, []) + + def test_message_with_empty_text_skipped(self): + otel = OpenTelemetry() + output = [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": ""}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result, []) + + def test_message_default_role(self): + """Messages without explicit role should default to assistant.""" + otel = OpenTelemetry() + output = [ + { + "type": "message", + "content": [{"type": "output_text", "text": "Hi"}], + } + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(result[0]["role"], "assistant") + + + def test_pydantic_like_objects_accepted(self): + """Items with .get() but not isinstance(dict) should be accepted.""" + + class FakeOutputItem: + """Mimics BaseLiteLLMOpenAIResponseObject duck-typing.""" + + def __init__(self, data): + self._data = data + + def get(self, key, default=None): + return self._data.get(key, default) + + class FakeContent: + def __init__(self, data): + self._data = data + + def get(self, key, default=None): + return self._data.get(key, default) + + otel = OpenTelemetry() + output = [ + FakeOutputItem( + { + "type": "message", + "role": "assistant", + "content": [ + FakeContent({"type": "output_text", "text": "Pydantic works!"}), + ], + } + ) + ] + result = otel._transform_responses_api_output_to_otel(output) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["parts"][0]["content"], "Pydantic works!") + + +class TestSystemInstructionsPrecedence(unittest.TestCase): + """Tests for the is-not-None precedence in system_instructions coalescing.""" + + def _get_attr(self, mock_span, attr_name): + calls = [ + call + for call in mock_span.set_attribute.call_args_list + if call[0][0] == attr_name + ] + if not calls: + return None + return calls[0][0][1] + + def _base_kwargs(self, **overrides): + kwargs = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hi"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "test-id", + "call_type": "responses", + "metadata": {}, + }, + } + kwargs.update(overrides) + return kwargs + + def test_empty_list_system_instructions_does_not_fallthrough(self): + """An empty list for system_instructions should NOT fall through to instructions.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + kwargs = self._base_kwargs( + system_instructions=[], + instructions="Should not be used", + ) + response_obj = {"id": "r1", "model": "gpt-4o"} + + otel.set_attributes(span=mock_span, kwargs=kwargs, response_obj=response_obj) + + # system_instructions is [] (falsy but not None), so it wins. + # Since it's an empty list, no attribute should be set (nothing to transform). + value = self._get_attr(mock_span, "gen_ai.system_instructions") + # The empty list is truthy for `is not None` but produces empty + # transformed output — the attribute should NOT contain "Should not be used". + if value is not None: + self.assertNotIn("Should not be used", str(value)) + + +class TestResponsesAPIToolCallSpanAttributes(unittest.TestCase): + """Tests for per-tool-call span attributes on Responses API function_call items.""" + + def _base_kwargs(self): + return { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "What is the weather?"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": { + "id": "resp_tc", + "call_type": "responses", + "metadata": {}, + }, + } + + def test_per_tool_call_attributes_emitted(self): + """function_call output items should produce per-tool-call span attributes.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_tc", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_abc", + "arguments": '{"location": "SF"}', + } + ], + } + + otel.set_attributes(span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj) + + # Verify per-tool-call attributes were set (same format as choices branch) + attr_names = [call[0][0] for call in mock_span.set_attribute.call_args_list] + tool_call_attrs = [a for a in attr_names if "function_call" in a] + self.assertTrue(len(tool_call_attrs) > 0, "Per-tool-call span attributes should be emitted") + + # Verify the name attribute specifically + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.name", "get_weather" + ) + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.arguments", '{"location": "SF"}' + ) + + def test_multiple_tool_calls_indexed(self): + """Multiple function_call items should be indexed correctly.""" + otel = OpenTelemetry() + mock_span = MagicMock() + + response_obj = { + "id": "resp_tc2", + "model": "gpt-4o", + "status": "completed", + "output": [ + { + "type": "function_call", + "name": "get_weather", + "call_id": "call_1", + "arguments": "{}", + }, + { + "type": "function_call", + "name": "get_time", + "call_id": "call_2", + "arguments": "{}", + }, + ], + } + + otel.set_attributes(span=mock_span, kwargs=self._base_kwargs(), response_obj=response_obj) + + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.0.function_call.name", "get_weather" + ) + mock_span.set_attribute.assert_any_call( + "gen_ai.completion.1.function_call.name", "get_time" + ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1764d9c609f..e84baf5e137 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -36,6 +36,19 @@ def test_get_masked_api_base(logging_obj): assert type(masked_api_base) == str +def test_post_call_serializes_dict_with_datetime(logging_obj): + import datetime + + response = { + "status": "InProgress", + "submitTime": datetime.datetime(2026, 5, 11, 23, 49, 13, 132000), + } + logging_obj.post_call(original_response=response) + serialized = logging_obj.model_call_details["original_response"] + assert isinstance(serialized, str) + assert "2026-05-11" in serialized + + def test_sentry_sample_rate(): existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE") try: diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index e38698c9100..a19752dc648 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2135,6 +2135,53 @@ def test_validate_effort_for_model_centralises_per_model_gating( assert err is None +def test_transform_request_injects_dummy_tool_without_tools_param(): + """ + Anthropic rejects messages that contain tool turns when ``tools`` is omitted. + LiteLLM must inject a dummy tool without ``litellm.modify_params``. + """ + config = AnthropicConfig() + prev_modify_params = litellm.modify_params + litellm.modify_params = False + try: + messages = [ + {"role": "user", "content": "Hello"}, + { + "role": "assistant", + "content": "Calling tool", + "tool_calls": [ + { + "id": "toolu_test_dummy", + "type": "function", + "function": {"name": "get_x", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "toolu_test_dummy", + "content": "{}", + }, + ] + result = config.transform_request( + model="claude-3-5-haiku-20241022", + messages=messages, + optional_params={"max_tokens": 256}, + litellm_params={}, + headers={}, + ) + finally: + litellm.modify_params = prev_modify_params + + assert "tools" in result + names = [ + t.get("name") + for t in result["tools"] + if isinstance(t, dict) and t.get("name") is not None + ] + assert "dummy_tool" in names + + def test_transform_request_uses_dynamic_max_tokens(): """ Test that transform_request uses dynamic max_tokens based on model diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/test_litellm/llms/bedrock/batches/test_handler.py new file mode 100644 index 00000000000..18780ccce0f --- /dev/null +++ b/tests/test_litellm/llms/bedrock/batches/test_handler.py @@ -0,0 +1,338 @@ +"""Unit tests for ``BedrockBatchesHandler._handle_model_invocation_job_status``. + +These cover the upstream support for retrieving Bedrock bulk batch jobs +(``arn:aws:bedrock:::model-invocation-job/``) — the ARN +type returned by ``CreateModelInvocationJob``. We mock the boto3 client so +the tests don't hit AWS. +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.bedrock.batches.handler import ( # noqa: E402 + BedrockBatchesHandler, + _extract_job_id_from_arn, + _extract_region_from_bedrock_arn, + _predict_output_file_uri, + _to_epoch, +) + +JOB_ID = "abc1234567" +JOB_ARN = f"arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/{JOB_ID}" +INPUT_URI = "s3://my-bucket/inputs/qwen3-235b-a22b-2507-batch.jsonl" +OUTPUT_PREFIX = "s3://my-bucket/litellm-batch-outputs/litellm-bedrock-files-qwen-uuid/" +SUBMIT_TIME = datetime(2026, 4, 28, 12, 0, 0, tzinfo=timezone.utc) +END_TIME = datetime(2026, 4, 28, 12, 30, 0, tzinfo=timezone.utc) + + +def _fake_boto3_response(status: str = "Completed", end_time=END_TIME): + return { + "jobArn": JOB_ARN, + "jobName": "litellm-bedrock-files-qwen-uuid", + "modelId": "bedrock/qwen.qwen3-235b-a22b-2507-v1:0", + "status": status, + "submitTime": SUBMIT_TIME, + "lastModifiedTime": end_time, + "endTime": end_time, + "inputDataConfig": {"s3InputDataConfig": {"s3Uri": INPUT_URI}}, + "outputDataConfig": {"s3OutputDataConfig": {"s3Uri": OUTPUT_PREFIX}}, + } + + +@pytest.fixture +def patched_boto3(): + """Yield a stub bedrock client whose `get_model_invocation_job` is a MagicMock.""" + fake_client = MagicMock() + fake_client.get_model_invocation_job.return_value = _fake_boto3_response() + with ( + patch("boto3.client", return_value=fake_client) as boto_client_factory, + patch( + "litellm.llms.bedrock.batches.transformation.BedrockBatchesConfig.get_credentials", + return_value=MagicMock(access_key="AKIA", secret_key="SECRET", token=None), + ), + ): + yield fake_client, boto_client_factory + + +def test_extract_region_from_arn(): + assert _extract_region_from_bedrock_arn(JOB_ARN) == "us-west-2" + assert _extract_region_from_bedrock_arn("arn:aws:bedrock::123:foo/bar") is None + assert _extract_region_from_bedrock_arn("not-an-arn") is None + + +def test_extract_region_swallows_unexpected_split_errors(): + """Defensive `except Exception` branch — anything that isn't a plain str + should fall through to ``None`` rather than blow up.""" + + class WeirdArn: + def split(self, _sep): + raise RuntimeError("boom") + + assert _extract_region_from_bedrock_arn(WeirdArn()) is None # type: ignore[arg-type] + + +def test_predict_output_file_uri_returns_none_for_directory_input_uri(): + """Input URI ending in `/` has an empty basename — we must bail rather + than emit ``//.out``.""" + assert ( + _predict_output_file_uri(OUTPUT_PREFIX, "s3://bucket/inputs/", JOB_ID) is None + ) + + +_DT = datetime(2026, 4, 28, 12, 0, 0, tzinfo=timezone.utc) + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, None), + (1730000000, 1730000000), + (1730000000.5, 1730000000), + (_DT, int(_DT.timestamp())), + ("2026-04-28T12:00:00Z", None), # strings aren't supported -> None + ], +) +def test_to_epoch_handles_supported_types(value, expected): + assert _to_epoch(value) == expected + + +def test_extract_job_id_from_arn(): + assert _extract_job_id_from_arn(JOB_ARN) == JOB_ID + assert ( + _extract_job_id_from_arn("arn:aws:bedrock:us-west-2:1:async-invoke/x") is None + ) + + +def test_predict_output_file_uri_happy_path(): + expected = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert _predict_output_file_uri(OUTPUT_PREFIX, INPUT_URI, JOB_ID) == expected + + +def test_predict_output_file_uri_adds_trailing_slash(): + prefix_no_slash = OUTPUT_PREFIX.rstrip("/") + expected = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert _predict_output_file_uri(prefix_no_slash, INPUT_URI, JOB_ID) == expected + + +@pytest.mark.parametrize( + "missing_arg", + [ + ("", INPUT_URI, JOB_ID), + (OUTPUT_PREFIX, "", JOB_ID), + (OUTPUT_PREFIX, INPUT_URI, None), + ], +) +def test_predict_output_file_uri_returns_none_when_missing_input(missing_arg): + assert _predict_output_file_uri(*missing_arg) is None + + +def test_handle_model_invocation_job_status_completed(patched_boto3): + fake_client, boto_client_factory = patched_boto3 + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + fake_client.get_model_invocation_job.assert_called_once_with(jobIdentifier=JOB_ARN) + + # Region should be sniffed from the ARN. + _, kwargs = boto_client_factory.call_args + assert kwargs["region_name"] == "us-west-2" + + assert batch.id == JOB_ARN + assert batch.status == "completed" + assert batch.input_file_id == INPUT_URI + expected_out = f"{OUTPUT_PREFIX}{JOB_ID}/qwen3-235b-a22b-2507-batch.jsonl.out" + assert batch.output_file_id == expected_out + assert batch.completed_at == int(END_TIME.timestamp()) + assert batch.failed_at is None + assert batch.cancelled_at is None + # Per-record counts aren't reported by GetModelInvocationJob, so we leave + # them zeroed; consumers should parse manifest.json.out for accurate counts. + assert batch.request_counts.total == 0 + assert batch.metadata["job_arn"] == JOB_ARN + assert batch.metadata["output_file_uri"] == expected_out + assert batch.metadata["output_s3_uri"] == OUTPUT_PREFIX + + +@pytest.mark.parametrize( + "bedrock_status,openai_status", + [ + ("Submitted", "validating"), + ("Validating", "validating"), + ("Scheduled", "validating"), + ("InProgress", "in_progress"), + ("Stopping", "cancelling"), + ("Stopped", "cancelled"), + ("Completed", "completed"), + ("PartiallyCompleted", "completed"), + ("Failed", "failed"), + ("Expired", "expired"), + # Unknown/unmapped Bedrock status falls back to "in_progress" so we + # don't 500 on a future AWS-side enum addition. + ("MyBrandNewStatus", "in_progress"), + ], +) +def test_status_mapping(patched_boto3, bedrock_status, openai_status): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status=bedrock_status + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == openai_status + # output_file_id is only populated for terminal-completed jobs, so callers + # don't accidentally try to download a non-existent file mid-run. + if openai_status == "completed": + assert batch.output_file_id is not None + else: + assert batch.output_file_id is None + + +def test_explicit_region_overrides_arn(patched_boto3): + _, boto_client_factory = patched_boto3 + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ARN, aws_region_name="eu-central-1" + ) + _, kwargs = boto_client_factory.call_args + assert kwargs["region_name"] == "eu-central-1" + + +def test_failure_message_propagates(patched_boto3): + fake_client, _ = patched_boto3 + failed_response = _fake_boto3_response(status="Failed") + failed_response["message"] = "Input file failed validation" + fake_client.get_model_invocation_job.return_value = failed_response + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "failed" + assert batch.failed_at == int(END_TIME.timestamp()) + assert batch.metadata["failure_message"] == "Input file failed validation" + + +def test_completed_with_unpredictable_output_uri_stays_none(patched_boto3): + """ + Regression guard for the original NoSuchKey bug: if Bedrock's response is + missing pieces we need to compute the per-job output file path (here, the + input s3Uri), `output_file_id` must stay `None` rather than fall back to + the bare prefix. Falling back to the prefix is what produced the original + NoSuchKey error this PR fixes. + """ + fake_client, _ = patched_boto3 + incomplete_response = _fake_boto3_response(status="Completed") + incomplete_response["inputDataConfig"] = {"s3InputDataConfig": {"s3Uri": ""}} + fake_client.get_model_invocation_job.return_value = incomplete_response + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "completed" + # output_file_id MUST be None (not the bare prefix) — that's the whole + # point of this regression test. Callers branch on this field. + assert batch.output_file_id is None + # The metadata field uses "" because OpenAI Batch metadata is dict[str, str]; + # callers should branch on `output_file_id` (above) instead. + assert batch.metadata["output_file_uri"] == "" + # The bare prefix is still preserved in metadata so callers can list it. + assert batch.metadata["output_s3_uri"] == OUTPUT_PREFIX + + +def test_cancelled_status_sets_cancelled_at(patched_boto3): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status="Stopped" + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "cancelled" + assert batch.cancelled_at == int(END_TIME.timestamp()) + assert batch.completed_at is None + assert batch.failed_at is None + assert batch.expired_at is None + + +def test_expired_status_sets_expired_at(patched_boto3): + fake_client, _ = patched_boto3 + fake_client.get_model_invocation_job.return_value = _fake_boto3_response( + status="Expired" + ) + + batch = BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + assert batch.status == "expired" + assert batch.expired_at == int(END_TIME.timestamp()) + assert batch.completed_at is None + assert batch.failed_at is None + assert batch.cancelled_at is None + + +def test_logging_obj_pre_and_post_call_invoked(patched_boto3): + """`pre_call` / `post_call` get called with sensible payloads when a + `logging_obj` is supplied.""" + _, _ = patched_boto3 + logging_obj = MagicMock() + + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ARN, logging_obj=logging_obj + ) + + logging_obj.pre_call.assert_called_once() + logging_obj.post_call.assert_called_once() + + pre_kwargs = logging_obj.pre_call.call_args.kwargs + assert pre_kwargs["input"] == JOB_ARN + assert pre_kwargs["additional_args"]["complete_input_dict"] == { + "jobIdentifier": JOB_ARN + } + # Logged URL must use the bare job id, not the full ARN, so it doesn't + # double the `model-invocation-job/` segment or embed colons in the path. + assert pre_kwargs["additional_args"]["api_base"] == ( + f"https://bedrock.us-west-2.amazonaws.com/model-invocation-job/{JOB_ID}" + ) + + post_kwargs = logging_obj.post_call.call_args.kwargs + assert post_kwargs["input"] == JOB_ARN + assert post_kwargs["original_response"]["jobArn"] == JOB_ARN + + +def test_missing_boto3_raises_helpful_import_error(): + """If boto3 isn't installed we should raise a clear, actionable + ImportError rather than letting a NameError escape.""" + real_import = ( + __builtins__["__import__"] + if isinstance(__builtins__, dict) + else __builtins__.__import__ + ) + + def fake_import(name, *args, **kwargs): + if name == "boto3": + raise ImportError("No module named 'boto3'") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=fake_import): + with pytest.raises(ImportError, match="pip install boto3"): + BedrockBatchesHandler._handle_model_invocation_job_status(batch_id=JOB_ARN) + + +def test_logging_url_uses_bare_id_when_only_id_passed(patched_boto3): + """If the caller passes just the trailing job id (also valid for + `GetModelInvocationJob`), the logged URL should use it as-is.""" + _, _ = patched_boto3 + logging_obj = MagicMock() + + BedrockBatchesHandler._handle_model_invocation_job_status( + batch_id=JOB_ID, aws_region_name="us-west-2", logging_obj=logging_obj + ) + + pre_kwargs = logging_obj.pre_call.call_args.kwargs + assert pre_kwargs["additional_args"]["api_base"] == ( + f"https://bedrock.us-west-2.amazonaws.com/model-invocation-job/{JOB_ID}" + ) diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index c98f7840343..9ecdad1fcff 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -867,10 +867,12 @@ def test_bedrock_messages_explicit_output_config_wins_over_reasoning_effort(): def test_bedrock_messages_strips_context_management(): """ Ensure context_management is stripped from the request before sending to - Bedrock Invoke, which doesn't support this Anthropic-specific parameter. + Bedrock Invoke when it carries only LiteLLM-internal edits (e.g. + clear_thinking_20251015, which is consumed via thinking injection). - Claude Code sends context_management on every request; leaving it in the body - causes a 400 "context_management: Extra inputs are not permitted" from Bedrock. + Claude Code sends context_management on every request; leaving such edits + in the body causes a 400 "context_management: Extra inputs are not + permitted" from Bedrock. """ from litellm.types.router import GenericLiteLLMParams @@ -897,6 +899,77 @@ def test_bedrock_messages_strips_context_management(): assert result.get("max_tokens") == 4096 +def test_bedrock_messages_preserves_compact_context_management_and_adds_beta(): + """ + Bedrock InvokeModel supports compaction when paired with the + ``compact-2026-01-12`` anthropic-beta header, even though the Converse API + does not. The transformation should: + 1. Keep ``context_management`` with compact_20260112 edits in the body + (Bedrock rejects unknown top-level fields, but accepts this one with + the right beta). + 2. Auto-inject ``compact-2026-01-12`` into ``anthropic_beta``. + + Ref: https://github.com/BerriAI/litellm/issues/27532 + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [{"type": "compact_20260112"}] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-6-20250929-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("context_management") == { + "edits": [{"type": "compact_20260112"}] + } + assert "compact-2026-01-12" in result.get("anthropic_beta", []) + assert result["max_tokens"] == 4096 + + +def test_bedrock_messages_filters_unsupported_context_management_edits(): + """ + Mixed edit lists must drop the LiteLLM-internal ``clear_thinking_20251015`` + entries while keeping ``compact_20260112`` and adding the compact beta. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + {"type": "compact_20260112"}, + ] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-6-20250929-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("context_management") == { + "edits": [{"type": "compact_20260112"}] + } + assert "compact-2026-01-12" in result.get("anthropic_beta", []) + + def test_bedrock_messages_allowlist_filters_anthropic_only_fields(): """ Bedrock Invoke rejects any top-level body field it doesn't recognize with diff --git a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py new file mode 100644 index 00000000000..74cfbb265cd --- /dev/null +++ b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py @@ -0,0 +1,312 @@ +import json +from unittest.mock import patch + +import httpx +import pytest + + +def _anthropic_response(url: str) -> httpx.Response: + return httpx.Response( + status_code=200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=httpx.Request("POST", url), + ) + + +def _capture_request(url: str, headers: dict, data: bytes | str | None) -> dict: + raw_body = data.decode("utf-8") if isinstance(data, bytes) else data or "{}" + return { + "path": httpx.URL(url).path, + "headers": headers, + "body": json.loads(raw_body), + } + + +def test_claude_platform_builds_default_messages_url_from_region(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + + assert ( + config.get_complete_url( + api_base=None, + api_key=None, + model="claude-sonnet-4-6", + optional_params={"aws_region_name": "us-west-2"}, + litellm_params={}, + ) + == "https://aws-external-anthropic.us-west-2.api.aws/v1/messages" + ) + + +def test_claude_platform_ignores_standard_anthropic_base_url(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://api.anthropic.example") + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.anthropic-api.example") + + config = BedrockClaudePlatformConfig() + + assert ( + config.get_complete_url( + api_base=None, + api_key=None, + model="claude-sonnet-4-6", + optional_params={"aws_region_name": "us-west-2"}, + litellm_params={}, + ) + == "https://aws-external-anthropic.us-west-2.api.aws/v1/messages" + ) + + +def test_claude_platform_uses_bedrock_subroute(): + import litellm + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + model, provider, _, _ = litellm.get_llm_provider( + model="bedrock/claude_platform/claude-sonnet-4-6" + ) + + assert provider == "bedrock" + assert model == "claude_platform/claude-sonnet-4-6" + assert BedrockModelInfo.get_bedrock_route(model) == "claude_platform" + assert BedrockModelInfo.get_claude_platform_model(model) == "claude-sonnet-4-6" + + +def test_claude_platform_requires_workspace_header(): + from litellm import AuthenticationError + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + + with pytest.raises(AuthenticationError) as exc_info: + config.validate_environment( + api_key="fake-platform-key", + headers={}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + ) + + assert "workspace" in str(exc_info.value).lower() + + +def test_claude_platform_api_key_auth_sets_workspace_and_key_headers(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + headers = config.validate_environment( + api_key="fake-platform-key", + headers={"anthropic-beta": "skills-2025-10-02"}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"workspace_id": "wrkspc_test"}, + litellm_params={}, + ) + + assert headers["x-api-key"] == "fake-platform-key" + assert headers["anthropic-workspace-id"] == "wrkspc_test" + assert headers["anthropic-beta"] == "skills-2025-10-02" + + +def test_claude_platform_does_not_use_standard_anthropic_api_key(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_API_KEY", "standard-anthropic-key") + + config = BedrockClaudePlatformConfig() + headers = config.validate_environment( + api_key=None, + headers={}, + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"workspace_id": "wrkspc_test"}, + litellm_params={}, + ) + + assert "x-api-key" not in headers + + +def test_claude_platform_sigv4_signs_transformed_request_body(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + request_body = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + with patch.object( + config, + "_sign_request", + return_value=({"Authorization": "signed"}, json.dumps(request_body).encode()), + ) as mock_sign_request: + headers, signed_body = config.sign_request( + headers={"anthropic-workspace-id": "wrkspc_test"}, + optional_params={"aws_region_name": "us-west-2"}, + request_data=request_body, + api_base="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + api_key=None, + model="claude-sonnet-4-6", + ) + + assert signed_body == json.dumps(request_body).encode() + assert headers["Authorization"] == "signed" + mock_sign_request.assert_called_once() + assert ( + mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" + ) + assert mock_sign_request.call_args.kwargs["request_data"] == request_body + + +def test_claude_platform_standard_anthropic_api_key_does_not_skip_sigv4(monkeypatch): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + monkeypatch.setenv("ANTHROPIC_API_KEY", "standard-anthropic-key") + config = BedrockClaudePlatformConfig() + request_body = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + with patch.object( + config, + "_sign_request", + return_value=({"Authorization": "signed"}, json.dumps(request_body).encode()), + ) as mock_sign_request: + headers, signed_body = config.sign_request( + headers={"anthropic-workspace-id": "wrkspc_test"}, + optional_params={"aws_region_name": "us-west-2"}, + request_data=request_body, + api_base="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + api_key=None, + model="claude-sonnet-4-6", + ) + + assert signed_body == json.dumps(request_body).encode() + assert headers["Authorization"] == "signed" + mock_sign_request.assert_called_once() + + +def test_bedrock_claude_platform_messages_config_round_trips_native_body(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + + assert config is not None + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={"max_tokens": 10}, + litellm_params={"workspace_id": "wrkspc_test"}, + ) + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params={"max_tokens": 10}, + litellm_params={}, + headers=headers, + ) + + assert headers["anthropic-workspace-id"] == "wrkspc_test" + assert headers["x-api-key"] == "fake-platform-key" + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + + +def test_chat_completion_routes_bedrock_claude_platform_to_messages_api(): + import litellm + + requests = [] + + def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): + response = litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + api_base="https://aws-external-anthropic.us-west-2.api.aws", + api_key="fake-platform-key", + workspace_id="wrkspc_test", + ) + + assert response.choices[0].message.content == "ok" + assert len(requests) == 1 + assert requests[0]["path"] == "/v1/messages" + assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" + assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" + assert requests[0]["body"]["model"] == "claude-sonnet-4-6" + + +@pytest.mark.asyncio +async def test_anthropic_messages_routes_bedrock_claude_platform_to_messages_api(): + import litellm + + requests = [] + + async def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + response = await litellm.anthropic_messages( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + api_base="https://aws-external-anthropic.us-west-2.api.aws", + api_key="fake-platform-key", + workspace_id="wrkspc_test", + ) + finally: + await litellm.close_litellm_async_clients() + + assert response["content"][0]["text"] == "ok" + assert len(requests) == 1 + assert requests[0]["path"] == "/v1/messages" + assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" + assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" + assert requests[0]["body"]["messages"] == [{"role": "user", "content": "hello"}] + assert requests[0]["body"]["max_tokens"] == 10 + assert requests[0]["body"]["model"] == "claude-sonnet-4-6" diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 678aa6b56c1..45ce5d58405 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -94,27 +94,33 @@ def test_github_copilot_config_get_openai_compatible_provider_info(): @patch("litellm.llms.github_copilot.authenticator.Authenticator.get_api_key") +@patch("litellm.main.openai_chat_completions.completion") @patch("litellm.llms.openai.openai.OpenAIChatCompletion.completion") -def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_key): +def test_completion_github_copilot_mock_response( + mock_class_completion, mock_instance_completion, mock_get_api_key, monkeypatch +): """Test the completion function with GitHub Copilot provider.""" - # Mock the API key return value + # Force chat path through the patched openai_chat_completions instance even if + # a previous test left EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER set in the env. + monkeypatch.delenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", raising=False) + mock_api_key = "gh.test-key-123456789" mock_get_api_key.return_value = mock_api_key - # Mock completion response mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Hello, I'm GitHub Copilot!" - mock_completion.return_value = mock_response + # Patch both the class method and the live module-level instance to survive + # conftest module reloads that can swap which class object is in use. + mock_class_completion.return_value = mock_response + mock_instance_completion.return_value = mock_response - # Test non-streaming completion messages = [ {"role": "system", "content": "You're GitHub Copilot, an AI assistant."}, {"role": "user", "content": "Hello, who are you?"}, ] - # Create a properly formatted headers dictionary headers = { "editor-version": "Neovim/0.9.0", "Copilot-Integration-Id": "vscode-chat", @@ -128,19 +134,16 @@ def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_k assert response is not None - # Verify the get_api_key call was made (can be called multiple times) assert mock_get_api_key.call_count >= 1 - # Verify the completion call was made with the expected params - mock_completion.assert_called_once() - args, kwargs = mock_completion.call_args + # Exactly one of the two patched targets should have been used. + invoked = [m for m in (mock_class_completion, mock_instance_completion) if m.called] + assert len(invoked) == 1 + invoked[0].assert_called_once() + _, kwargs = invoked[0].call_args - # Check that the proper authorization header is set assert "headers" in kwargs - # Check that the model name is correctly formatted - assert ( - kwargs.get("model") == "gpt-4" - ) # Model name should be without provider prefix + assert kwargs.get("model") == "gpt-4" assert kwargs.get("messages") == messages diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index ebf7681f2f3..d279b119efe 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1,10 +1,15 @@ import pytest import litellm +import litellm.main as litellm_main from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig -from litellm.utils import _is_explicitly_disabled_factory +from litellm.utils import ( + _is_explicitly_disabled_factory, + peek_reasoning_summary_aliases, + strip_reasoning_summary_aliases_from_optional_params, +) @pytest.fixture() @@ -1007,6 +1012,76 @@ def test_gpt5_search_drops_unsupported_params(config: OpenAIConfig): assert "tools" not in params +def test_gpt5_chat_strips_reasoning_summary_aliases_after_bridge_check( + monkeypatch: pytest.MonkeyPatch, +): + """Non-bridged GPT-5 chat calls strip Responses-only reasoning summary aliases.""" + captured_kwargs = {} + + def fake_openai_completion(**kwargs): + captured_kwargs.update(kwargs) + return {} + + monkeypatch.setattr( + litellm_main.openai_chat_completions, + "completion", + fake_openai_completion, + ) + + litellm.completion( + model="gpt-5", + messages=[{"role": "user", "content": "ok"}], + reasoningSummary="auto", + extra_body={"reasoning_summary": "ignored", "metadata": "ok"}, + api_key="fake-key", + ) + + optional_params = captured_kwargs["optional_params"] + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + assert optional_params["extra_body"] == {"metadata": "ok"} + + +def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): + optional_params = {"reasoningSummary": False, "reasoning_summary": "ignored"} + + assert peek_reasoning_summary_aliases(optional_params) is False + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val is False + assert stripped == {} + + optional_params = { + "extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"} + } + + assert peek_reasoning_summary_aliases(optional_params) is False + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val is False + assert stripped == {} + + optional_params = { + "extra_body": { + "reasoningSummary": "auto", + "reasoning_summary": "ignored", + "metadata": "ok", + } + } + + assert peek_reasoning_summary_aliases(optional_params) == "auto" + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) + + assert rs_val == "auto" + assert stripped == {"extra_body": {"metadata": "ok"}} + + # GPT-5 unsupported params audit (validated via direct API calls) def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): """Params that OpenAI rejects for all GPT-5 reasoning models.""" diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 8cc46dc98d0..c8751fb2d95 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -54,3 +54,61 @@ def test_ovhcloud_audio_transcription_config_installed(): assert config is not None assert isinstance(config, BaseAudioTranscriptionConfig) + + + +class TestOVHCloudDurationFieldMigration: + """Tests for OVHCloud duration -> seconds field migration.""" + + def test_seconds_field_mapped_to_duration(self): + """New `seconds` field should be normalized to `duration`.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "seconds": 3.14, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 3.14 + + def test_legacy_duration_field_still_works(self): + """Legacy `duration` field should still be accepted.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "duration": 2.71, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 2.71 + + + + def test_seconds_zero_mapped_to_duration(self): + """seconds=0.0 must not be treated as falsy and lost.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = {"text": "silence", "seconds": 0.0} + result = config.transform_audio_transcription_response(mock_response) + assert result._hidden_params["duration"] == 0.0 \ No newline at end of file diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index a1b3b31f786..40d57c76d02 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -292,3 +292,78 @@ def test_ovhcloud_with_custom_base_url(): if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +class TestOVHCloudReasoningFieldMigration: + """Tests for OVHCloud reasoning_content -> reasoning field migration.""" + + def test_streaming_new_reasoning_field(self): + """New `reasoning` field should be mapped to `reasoning_content`.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning": "Let me think...", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Let me think..." + + def test_streaming_legacy_reasoning_content_unchanged(self): + """Legacy `reasoning_content` field should pass through untouched.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning_content": "Already correct field.", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Already correct field." + + def test_streaming_both_fields_legacy_wins(self): + """When both fields present, existing `reasoning_content` is not overwritten.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "reasoning": "new field", + "reasoning_content": "legacy field", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" + + diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/test_litellm/llms/test_file_search_responses.py index 9943b456083..2f7ad3874fa 100644 --- a/tests/test_litellm/llms/test_file_search_responses.py +++ b/tests/test_litellm/llms/test_file_search_responses.py @@ -327,7 +327,7 @@ class TestFileSearchGuardInResponsesMain: class TestManagedFilesVectorStoreAccess: def _make_hook(self): """Return a ManagedFiles instance with prisma_client mocked.""" - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) @@ -471,7 +471,7 @@ class TestManagedFilesVectorStoreAccess: @pytest.mark.asyncio async def test_F6_non_responses_call_type_skipped(self): """Access check only runs for aresponses/responses call types.""" - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) from litellm.proxy._types import CallTypes @@ -499,7 +499,7 @@ class TestManagedFilesVectorStoreAccess: class TestGetVectorStoreIdsFromFileSearchTools: def _make_hook(self): - from enterprise.litellm_enterprise.proxy.hooks.managed_files import ( + from litellm_enterprise.proxy.hooks.managed_files import ( _PROXY_LiteLLMManagedFiles as ManagedFiles, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py index 89992c510f5..9f004318488 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py @@ -1135,3 +1135,69 @@ def test_validate_loopback_redirect_uri_rejects_malformed_cleanly(): with pytest.raises(HTTPException) as exc: validate_loopback_redirect_uri("http://[not-an-ip]/cb") assert exc.value.status_code == 400 + + +def _mock_request_with_base_url(base_url: str): + req = MagicMock() + req.base_url = base_url + req.headers = {} + return req + + +def test_validate_trusted_redirect_uri_accepts_same_origin(): + """UI OAuth flow: redirect_uri on the proxy's own origin is allowed.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + # Should not raise. + validate_trusted_redirect_uri( + req, "https://proxy.example.com/ui/mcp/oauth/callback" + ) + + +def test_validate_trusted_redirect_uri_accepts_loopback(): + """Native MCP client flow: loopback is still allowed.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + validate_trusted_redirect_uri(req, "http://127.0.0.1:3000/cb") + validate_trusted_redirect_uri(req, "http://localhost:3000/cb") + + +def test_validate_trusted_redirect_uri_rejects_external_origin(): + """An attacker-controlled origin must still be rejected.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "https://attacker.example.com/cb") + assert exc.value.status_code == 400 + + +def test_validate_trusted_redirect_uri_rejects_scheme_mismatch(): + """https→http (or vice versa) on the same host is not same-origin.""" + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "http://proxy.example.com/ui/callback") + assert exc.value.status_code == 400 + + +def test_validate_trusted_redirect_uri_rejects_fragment(): + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + + req = _mock_request_with_base_url("https://proxy.example.com/") + with pytest.raises(HTTPException) as exc: + validate_trusted_redirect_uri(req, "https://proxy.example.com/ui/cb#code=1") + assert exc.value.status_code == 400 diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 85d5d6ba466..581324d47d2 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -23,6 +23,20 @@ def mock_mcp_client_ip(): yield +def _mock_callback_request(base_url: str = "http://localhost:3000/"): + """Return a MagicMock Request for callback/authorize same-origin tests. + + The callback handler only uses ``request`` to compute the proxy's own + base URL via ``get_request_base_url`` (which reads ``request.base_url`` + and trusted ``X-Forwarded-*`` headers). A simple MagicMock with the + right attributes is sufficient. + """ + req = MagicMock() + req.base_url = base_url + req.headers = {} + return req + + @pytest.fixture def trust_xff(): """Force ``IPAddressUtils.is_request_from_trusted_proxy`` to True. @@ -1844,6 +1858,7 @@ async def test_oauth_callback_redirects_with_state(): # Call callback endpoint with code and state response = await callback( + request=_mock_callback_request(), code="test_authorization_code_12345", state="encrypted_state_value", ) @@ -1887,6 +1902,7 @@ async def test_oauth_callback_preserves_client_redirect_uri_query(): } response = await callback( + request=_mock_callback_request(), code="test_authorization_code_12345", state="encrypted_state_value", ) @@ -1917,6 +1933,7 @@ async def test_oauth_callback_handles_invalid_state(): # Call callback endpoint with invalid state response = await callback( + request=_mock_callback_request(), code="test_code", state="invalid_encrypted_state", ) @@ -1926,6 +1943,40 @@ async def test_oauth_callback_handles_invalid_state(): assert "Authentication incomplete" in response.body.decode() +@pytest.mark.asyncio +async def test_oauth_callback_accepts_same_origin_ui_redirect(): + """UI OAuth flow: the callback should redirect to the proxy's own UI + origin when the encrypted state carries a same-origin client_redirect_uri.""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + callback, + ) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.decode_state_hash" + ) as mock_decode: + mock_decode.return_value = { + "base_url": "https://proxy.example.com/ui/mcp/oauth/callback", + "original_state": "state-123", + "code_challenge": None, + "code_challenge_method": None, + "client_redirect_uri": "https://proxy.example.com/ui/mcp/oauth/callback", + } + + response = await callback( + request=_mock_callback_request(base_url="https://proxy.example.com/"), + code="auth-code-123", + state="encrypted_state", + ) + + assert response.status_code == 302 + assert ( + "https://proxy.example.com/ui/mcp/oauth/callback" + in response.headers["location"] + ) + assert "code=auth-code-123" in response.headers["location"] + assert "state=state-123" in response.headers["location"] + + @pytest.mark.asyncio async def test_oauth_authorize_includes_scopes_from_server_config(): """Test that authorize endpoint includes scopes from server configuration.""" @@ -2307,7 +2358,11 @@ async def test_callback_revalidates_loopback_on_decoded_base_url(): "client_redirect_uri": "https://attacker.example.com/cb", } with pytest.raises(HTTPException) as exc_info: - await callback(code="stolen_code", state="encrypted_stale_state") + await callback( + request=_mock_callback_request(), + code="stolen_code", + state="encrypted_stale_state", + ) assert exc_info.value.status_code == 400 @@ -2329,7 +2384,11 @@ async def test_callback_revalidates_loopback_on_decoded_client_redirect_uri(): "client_redirect_uri": "https://attacker.example.com/cb", } with pytest.raises(HTTPException) as exc_info: - await callback(code="stolen_code", state="encrypted_stale_state") + await callback( + request=_mock_callback_request(), + code="stolen_code", + state="encrypted_stale_state", + ) assert exc_info.value.status_code == 400 @@ -2349,7 +2408,11 @@ async def test_callback_rejects_state_missing_redirect_uri(): "code_challenge_method": None, } with pytest.raises(HTTPException) as exc_info: - await callback(code="code", state="encrypted_malformed_state") + await callback( + request=_mock_callback_request(), + code="code", + state="encrypted_malformed_state", + ) assert exc_info.value.status_code == 400 diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 11e9dbbdd57..b53420f0000 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -29,7 +29,11 @@ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, _deserialize_json_dict, ) -from litellm.proxy._types import LiteLLM_MCPServerTable, MCPTransport +from litellm.proxy._types import ( + LiteLLM_MCPServerTable, + MCPApprovalStatus, + MCPTransport, +) from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPOAuthMetadata, MCPServer @@ -3311,5 +3315,126 @@ class TestOAuthDiscoverySSRFGuard: mock_client.get.assert_not_called() +class TestApprovalStatusGate: + """ + Regression tests for GHSA-gm4g-h72v-jhc3. + + The runtime registry must only contain servers an admin has approved. + A non-admin can submit a pending stdio MCP server with an attacker-chosen + command/args; before this gate, an admin opening the per-row endpoint + triggered ``add_server`` + ``health_check_server``, which spawned the + attacker's process under the proxy. The data-layer gate in + ``add_server`` / ``update_server`` blocks pending and rejected rows + from entering the registry regardless of which caller passes them in. + """ + + def _make_server(self, server_id: str, approval_status): + return LiteLLM_MCPServerTable( + server_id=server_id, + alias=f"server_{server_id}", + description="test", + url=None, + transport=MCPTransport.stdio, + command="python", + args=["-c", "print('attacker payload')"], + env={}, + approval_status=approval_status, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + @pytest.mark.parametrize( + "approval_status,expect_in_registry", + [ + (MCPApprovalStatus.pending_review, False), + (MCPApprovalStatus.rejected, False), + (MCPApprovalStatus.active, True), + # Legacy rows: NULL predates the approval workflow; "approved" is + # a legacy alias for "active" still present in older deployments. + # Both must continue to load to match the DB-level filter in + # reload_servers_from_database(). + (None, True), + ("approved", True), + ], + ) + async def test_add_server_respects_approval_status( + self, approval_status, expect_in_registry + ): + manager = MCPServerManager() + server_id = f"sid-{approval_status}" + await manager.add_server(self._make_server(server_id, approval_status)) + assert (server_id in manager.registry) is expect_in_registry + + async def test_update_server_evicts_when_transitioned_away_from_active(self): + # An admin updates a previously-active server to rejected (or pending). + # The stale registry entry must be evicted so subsequent tool calls + # and health probes can't reach it. + manager = MCPServerManager() + await manager.add_server( + self._make_server("evict-me", MCPApprovalStatus.active) + ) + assert "evict-me" in manager.registry + + await manager.update_server( + self._make_server("evict-me", MCPApprovalStatus.rejected) + ) + assert "evict-me" not in manager.registry + + async def test_update_server_eviction_clears_openapi_routing_artifacts( + self, tmp_path + ): + """Rejecting a server must remove its OpenAPI tools and name mappings.""" + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + from litellm.proxy._experimental.mcp_server.utils import ( + add_server_prefix_to_name, + get_server_prefix, + ) + + manager = MCPServerManager() + await manager.add_server( + self._make_server("evict-openapi", MCPApprovalStatus.active) + ) + assert "evict-openapi" in manager.registry + + server = manager.registry["evict-openapi"] + server.spec_path = str(tmp_path / "unused.yaml") + prefix = get_server_prefix(server) + prefixed = add_server_prefix_to_name("demo_tool", prefix) + + async def _noop_handler(**kwargs): + return None + + global_mcp_tool_registry.register_tool( + name=prefixed, + description="demo", + input_schema={"type": "object"}, + handler=_noop_handler, + ) + manager.tool_name_to_mcp_server_name_mapping["demo_tool"] = prefix + manager.tool_name_to_mcp_server_name_mapping[prefixed] = prefix + + await manager.update_server( + self._make_server("evict-openapi", MCPApprovalStatus.rejected) + ) + + assert "evict-openapi" not in manager.registry + assert prefixed not in global_mcp_tool_registry.tools + assert "demo_tool" not in manager.tool_name_to_mcp_server_name_mapping + assert prefixed not in manager.tool_name_to_mcp_server_name_mapping + + async def test_update_server_noop_for_unregistered_pending(self): + # update_server called with a pending row that was never registered + # should silently return without adding it. Locks in the early-return + # so a future refactor can't accidentally route the pending row to + # build_mcp_server_from_table. + manager = MCPServerManager() + await manager.update_server( + self._make_server("never-seen", MCPApprovalStatus.pending_review) + ) + assert "never-seen" not in manager.registry + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 957dea22f3c..39f3c767220 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -15,6 +15,8 @@ from unittest.mock import AsyncMock, patch import pytest from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + _request_extra_headers, _resolve_param_list, _resolve_ref, build_input_schema, @@ -1011,3 +1013,197 @@ class TestRegisterToolsFromOpenAPI: assert re.match( r"^[a-zA-Z0-9_-]+$", name ), f"fallback tool name {name!r} not sanitized" + + +class TestRequestExtraHeaders: + """Tests for _request_extra_headers ContextVar forwarding in tool_function.""" + + @pytest.mark.asyncio + async def test_extra_headers_forwarded_to_upstream(self): + """Extra headers set via ContextVar are included in the upstream request.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "secret-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-TOKEN") == "secret-value" + + @pytest.mark.asyncio + async def test_no_extra_headers_by_default(self): + """Without setting _request_extra_headers, no extra headers are injected.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + result = await func() + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent == {"X-Static": "static-value"} + assert "X-TOKEN" not in headers_sent + + @pytest.mark.asyncio + async def test_extra_headers_merged_with_static_headers(self): + """Forwarded headers are passed through alongside non-conflicting static headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="post", + operation=operation, + base_url="https://api.example.com", + headers={"X-Static": "static-value"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("post", "created") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "dynamic-value"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "created" + call_args = async_client.post.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Static") == "static-value" + assert headers_sent.get("X-TOKEN") == "dynamic-value" + + @pytest.mark.asyncio + async def test_static_headers_win_over_forwarded_on_conflict(self): + """Static (operator) headers must override forwarded (caller) headers on name conflict.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-Tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_static_headers_win_case_insensitively(self): + """Forwarded header with different casing must not bypass the static-wins rule.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + headers={"X-Tenant": "operator-tenant"}, + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"x-tenant": "caller-spoofed"}) + try: + result = await func() + finally: + _request_extra_headers.reset(token) + + assert result == "ok" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("X-Tenant") == "operator-tenant" + assert "x-tenant" not in headers_sent + assert "caller-spoofed" not in headers_sent.values() + + @pytest.mark.asyncio + async def test_auth_header_still_overrides_extra_headers(self): + """_request_auth_header takes precedence for Authorization over extra headers.""" + operation = {} + func = create_tool_function( + path="/secure", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "secure-data") + mock_client.return_value = async_client + + extra_token = _request_extra_headers.set( + {"Authorization": "Bearer extra", "X-TOKEN": "token-value"} + ) + auth_token = _request_auth_header.set("Bearer byok-credential") + try: + result = await func() + finally: + _request_auth_header.reset(auth_token) + _request_extra_headers.reset(extra_token) + + assert result == "secure-data" + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert headers_sent.get("Authorization") == "Bearer byok-credential" + assert headers_sent.get("X-TOKEN") == "token-value" + + @pytest.mark.asyncio + async def test_extra_headers_not_leaked_between_calls(self): + """After resetting the ContextVar, subsequent calls do not see the headers.""" + operation = {} + func = create_tool_function( + path="/data", + method="get", + operation=operation, + base_url="https://api.example.com", + ) + + with patch(GET_ASYNC_CLIENT_TARGET) as mock_client: + async_client = _create_mock_client("get", "ok") + mock_client.return_value = async_client + + token = _request_extra_headers.set({"X-TOKEN": "first-call"}) + _request_extra_headers.reset(token) + + await func() + + call_args = async_client.get.call_args + headers_sent = call_args[1]["headers"] + assert "X-TOKEN" not in headers_sent diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 8a854bcd6a8..26f04a4abcb 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta import httpx import pytest +from fastapi import status import litellm from litellm.proxy._types import ( @@ -31,6 +32,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, + _can_object_call_model, _can_object_call_vector_stores, _check_end_user_budget, _check_team_member_budget, @@ -206,6 +208,52 @@ def test_get_key_object_from_ui_hash_key_invalid(): assert key_object is None +@pytest.mark.parametrize( + "object_type,expected_error_type", + [ + ("key", ProxyErrorTypes.key_model_access_denied), + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ("project", ProxyErrorTypes.project_model_access_denied), + ], +) +def test_can_object_call_model_denials_return_forbidden( + object_type, expected_error_type +): + with pytest.raises(ProxyException) as exc_info: + _can_object_call_model( + model="restricted-model", + llm_router=None, + models=["allowed-model"], + object_type=object_type, + ) + + assert exc_info.value.type == expected_error_type + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + +@pytest.mark.asyncio +async def test_can_user_call_model_no_default_models_returns_forbidden(): + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import can_user_call_model + + user_object = LiteLLM_UserTable( + user_id="test-user", + models=[SpecialModelNames.no_default_models.value], + ) + + with pytest.raises(ProxyException) as exc_info: + await can_user_call_model( + model="restricted-model", + llm_router=None, + user_object=user_object, + ) + + assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN + + @pytest.mark.asyncio async def test_get_key_object_should_reconnect_once_on_db_connection_error(): mock_prisma_client = MagicMock() @@ -1144,6 +1192,7 @@ async def test_check_team_member_model_access_denied_model(): proxy_logging_obj=MagicMock(), ) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 2d1586f0b17..4ccde85dae2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -140,6 +140,7 @@ async def test_handle_authentication_error_budget_exceeded(): ) assert exc_info.value.type == ProxyErrorTypes.budget_exceeded + assert int(exc_info.value.code) == status.HTTP_429_TOO_MANY_REQUESTS @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index b7dba9c1d16..90c5d4f4fc5 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -1,5 +1,5 @@ from typing import Optional -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -494,6 +494,80 @@ async def test_sync_user_role_and_teams_no_cache_write_when_nothing_changes(): mock_cache.async_set_cache.assert_not_called() +def test_get_all_jwt_team_ids_unions_singular_and_plural(): + """get_all_jwt_team_ids must include the singular team_id_jwt_field claim + in addition to the plural team_ids_jwt_field, deduplicated.""" + jwt_handler = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_id_jwt_field="team_id", + team_ids_jwt_field="teams", + ), + ) + + # singular only — Okta/Auth0 default shape + assert jwt_handler.get_all_jwt_team_ids({"team_id": "team-low"}) == ["team-low"] + + # plural only — pre-fix shape + assert jwt_handler.get_all_jwt_team_ids({"teams": ["a", "b"]}) == ["a", "b"] + + # both populated, no overlap + assert jwt_handler.get_all_jwt_team_ids( + {"team_id": "primary", "teams": ["a", "b"]} + ) == ["a", "b", "primary"] + + # both populated with overlap — singular dedup'd + assert jwt_handler.get_all_jwt_team_ids({"team_id": "a", "teams": ["a", "b"]}) == [ + "a", + "b", + ] + + # singular field as multi-element list (some IdPs) — merge all, preserve plural-first order + assert jwt_handler.get_all_jwt_team_ids( + {"team_id": ["primary", "secondary"], "teams": ["a"]} + ) == ["a", "primary", "secondary"] + + # neither populated + assert jwt_handler.get_all_jwt_team_ids({}) == [] + + +def test_get_all_jwt_team_ids_does_not_use_team_id_default(): + """team_id_default is a JWT-bearer-flow auth-builder fallback, not a token + claim. It must NOT leak into get_all_jwt_team_ids — otherwise SSO logins + would silently start adding users to the default team for any tenant that + has team_id_default configured.""" + jwt_handler = JWTHandler() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_id_jwt_field="team_id", + team_ids_jwt_field="teams", + team_id_default="default-team", + ), + ) + + # team_id claim missing — must not fall back to default-team + assert jwt_handler.get_all_jwt_team_ids({"teams": []}) == [] + assert jwt_handler.get_all_jwt_team_ids({}) == [] + + # only the plural is populated — default still must not be added + assert jwt_handler.get_all_jwt_team_ids({"teams": ["a"]}) == ["a"] + + # team_id_jwt_field unset entirely + only default configured: still no default + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=MagicMock(), + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field="teams", + team_id_default="default-team", + ), + ) + assert jwt_handler.get_all_jwt_team_ids({"teams": []}) == [] + + @pytest.mark.asyncio async def test_map_jwt_role_to_litellm_role(): """Test JWT role mapping to LiteLLM roles with various patterns""" diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 268cdff1f7d..fa56383e9bb 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -53,14 +53,20 @@ def test_non_admin_config_update_route_rejected(): assert "Your role=internal_user" in str(exc_info.value) +@pytest.mark.parametrize( + "role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) @pytest.mark.parametrize( "route", ["/compliance/eu-ai-act", "/compliance/gdpr"], ) -def test_compliance_routes_open_to_internal_user(route): +def test_compliance_routes_open_to_non_admin_roles(role, route): """Compliance routes are stateless validators on caller-supplied log data - - non-admin internal_user roles can call them.""" - role = LitellmUserRoles.INTERNAL_USER.value + — both non-admin internal_user roles can call them.""" user_obj = LiteLLM_UserTable( user_id="test_user", user_email="test@example.com", @@ -80,34 +86,6 @@ def test_compliance_routes_open_to_internal_user(route): ) -@pytest.mark.parametrize( - "route", - ["/compliance/eu-ai-act", "/compliance/gdpr"], -) -def test_compliance_routes_blocked_for_internal_user_view_only(route): - """Deprecated internal_user_viewer role must not gain compliance route access.""" - role = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value - user_obj = LiteLLM_UserTable( - user_id="test_user", - user_email="test@example.com", - user_role=role, - ) - valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) - request = MagicMock(spec=Request) - request.query_params = {} - - with pytest.raises(Exception) as exc_info: - RouteChecks.non_proxy_admin_allowed_routes_check( - user_obj=user_obj, - _user_role=role, - route=route, - request=request, - valid_token=valid_token, - request_data={}, - ) - assert "Only proxy admin can be used" in str(exc_info.value) - - def test_proxy_admin_viewer_config_update_route_rejected(): """Test that proxy admin viewer users are rejected when trying to call /config/update""" @@ -1911,6 +1889,41 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re assert "Only proxy admin can be used to generate" in str(exc_info.value) +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +@pytest.mark.parametrize("route", ["/tag/list", "/tag/daily/activity"]) +def test_internal_users_can_access_scoped_tag_usage_routes(user_role, route): + """ + Internal users can read tag usage endpoints because the endpoint handlers + scope results to the caller's own keys. + """ + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=user_role, + ) + valid_token = UserAPIKeyAuth( + user_id="test_user", + user_role=user_role, + ) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "user_role", [ diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 95b3d746c66..442625c75a7 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,6 +1,7 @@ import json import os import sys +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -9,6 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest +from fastapi import status import litellm import litellm.proxy.proxy_server @@ -29,8 +31,10 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( - _route_requires_auth_despite_public, + _matches_routing_override, _reserve_budget_after_common_checks, + _route_requires_auth_despite_public, + _routing_selector_matches_claim, _run_centralized_common_checks, _run_post_custom_auth_checks, get_api_key, @@ -178,6 +182,26 @@ async def test_custom_auth_does_not_enforce_key_model_access_by_default(): mock_can_key.assert_not_awaited() +@pytest.mark.asyncio +async def test_post_custom_auth_expired_key_returns_unauthorized(): + expired_token = UserAPIKeyAuth( + token="test_token", + expires=datetime.now() - timedelta(minutes=1), + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_post_custom_auth_checks( + valid_token=expired_token, + request=MagicMock(), + request_data={}, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc_info.value.type == ProxyErrorTypes.expired_key + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) @@ -572,6 +596,151 @@ def _assert_get_api_key_with_custom_litellm_key_header( ) == (api_key, passed_in_key) +@pytest.mark.parametrize( + "selector_value, claim_value, expected, split_space_delimited", + [ + (None, "any-value", True, False), + ("issuer.example.com", "issuer.example.com", True, False), + ("issuer.example.com", "other-issuer.example.com", False, False), + # iss (and other non-scope claims) must not match via space-split injection + ( + "trusted.example.com", + "trusted.example.com attacker.example.com", + False, + False, + ), + # Wildcard iss must not match space-containing claim strings (fnmatch * spans spaces) + ( + "trusted.*", + "trusted.example.com attacker.example.com", + False, + False, + ), + ("trusted.*", "trusted.example.com", True, False), + ( + ["issuer-a.example.com", "issuer-b.example.com"], + "issuer-b.example.com", + True, + False, + ), + ("*MID_LITELLM", "STREAM_MID_LITELLM", True, False), + ("*MID_LITELLM", "REDIS_LITELLM", False, False), + ("machine-??", "machine-01", True, False), + ("machine-??", "machine-001", False, False), + # Wildcard matching is case-sensitive (fnmatch.fnmatchcase) + ("*litellm", "BATCH_LITELLM", False, False), + ("*LITELLM", "BATCH_LITELLM", True, False), + ("App:LiteLLM", "App:LiteLLM openid", True, True), + ("App:*", "App:LiteLLM openid", True, True), + (["openid", "App:LiteLLM"], "openid profile", True, True), + (["service-*", "batch-*"], "batch-123", True, False), + (["service-*", "batch-*"], "other-123", False, False), + ("App:LiteLLM", ["openid", "App:LiteLLM"], True, False), + ("App:LiteLLM", None, False, False), + ], +) +def test_routing_selector_matches_claim_parametrized( + selector_value, claim_value, expected, split_space_delimited +): + assert ( + _routing_selector_matches_claim( + selector_value=selector_value, + claim_value=claim_value, + split_space_delimited=split_space_delimited, + ) + is expected + ) + + +@pytest.mark.parametrize( + "override, token_claims, expected", + [ + # Only iss selector is required and should match. + ( + JWTRoutingOverride(iss="oauth-issuer.example.com", path="oauth2"), + {"iss": "oauth-issuer.example.com"}, + True, + ), + # Scope selector narrows the match. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "scope": "App:LiteLLM openid"}, + True, + ), + # client_id wildcard selector narrows the match. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + client_id="*MID_LITELLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "client_id": "BATCH_MID_LITELLM"}, + True, + ), + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + client_id="*MID_LITELLM", + path="oauth2", + ), + {"iss": "oauth-issuer.example.com", "client_id": "BATCH_PORTAL"}, + False, + ), + # aud selector still works with list claims. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + aud=["api://litellm", "api://fallback"], + path="oauth2", + ), + { + "iss": "oauth-issuer.example.com", + "aud": ["api://other", "api://litellm"], + }, + True, + ), + # All provided selectors are AND-ed. + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ), + { + "iss": "oauth-issuer.example.com", + "scope": "App:LiteLLM openid", + "client_id": "BATCH_MID_LITELLM", + }, + True, + ), + ( + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ), + { + "iss": "oauth-issuer.example.com", + "scope": "App:Other openid", + "client_id": "BATCH_MID_LITELLM", + }, + False, + ), + ], +) +def test_matches_routing_override_parametrized(override, token_claims, expected): + assert ( + _matches_routing_override(token_claims=token_claims, override=override) + is expected + ) + + def test_get_api_key_with_custom_litellm_key_header_bearer_prefix(): token = "sk-" + "1" * 8 header = f"Bearer {token}" @@ -934,6 +1103,7 @@ async def test_proxy_admin_expired_key_from_cache(): assert ( exc_info.value.type == ProxyErrorTypes.expired_key ), f"Expected expired_key error type, got {exc_info.value.type}" + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED assert "Expired Key" in str( exc_info.value.message ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" @@ -1578,6 +1748,206 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_not_called() assert result.user_id == "machine-client-aud-list" + @pytest.mark.asyncio + async def test_routing_override_matches_scope_claim(self): + """ + Match routing override when scope selector is configured and scope claim matches. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpMaXRlTExNIiwiY2xpZW50X2lkIjoiTUFDSElORV9NSURfTElURUxMTSJ9." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-scope-match", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth, + ): + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-scope-match" + + @pytest.mark.asyncio + async def test_routing_override_scope_mismatch_falls_back_to_jwt(self): + """ + If scope selector does not match, continue default JWT flow. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpPdGhlciIsImNsaWVudF9pZCI6IlBPUlRBTF9NSURfTElURUxMTSJ9." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": "jwt-team", + "user_id": "jwt-user-scope-mismatch", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth, + ): + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_not_called() + mock_jwt_auth.assert_called_once() + assert result.user_id == "jwt-user-scope-mismatch" + + @pytest.mark.asyncio + async def test_routing_override_matches_scope_and_client_wildcard_when_scope_claim_is_space_delimited( + self, + ): + """ + Integration check: combined scope + wildcard selectors match on OAuth2 path + when scope claim is a space-delimited string. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJvYXV0aC1pc3N1ZXIuZXhhbXBsZS5jb20iLCJzY29wZSI6IkFwcDpMaXRlTExNIG9wZW5pZCIsImNsaWVudF9pZCI6IkJBVENIX01JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-space-delimited-scope-match", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with ( + patch("litellm.proxy.proxy_server.general_settings", general_settings), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, + patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth, + ): + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="oauth-issuer.example.com", + scope="App:LiteLLM", + client_id="*MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-space-delimited-scope-match" + @pytest.mark.asyncio async def test_routing_override_routes_jwt_to_oauth2_when_oauth2_globally_disabled( self, diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 5c86f9057a1..8a47c78db05 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -39,6 +39,46 @@ class MockLiteLLMVerificationToken: return {"count": 1} +class MockLiteLLMOrganizationTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + +class MockLiteLLMTagTable: + def __init__(self): + self.update_many_calls: List[Dict[str, Any]] = [] + self.find_many_calls: List[Dict[str, Any]] = [] + self._find_many_results: List[Any] = [] + + def set_find_many_results(self, results: List[Any]): + self._find_many_results = results + + async def find_many(self, where: Dict[str, Any]) -> List[Any]: + self.find_many_calls.append({"where": where}) + return self._find_many_results + + async def update_many( + self, where: Dict[str, Any], data: Dict[str, Any] + ) -> Dict[str, Any]: + self.update_many_calls.append({"where": where, "data": data}) + return {"count": 1} + + class MockLiteLLMEndUserTable: def __init__(self): self.find_many_calls: List[Dict[str, Any]] = [] @@ -57,6 +97,8 @@ class MockDB: self.litellm_teammembership = MockLiteLLMTeamMembership() self.litellm_verificationtoken = MockLiteLLMVerificationToken() self.litellm_endusertable = MockLiteLLMEndUserTable() + self.litellm_organizationtable = MockLiteLLMOrganizationTable() + self.litellm_tagtable = MockLiteLLMTagTable() class MockPrismaClient: @@ -459,6 +501,100 @@ def test_reset_budget_for_keys_linked_to_budgets_empty( assert len(calls) == 0 +def test_reset_budget_for_orgs_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, orgs linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_orgs_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the organization table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_orgs_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 0 + + +def test_reset_budget_for_tags_linked_to_budgets(reset_budget_job, mock_prisma_client): + """ + Test that when a budget tier is reset, tags linked to that budget + (via budget_id) also get their spend reset. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets( + budgets_to_reset=[test_budget] + ) + ) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1 + call = calls[0] + assert call["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert call["where"]["spend"] == {"gt": 0} + assert call["data"]["spend"] == 0 + + +def test_reset_budget_for_tags_linked_to_budgets_empty( + reset_budget_job, mock_prisma_client +): + """ + Test that when there are no budgets to reset, no update is performed + on the tag table. + """ + asyncio.run( + reset_budget_job.reset_budget_for_tags_linked_to_budgets(budgets_to_reset=[]) + ) + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 0 + + @pytest.mark.parametrize( "budget_duration, expected_day, expected_month", [ @@ -618,6 +754,75 @@ def test_budget_table_reset_also_resets_linked_keys( assert calls[0]["data"]["spend"] == 0 +def test_budget_table_reset_also_resets_linked_orgs( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for orgs linked to the expiring budget tiers + (in addition to end-users, team members, and keys). + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-org-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_organizationtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset orgs " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-org-budget"]} + assert calls[0]["data"]["spend"] == 0 + + +def test_budget_table_reset_also_resets_linked_tags( + reset_budget_job, mock_prisma_client +): + """ + Integration-style test: when reset_budget_for_litellm_budget_table runs, + it should also reset spend for tags linked to the expiring budget tiers. + """ + now = datetime.now(timezone.utc) + + test_budget = type( + "LiteLLM_BudgetTableFull", + (), + { + "max_budget": 50.0, + "budget_duration": "30d", + "budget_reset_at": now - timedelta(hours=1), + "budget_id": "30d-tag-budget", + "created_at": now - timedelta(days=30), + }, + ) + + mock_prisma_client.data["budget"] = [test_budget] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + calls = mock_prisma_client.db.litellm_tagtable.update_many_calls + assert len(calls) == 1, ( + "Expected reset_budget_for_litellm_budget_table to also reset tags " + f"linked to expiring budgets, but got {len(calls)} update_many calls" + ) + assert calls[0]["where"]["budget_id"] == {"in": ["30d-tag-budget"]} + assert calls[0]["data"]["spend"] == 0 + + def test_reset_budget_resets_endusers_with_null_budget_id( reset_budget_job, mock_prisma_client ): @@ -1057,16 +1262,26 @@ def test_reset_budget_windows_query_error_does_not_break_team_path(monkeypatch): def _make_counter_invalidation_job(monkeypatch): - """Stub spend_counter_cache so we can observe invalidation calls.""" + """Stub spend_counter_cache (and user_api_key_cache) so we can observe + invalidation calls. + + Both caches are looked up via ``from litellm.proxy.proxy_server import + `` inside the reset job, so we publish them on a fake module. + """ spend_counter_cache = MagicMock() spend_counter_cache.in_memory_cache.set_cache = MagicMock() spend_counter_cache.redis_cache = MagicMock() spend_counter_cache.redis_cache.async_set_cache = AsyncMock() + user_api_key_cache = MagicMock() + user_api_key_cache.async_delete_cache = AsyncMock() + fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.user_api_key_cache = user_api_key_cache monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) + spend_counter_cache.user_api_key_cache = user_api_key_cache return spend_counter_cache @@ -1205,3 +1420,223 @@ def test_reset_budget_for_keys_linked_to_budgets_invalidates_redis_counter(monke counter_cache.in_memory_cache.set_cache.assert_any_call( key="spend:key:sk-linked", value=0.0, ttl=60 ) + + +def test_reset_budget_for_orgs_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting orgs via budget tier must clear each linked org's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_org = type("Org", (), {"organization_id": "org-acme"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_organizationtable.find_many = AsyncMock( + return_value=[linked_org] + ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:org:org-acme", value=0.0, ttl=60 + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_redis_counter(monkeypatch): + """Resetting tags via budget tier must clear each linked tag's counter.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + counter_cache.in_memory_cache.set_cache.assert_any_call( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) + counter_cache.redis_cache.async_set_cache.assert_any_await( + key="spend:tag:tenant-42", value=0.0, ttl=60 + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_management_cache( + monkeypatch, +): + """Regression guard for the bug where tag spend stayed frozen across cycles. + + ``SpendCounterReseed.from_db`` returns ``None`` for ``spend:tag:*`` keys, + so once the spend counter expires the tag budget check falls back to the + cached ``LiteLLM_TagTable.spend``. If we don't drop the management cache + entry on reset, that cached object lingers (TTL 60s) with the pre-reset + spend, and ``_tag_max_budget_check`` keeps returning HTTP 400 even though + the DB row has been zeroed. + """ + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="tag:tenant-42" + ) + + +def test_reset_budget_for_tags_linked_to_budgets_invalidates_each_tag_management_cache( + monkeypatch, +): + """When multiple tags share the expired budget tier, every one of them + has its ``user_api_key_cache`` entry dropped — not just the first.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tags = [ + type("Tag", (), {"tag_name": "tenant-a"}), + type("Tag", (), {"tag_name": "tenant-b"}), + type("Tag", (), {"tag_name": "tenant-c"}), + ] + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=linked_tags) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 3}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + deleted_keys = { + call.kwargs.get("key") + for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + } + assert deleted_keys == {"tag:tenant-a", "tag:tenant-b", "tag:tenant-c"} + + +def test_reset_budget_for_keys_linked_to_budgets_invalidates_management_cache( + monkeypatch, +): + """Budget-tier key resets must drop the cached key object (hashed token key). + + Historically this test used ``assert_not_awaited()`` on + ``user_api_key_cache.async_delete_cache``, reflecting the assumption that + ``SpendCounterReseed.from_db`` alone kept spend consistent for keys and + that invalidating the management cache was unnecessary. That was flipped to + ``assert_any_await(...)`` because the old invariant fails across pods: a + budget reset on one instance can leave another pod's cached key object + (including embedded ``.spend``) stale until TTL expiry. Eviction now matches + tags/orgs/teams. Do not treat the ``cache_key_fn`` / invalidation wiring as + redundant without revisiting that cross-pod consistency story. + """ + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_key = type("Key", (), {"token": "sk-linked"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[linked_key] + ) + prisma_client.db.litellm_verificationtoken.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_keys_linked_to_budgets([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="sk-linked" + ) + + +def test_reset_budget_for_orgs_linked_to_budgets_invalidates_management_cache( + monkeypatch, +): + """Org rows use both base and budget-table cache keys — evict both on reset.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_org = type("Org", (), {"organization_id": "org-acme"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_organizationtable.find_many = AsyncMock( + return_value=[linked_org] + ) + prisma_client.db.litellm_organizationtable.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_orgs_linked_to_budgets([expired_budget])) + + deleted_keys = { + call.kwargs.get("key") + for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list + } + assert deleted_keys == { + "org_id:org-acme", + "org_id:org-acme:with_budget", + } + + +def test_reset_budget_for_team_members_invalidates_management_cache(monkeypatch): + """Team membership cache key matches auth: ``{team_id}_{user_id}``.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + membership = type( + "Membership", + (), + {"user_id": "alice", "team_id": "team-x", "budget_id": "budget-1"}, + ) + + prisma_client = MagicMock() + prisma_client.db.litellm_teammembership.find_many = AsyncMock( + return_value=[membership] + ) + prisma_client.db.litellm_teammembership.update_many = AsyncMock( + return_value={"count": 1} + ) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_litellm_team_members([expired_budget])) + + counter_cache.user_api_key_cache.async_delete_cache.assert_any_await( + key="team-x_alice" + ) + + +def test_reset_budget_for_tags_linked_to_budgets_management_cache_delete_failure_still_resets( + monkeypatch, +): + """If ``async_delete_cache`` raises, the DB cascade must still complete.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + counter_cache.user_api_key_cache.async_delete_cache = AsyncMock( + side_effect=RuntimeError("cache unavailable") + ) + + expired_budget = type("B", (), {"budget_id": "budget-1"}) + linked_tag = type("Tag", (), {"tag_name": "tenant-42"}) + + prisma_client = MagicMock() + prisma_client.db.litellm_tagtable.find_many = AsyncMock(return_value=[linked_tag]) + prisma_client.db.litellm_tagtable.update_many = AsyncMock(return_value={"count": 1}) + + job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) + asyncio.run(job.reset_budget_for_tags_linked_to_budgets([expired_budget])) + + prisma_client.db.litellm_tagtable.update_many.assert_awaited_once() diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index c6017752814..79e6494eab0 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1513,3 +1513,146 @@ async def test_commit_spend_updates_uses_pipeline(): mock_redis_update_buffer.get_all_daily_end_user_spend_update_transactions_from_redis_buffer.assert_not_called() mock_redis_update_buffer.get_all_daily_agent_spend_update_transactions_from_redis_buffer.assert_not_called() mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called() + + +@pytest.mark.parametrize( + "bucket_name,input_dict,table_attr,method_name,where_key,expected_order", + [ + pytest.param( + "user_list_transactions", + {"user_c": 0.1, "user_a": 0.2, "user_b": 0.3}, + "litellm_usertable", + "update_many", + "user_id", + ["user_a", "user_b", "user_c"], + id="user", + ), + pytest.param( + "key_list_transactions", + {"tok_c": 0.1, "tok_a": 0.2, "tok_b": 0.3}, + "litellm_verificationtoken", + "update_many", + "token", + ["tok_a", "tok_b", "tok_c"], + id="key", + ), + pytest.param( + "team_list_transactions", + {"team_c": 0.1, "team_a": 0.2, "team_b": 0.3}, + "litellm_teamtable", + "update_many", + "team_id", + ["team_a", "team_b", "team_c"], + id="team", + ), + pytest.param( + "team_member_list_transactions", + { + "team_id::team_c::user_id::user_x": 0.1, + "team_id::team_a::user_id::user_x": 0.2, + "team_id::team_b::user_id::user_x": 0.3, + }, + "litellm_teammembership", + "update_many", + "team_id", + ["team_a", "team_b", "team_c"], + id="team_member", + ), + pytest.param( + "org_list_transactions", + {"org_c": 0.1, "org_a": 0.2, "org_b": 0.3}, + "litellm_organizationtable", + "update_many", + "organization_id", + ["org_a", "org_b", "org_c"], + id="org", + ), + pytest.param( + "end_user_list_transactions", + {"eu_c": 0.1, "eu_a": 0.2, "eu_b": 0.3}, + "litellm_endusertable", + "upsert", + "user_id", + ["eu_a", "eu_b", "eu_c"], + id="end_user", + ), + pytest.param( + "tag_list_transactions", + {"prod": 0.1, "customer-x": 0.2, "test": 0.3}, + "litellm_tagtable", + "update_many", + "tag_name", + ["customer-x", "prod", "test"], + id="tag", + ), + pytest.param( + "agent_list_transactions", + {"agent_c": 0.1, "agent_a": 0.2, "agent_b": 0.3}, + "litellm_agentstable", + "update_many", + "agent_id", + ["agent_a", "agent_b", "agent_c"], + id="agent", + ), + ], +) +@pytest.mark.asyncio +async def test_commit_spend_updates_iterates_in_sorted_order( + bucket_name, input_dict, table_attr, method_name, where_key, expected_order +): + """ + Every spend-bucket code path in _commit_spend_updates_to_db must iterate + in sorted order so concurrent pods acquire row locks in the same order + and avoid PostgreSQL deadlocks. Covers the 5 direct loops (user/key/team/ + team_member/org), the end_user path in ProxyUpdateSpend.update_end_user_spend, + and the shared _update_entity_spend_in_db helper (tag, agent). + """ + db_writer = DBSpendUpdateWriter() + + captured_where_values = [] + + def capture(*, where, data): + captured_where_values.append(where[where_key]) + + mock_batcher = MagicMock() + table_mock = MagicMock() + setattr(table_mock, method_name, MagicMock(side_effect=capture)) + setattr(mock_batcher, table_attr, table_mock) + + mock_transaction = AsyncMock() + mock_transaction.__aenter__ = AsyncMock(return_value=mock_transaction) + mock_transaction.__aexit__ = AsyncMock(return_value=False) + mock_transaction.batch_ = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_batcher), + __aexit__=AsyncMock(return_value=False), + ) + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.tx = MagicMock(return_value=mock_transaction) + + mock_proxy_logging = MagicMock() + mock_proxy_logging.call_details = {} + + buckets = { + "user_list_transactions": {}, + "end_user_list_transactions": {}, + "key_list_transactions": {}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + buckets[bucket_name] = input_dict + + await db_writer._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=mock_proxy_logging, + db_spend_update_transactions=buckets, + ) + + assert captured_where_values == expected_order diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py new file mode 100644 index 00000000000..8c3a2b9e2d7 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -0,0 +1,887 @@ +import asyncio +import logging +import os +import sys +from typing import Any, Dict +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + + +# NOTE: do NOT patch sys.modules["prisma"] file-wide via an autouse fixture. +# Doing so leaks across pytest-xdist test scheduling: when a worker runs a +# routing test, then later runs test_exception_handler.py, the cached MagicMock +# attribute references break `isinstance(e, prisma.errors.X)` in +# `is_database_transport_error`. The two tests below that actually need to +# stub the prisma SDK do so per-test via monkeypatch, which is properly scoped. + + +def _make_wrappers(): + from litellm.proxy.db.prisma_client import PrismaWrapper + + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + return writer, writer_inner, reader, reader_inner + + +class _FakeActions: + """Stand-in for a Prisma per-model Actions instance (non-callable, has find_many/create).""" + + def __init__(self, name: str): + self._name = name + for method in ( + "find_many", + "find_unique", + "find_first", + "count", + "group_by", + "create", + "update", + "upsert", + "delete", + "delete_many", + "update_many", + ): + setattr(self, method, MagicMock(name=f"{name}.{method}")) + + +def _model_actions_mock(name: str) -> _FakeActions: + return _FakeActions(name) + + +def test_top_level_query_raw_routes_to_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # query_raw should resolve to the reader's underlying client. + assert routing.query_raw is reader_inner.query_raw + assert routing.query_first is reader_inner.query_first + + +def test_top_level_execute_raw_routes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # execute_raw, batch_, tx are write-side and must hit the writer. + assert routing.execute_raw is writer_inner.execute_raw + assert routing.batch_ is writer_inner.batch_ + assert routing.tx is writer_inner.tx + + +def test_per_model_reads_route_to_reader_writes_to_writer(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + + # Reads → reader actions. + assert actions.find_many is reader_inner.litellm_usertable.find_many + assert actions.find_unique is reader_inner.litellm_usertable.find_unique + assert actions.find_first is reader_inner.litellm_usertable.find_first + assert actions.count is reader_inner.litellm_usertable.count + assert actions.group_by is reader_inner.litellm_usertable.group_by + + # Writes → writer actions. + assert actions.create is writer_inner.litellm_usertable.create + assert actions.update is writer_inner.litellm_usertable.update + assert actions.upsert is writer_inner.litellm_usertable.upsert + assert actions.delete is writer_inner.litellm_usertable.delete + assert actions.update_many is writer_inner.litellm_usertable.update_many + assert actions.delete_many is writer_inner.litellm_usertable.delete_many + + +@pytest.mark.asyncio +async def test_connect_invokes_both_clients(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + await routing.connect() + + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_logs_writer_and_reader_success(caplog): + """Successful startup emits a positive INFO confirmation for both writer + and reader so operators can verify connectivity without inspecting the URL + in logs.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await routing.connect() + + messages = [r.getMessage() for r in caplog.records] + assert "[writer] DB connected" in messages + assert "[reader] DB connected" in messages + + +@pytest.mark.asyncio +async def test_disconnect_continues_when_one_side_fails(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.disconnect = AsyncMock(side_effect=RuntimeError("writer down")) + reader_inner.disconnect = AsyncMock() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with pytest.raises(RuntimeError, match="writer down"): + await routing.disconnect() + + # Reader still attempted even though writer raised. + reader_inner.disconnect.assert_awaited_once() + + +def test_is_connected_reflects_writer_only(): + """is_connected() must NOT depend on reader health — a healthy writer with + a degraded reader should report True so that PrismaClient.connect()'s + health check does not re-trigger a writer reconnect (which only fixes + writer-side problems and would loop indefinitely).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + writer_inner.is_connected = MagicMock(return_value=True) + reader_inner.is_connected = MagicMock(return_value=True) + assert routing.is_connected() is True + + # Reader down → still True (reader degradation is tracked separately). + reader_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is True + + # Writer down → False. + writer_inner.is_connected = MagicMock(return_value=False) + assert routing.is_connected() is False + + +def test_token_refresh_delegates_to_both_writer_and_reader(): + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.start_token_refresh_task = AsyncMock() + writer.stop_token_refresh_task = AsyncMock() + reader = MagicMock() + reader.start_token_refresh_task = AsyncMock() + reader.stop_token_refresh_task = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + asyncio.run(routing.start_token_refresh_task()) + asyncio.run(routing.stop_token_refresh_task()) + + # Both wrappers get start/stop — each manages its own IAM token. When + # IAM is disabled on a wrapper its task body is a no-op. + writer.start_token_refresh_task.assert_awaited_once() + writer.stop_token_refresh_task.assert_awaited_once() + reader.start_token_refresh_task.assert_awaited_once() + reader.stop_token_refresh_task.assert_awaited_once() + + +def test_routed_actions_falls_back_to_writer_for_unknown_methods(): + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + writer_actions.some_custom_method = "writer-custom" + reader_actions = _model_actions_mock("reader") + reader_actions.some_custom_method = "reader-custom" + + routed = _RoutedActions(writer_actions, reader_actions, lambda: True) + # Unknown method → defaults to writer (safe fallback for write-like ops). + assert routed.some_custom_method == "writer-custom" + + +def test_routed_actions_respects_should_use_reader_flag(): + """When the routing wrapper marks the reader unavailable, _RoutedActions + must redirect reads to the writer instead — without needing to re-fetch + the actions accessor.""" + from litellm.proxy.db.routing_prisma_wrapper import _RoutedActions + + writer_actions = _model_actions_mock("writer") + reader_actions = _model_actions_mock("reader") + + use_reader = {"value": True} + routed = _RoutedActions(writer_actions, reader_actions, lambda: use_reader["value"]) + + # Reader healthy → reads to reader. + assert routed.find_many is reader_actions.find_many + + # Reader degrades mid-flight → next read goes to writer. + use_reader["value"] = False + assert routed.find_many is writer_actions.find_many + + +# --------------------------------------------------------------------------- +# Reader graceful degradation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_connect_swallows_reader_failure_and_falls_back_to_writer(): + """A reader connect failure must NOT abort proxy startup. The wrapper + flips into degraded mode so subsequent reads route to the writer.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.connect = AsyncMock() + reader_inner.connect = AsyncMock(side_effect=RuntimeError("reader unreachable")) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Must not raise — reader failure is non-fatal. + await routing.connect() + + assert routing.reader_unavailable is True + writer_inner.connect.assert_awaited_once() + reader_inner.connect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_reads_route_to_writer_when_reader_unavailable(): + """Top-level read methods and per-model reads must fall through to the + writer while the reader is degraded.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, writer_inner, reader, reader_inner = _make_wrappers() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner.litellm_usertable = _model_actions_mock("reader_users") + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + # Top-level reads → writer. + assert routing.query_raw is writer_inner.query_raw + assert routing.query_first is writer_inner.query_first + + # Per-model reads → writer actions. + actions = routing.litellm_usertable + assert actions.find_many is writer_inner.litellm_usertable.find_many + assert actions.find_unique is writer_inner.litellm_usertable.find_unique + + +@pytest.mark.asyncio +async def test_recreate_prisma_client_recreates_both_writer_and_reader(): + """Writer reconnect path calls recreate_prisma_client. The routing wrapper + must recreate BOTH clients so a DB-wide event doesn't leave a stale reader.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url", http_client=None) + + writer.recreate_prisma_client.assert_awaited_once_with( + "writer-url", http_client=None + ) + reader.recreate_prisma_client.assert_awaited_once_with( + "reader-url", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_recovers_reader_after_prior_degradation(): + """If a previous connect/recreate degraded the reader, a successful + recreate must clear the flag so reads start hitting the reader again.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + routing._reader_unavailable = True + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + await routing.recreate_prisma_client("writer-url") + + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_if_reader_recreate_fails(): + """If the reader recreate fails, writer recreate still succeeds and the + routing wrapper degrades (does not raise).""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock( + side_effect=RuntimeError("reader still down") + ) + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + with patch.dict(os.environ, {"DATABASE_URL_READ_REPLICA": "reader-url"}): + # Must not raise — writer was recreated, reader is best-effort. + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_degrades_reader_when_replica_url_missing(): + """Non-IAM reader needs DATABASE_URL_READ_REPLICA. If it's missing + (configuration drift), the wrapper degrades instead of raising.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = False + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + # Ensure env var is absent. + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DATABASE_URL_READ_REPLICA", None) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +@pytest.mark.asyncio +async def test_recreate_iam_reader_refreshes_token(): + """IAM-enabled readers must refresh their token (reader has its own parsed + endpoint) and pass the fresh URL to recreate_prisma_client.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value="postgresql://u:fresh@h:5432/db") + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + reader.get_rds_iam_token.assert_called_once() + reader.recreate_prisma_client.assert_awaited_once_with( + "postgresql://u:fresh@h:5432/db", http_client=None + ) + assert routing.reader_unavailable is False + + +@pytest.mark.asyncio +async def test_recreate_degrades_when_iam_token_generation_returns_none(): + """If `get_rds_iam_token` returns None (e.g. AWS-side failure), the wrapper + must degrade rather than crash — this exercises the explicit `raise + RuntimeError` inside `_recreate_reader`'s IAM branch.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer = MagicMock() + writer.recreate_prisma_client = AsyncMock() + reader = MagicMock() + reader.iam_token_db_auth = True + reader.get_rds_iam_token = MagicMock(return_value=None) + reader.recreate_prisma_client = AsyncMock() + + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + await routing.recreate_prisma_client("writer-url") + + writer.recreate_prisma_client.assert_awaited_once() + reader.recreate_prisma_client.assert_not_awaited() + assert routing.reader_unavailable is True + + +def test_writer_and_reader_properties_expose_underlying_wrappers(): + """The `writer` and `reader` properties are used by PrismaClient.writer_db + to smoke-test the writer specifically during reconnect — they must return + the exact wrappers passed in.""" + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer, _, reader, _ = _make_wrappers() + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + assert routing.writer is writer + assert routing.reader is reader + + +def test_per_model_accessor_falls_back_when_reader_lacks_attr(): + """If the reader Prisma client somehow lacks a model accessor that the + writer has (older client / partial mock), the wrapper must fall back to + the writer accessor instead of raising AttributeError to the caller.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + # Plain class with only the accessor set on the writer side. Using a real + # class instead of MagicMock so attribute access raises AttributeError + # naturally instead of auto-creating mock attributes. + class _PartialPrisma: + pass + + writer_inner = _PartialPrisma() + writer_inner.litellm_usertable = _model_actions_mock("writer_users") + reader_inner = _PartialPrisma() # deliberately missing litellm_usertable + + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + + actions = routing.litellm_usertable + # Falls back to the writer's accessor verbatim — not a _RoutedActions wrapper. + assert actions is writer_inner.litellm_usertable + + +@pytest.mark.asyncio +async def test_writer_recreate_passes_http_client_through(monkeypatch): + """When PrismaClient is constructed with an http_client, recreate must + forward it to the new Prisma() so connection settings persist across + reconnects.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + sentinel_http = object() + await writer.recreate_prisma_client( + "postgresql://u:p@h:5432/db", http_client=sentinel_http + ) + + assert captured_kwargs == {"http": sentinel_http} + + +# --------------------------------------------------------------------------- +# IAM endpoint parsing + reader IAM refresh +# --------------------------------------------------------------------------- + + +def test_parse_iam_endpoint_from_url_extracts_all_fields(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url( + "postgresql://litellm_user:initial-token@aurora-reader.example.com:6543/litellm?schema=public" + ) + assert ep.host == "aurora-reader.example.com" + assert ep.port == "6543" + assert ep.user == "litellm_user" + assert ep.name == "litellm" + assert ep.schema == "public" + + +def test_parse_iam_endpoint_defaults_port_to_5432_and_skips_schema(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + ep = parse_iam_endpoint_from_url("postgresql://u@host/dbname") + assert ep.host == "host" + assert ep.port == "5432" + assert ep.user == "u" + assert ep.name == "dbname" + assert ep.schema is None + + +def test_parse_iam_endpoint_rejects_url_without_user_or_dbname(): + from litellm.proxy.db.prisma_client import parse_iam_endpoint_from_url + + with pytest.raises(ValueError, match="missing host or username"): + parse_iam_endpoint_from_url("postgresql://host:5432/db") + with pytest.raises(ValueError, match="missing database name"): + parse_iam_endpoint_from_url("postgresql://u@host:5432/") + + +def test_iam_endpoint_build_url_inserts_token_verbatim(): + from litellm.proxy.db.prisma_client import IAMEndpoint + + # `generate_iam_auth_token` already URL-encodes the presigned token, so + # `build_url` must NOT encode again — double-encoding turned `%3D` into + # `%253D` and broke RDS auth on the reader path. + ep = IAMEndpoint(host="h", port="5432", user="u", name="db", schema="public") + pre_encoded_token = "token%2Fwith%3Fweird%26chars%3Dyes" + url = ep.build_url(pre_encoded_token) + assert url == f"postgresql://u:{pre_encoded_token}@h:5432/db?schema=public" + # Sanity check: no `%25` (the encoding of `%`), confirming we didn't re-encode. + assert "%25" not in url + + +@pytest.mark.asyncio +async def test_iam_refresh_logs_carry_log_prefix(caplog): + """When `log_prefix` is set on a PrismaWrapper, every IAM-related log + line emitted by that wrapper must start with the prefix so writer and + reader can be told apart in interleaved output.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + log_prefix="[reader]", + ) + + with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): + await wrapper.start_token_refresh_task() + # Loop emits "RDS IAM token refresh loop started..." on first tick. + # Cancel immediately so the loop body runs once and we can assert. + await wrapper.stop_token_refresh_task() + + messages = [r.getMessage() for r in caplog.records] + # Both start and stop notifications carry the prefix. + assert any( + m.startswith("[reader] Started RDS IAM token proactive refresh") + for m in messages + ) + assert any( + m.startswith("[reader] Stopped RDS IAM token refresh background task") + for m in messages + ) + + +def test_get_rds_iam_token_returns_none_when_iam_disabled(): + """`get_rds_iam_token` short-circuits to None when iam_token_db_auth is + False — covers the early-return guard at the top of the method.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + wrapper = PrismaWrapper(original_prisma=MagicMock(), iam_token_db_auth=False) + assert wrapper.get_rds_iam_token() is None + + +@pytest.mark.asyncio +async def test_getattr_does_not_block_inside_running_loop_on_expired_token(monkeypatch): + """When `__getattr__` runs inside a running event loop and the IAM token + is expired, it MUST schedule the refresh as a background task and return + immediately. The previous `run_coroutine_threadsafe` + `future.result()` + pattern deadlocks the loop (loop thread blocks waiting for a coroutine + that needs the loop to run) and times out at 30s — exactly what was + breaking the reader on first query.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + # Stale URL — `is_token_expired` returns True because the password isn't + # a parseable IAM token, so we exercise the expired branch. + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader:placeholder@reader.aurora.local:5432/litellm", + ) + + inner = MagicMock() + inner.query_raw = MagicMock(name="query_raw_attr") + + wrapper = PrismaWrapper( + original_prisma=inner, + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + ) + + # Replace the heavy refresh coroutine with a no-op AsyncMock so we can + # observe whether it was scheduled without actually doing the recreate. + refresh_calls = {"count": 0} + + async def fake_refresh(): + refresh_calls["count"] += 1 + + monkeypatch.setattr(wrapper, "_safe_refresh_token", fake_refresh) + + # Direct attribute access from inside this async test runs __getattr__ + # on the loop thread, exercising the in-loop branch. If the previous + # `run_coroutine_threadsafe` + `future.result()` pattern were back, this + # line would deadlock the loop and the test would hang (and pytest's + # per-test timeout would catch it). + attr = wrapper.query_raw + # Yield once so the scheduled refresh task gets a chance to run. + await asyncio.sleep(0) + + assert attr is inner.query_raw + assert refresh_calls["count"] == 1 + + +def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): + """When DATABASE_PORT is unset, the writer must default to the Postgres + standard port instead of passing `None` through. Passing None to + `generate_iam_auth_token` makes botocore embed the literal string + \"None\" in the presigned URL during signing and crashes with + `ValueError: Port could not be cast to integer value as 'None'`.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.delenv("DATABASE_PORT", raising=False) + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.delenv("DATABASE_SCHEMA", raising=False) + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["port"] = db_port + return "TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + new_url = writer.get_rds_iam_token() + + assert captured["port"] == "5432" # default applied, NOT None + assert ":5432/litellm" in (new_url or "") + + +def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch): + """Writer's IAM path (no iam_endpoint configured) reads host/port/user/db + from the legacy DATABASE_HOST/PORT/USER/NAME env vars and writes the URL + back to DATABASE_URL — this is the pre-read-replica behavior the patch + must preserve.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + monkeypatch.setenv("DATABASE_HOST", "writer.aurora.local") + monkeypatch.setenv("DATABASE_PORT", "5432") + monkeypatch.setenv("DATABASE_USER", "litellm") + monkeypatch.setenv("DATABASE_NAME", "litellm") + monkeypatch.setenv("DATABASE_SCHEMA", "public") + monkeypatch.delenv("DATABASE_URL", raising=False) + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "WRITER-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + # No iam_endpoint → legacy DATABASE_HOST/etc. path. + ) + new_url = writer.get_rds_iam_token() + + assert captured == { + "host": "writer.aurora.local", + "port": "5432", + "user": "litellm", + } + assert new_url == ( + "postgresql://litellm:WRITER-TOKEN@writer.aurora.local:5432/litellm?schema=public" + ) + # Writer updates its own env var (DATABASE_URL by default), not the reader's. + assert os.environ["DATABASE_URL"] == new_url + + +def test_reader_iam_refresh_uses_parsed_endpoint(monkeypatch): + """The reader generates fresh tokens against its parsed endpoint and + writes the new URL to DATABASE_URL_READ_REPLICA — not DATABASE_URL.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + # Pre-seed env vars so we can prove the reader does NOT touch DATABASE_URL. + monkeypatch.setenv("DATABASE_URL", "writer-url-untouched") + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "stale-reader-url") + + captured: Dict[str, Any] = {} + + def fake_generate(db_host=None, db_port=None, db_user=None): + captured["host"] = db_host + captured["port"] = db_port + captured["user"] = db_user + return "FRESH-TOKEN" + + fake_module = MagicMock() + fake_module.generate_iam_auth_token = fake_generate + monkeypatch.setitem(sys.modules, "litellm.proxy.auth.rds_iam_token", fake_module) + + endpoint = IAMEndpoint( + host="reader.aurora.local", + port="5432", + user="lit", + name="litellm", + schema=None, + ) + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=endpoint, + recreate_uses_datasource=True, + ) + + new_url = reader.get_rds_iam_token() + + # IAM token generator was called with the reader's parsed endpoint, not + # the writer's DATABASE_HOST/PORT/USER env vars. + assert captured == { + "host": "reader.aurora.local", + "port": "5432", + "user": "lit", + } + assert new_url is not None + assert new_url.startswith( + "postgresql://lit:FRESH-TOKEN@reader.aurora.local:5432/litellm" + ) + # The reader updates its OWN env var; writer's DATABASE_URL is left alone. + assert os.environ["DATABASE_URL_READ_REPLICA"] == new_url + assert os.environ["DATABASE_URL"] == "writer-url-untouched" + + +@pytest.mark.asyncio +async def test_reader_recreate_uses_datasource_override(monkeypatch): + """Reader recreate must pass `datasource={"url": ...}` to Prisma() — Prisma + only auto-reads DATABASE_URL, so without the override the new reader URL + would be silently ignored.""" + from litellm.proxy.db.prisma_client import IAMEndpoint, PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + reader = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + db_url_env_var="DATABASE_URL_READ_REPLICA", + iam_endpoint=IAMEndpoint(host="h", port="5432", user="u", name="db"), + recreate_uses_datasource=True, + ) + + await reader.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert captured_kwargs == { + "datasource": {"url": "postgresql://u:newtoken@h:5432/db"} + } + + +@pytest.mark.asyncio +async def test_writer_recreate_does_not_use_datasource(monkeypatch): + """Writer keeps relying on Prisma reading DATABASE_URL from env — datasource + override must NOT leak into the writer path (would override the freshly + rotated env var).""" + from litellm.proxy.db.prisma_client import PrismaWrapper + + captured_kwargs: Dict[str, Any] = {} + + class FakePrisma: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + async def connect(self): + return None + + fake_module = MagicMock() + fake_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_module) + + writer = PrismaWrapper( + original_prisma=MagicMock(), + iam_token_db_auth=True, + ) + + await writer.recreate_prisma_client( + "postgresql://u:newtoken@h:5432/db", http_client=None + ) + + assert "datasource" not in captured_kwargs + + +def test_prisma_client_init_falls_back_to_writer_when_reader_iam_token_fails( + monkeypatch, caplog +): + """A transient AWS STS error (or any other failure) during the reader + IAM token mint must NOT abort proxy startup. The reader is opt-in, so + `PrismaClient.__init__` should log a warning and fall back to the + writer-only `PrismaWrapper`. The runtime contract in + `RoutingPrismaWrapper.connect` already says reader-side failures are + non-fatal — but that code never runs if construction throws first.""" + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "true") + monkeypatch.setenv( + "DATABASE_URL_READ_REPLICA", + "postgresql://reader_user@reader.aurora.local:5432/litellm", + ) + + class FakePrisma: + def __init__(self, **kwargs): + self.kwargs = kwargs + + async def connect(self): + return None + + fake_prisma_module = MagicMock() + fake_prisma_module.Prisma = FakePrisma + monkeypatch.setitem(sys.modules, "prisma", fake_prisma_module) + + fake_iam_module = MagicMock() + + def boom(**_kwargs): + raise RuntimeError("simulated AWS STS hiccup") + + fake_iam_module.generate_iam_auth_token = boom + monkeypatch.setitem( + sys.modules, "litellm.proxy.auth.rds_iam_token", fake_iam_module + ) + + from litellm.proxy.utils import PrismaClient + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + client = PrismaClient( + database_url="postgresql://writer@writer.aurora.local:5432/litellm", + proxy_logging_obj=MagicMock(), + ) + + # Construction did not raise, and the proxy is in writer-only mode — + # NOT a RoutingPrismaWrapper, so reads will go to the writer. + assert isinstance(client.db, PrismaWrapper) + assert not isinstance(client.db, RoutingPrismaWrapper) + # And the operator gets a clear warning. + assert any( + "Failed to initialize read replica Prisma client" in r.getMessage() + for r in caplog.records + ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 2418d7af04b..6e027fa4941 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -8,7 +8,9 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, + effective_skip_tool_message_for_guardrail, openai_messages_without_system, + openai_messages_without_tool, ) from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, @@ -180,6 +182,136 @@ class TestUnifiedLLMGuardrails: } assert "system" in roles + class TestSkipToolMessageForChatCompletions: + def test_openai_messages_without_tool(self): + msgs = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "content": "tool result", "tool_call_id": "call_1"}, + ] + out = openai_messages_without_tool(msgs) + assert len(out) == 2 + assert all(m["role"] != "tool" for m in out) + assert msgs[2]["content"] == "tool result" + + def test_effective_skip_tool_respects_per_guardrail_over_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + class G: + skip_tool_message_in_guardrail = False + + assert effective_skip_tool_message_for_guardrail(G()) is False + + class G2: + skip_tool_message_in_guardrail = None + + assert effective_skip_tool_message_for_guardrail(G2()) is True + + @pytest.mark.asyncio + async def test_openai_handler_skips_tool_in_guardrail_inputs(self, monkeypatch): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = None + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "content": "secret tool result", + "tool_call_id": "call_1", + }, + ], + "model": "gpt-4o", + } + + handler = OpenAIChatCompletionsHandler() + await handler.process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "secret tool result" not in captured["inputs"]["texts"] + sm = captured["inputs"].get("structured_messages") or [] + assert all(m.get("role") != "tool" for m in sm) + assert data["messages"][2]["content"] == "secret tool result" + + @pytest.mark.asyncio + async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global( + self, monkeypatch + ): + monkeypatch.setattr( + litellm, "skip_tool_message_in_guardrail", True, raising=False + ) + + captured = {} + + class MockGuardrail: + skip_tool_message_in_guardrail = False + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + captured["inputs"] = inputs + return inputs + + data = { + "messages": [ + {"role": "user", "content": "u"}, + {"role": "tool", "content": "tr", "tool_call_id": "call_1"}, + ], + } + + await OpenAIChatCompletionsHandler().process_input_messages( + data=data, + guardrail_to_apply=MockGuardrail(), + litellm_logging_obj=None, + ) + + assert "tr" in captured["inputs"]["texts"] + roles = { + m.get("role") + for m in (captured["inputs"].get("structured_messages") or []) + } + assert "tool" in roles + class TestAsyncPreCallHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py b/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py index 6def548b93f..4c19ee2906b 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_coverage.py @@ -660,7 +660,7 @@ async def test_azure_content_safety_post_call_checks_all_choices(user_api_key): @pytest.mark.asyncio async def test_secret_detection_redacts_multimodal_text_parts(user_api_key): - from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import ( + from litellm_enterprise.enterprise_callbacks.secret_detection import ( _ENTERPRISE_SecretDetection, ) @@ -696,7 +696,7 @@ async def test_secret_detection_redacts_multimodal_text_parts(user_api_key): @pytest.mark.asyncio async def test_secret_detection_redacts_responses_api_input(user_api_key): - from enterprise.litellm_enterprise.enterprise_callbacks.secret_detection import ( + from litellm_enterprise.enterprise_callbacks.secret_detection import ( _ENTERPRISE_SecretDetection, ) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 033deb3ff42..0d7becd3e2e 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -106,9 +106,11 @@ def mock_in_memory_handler(mocker): mock_handler = mocker.Mock(spec=InMemoryGuardrailHandler) mock_handler.list_in_memory_guardrails.return_value = [MOCK_CONFIG_GUARDRAIL] mock_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL + mock_handler.get_source.return_value = "config" mock_handler.initialize_guardrail = mocker.Mock() mock_handler.update_in_memory_guardrail = mocker.Mock() mock_handler.delete_in_memory_guardrail = mocker.Mock() + mock_handler.reconcile_db_guardrails = mocker.Mock(return_value=[]) return mock_handler @@ -162,6 +164,67 @@ async def test_list_guardrails_v2_with_db_and_config( assert isinstance(config_guardrail.litellm_params, BaseLitellmParams) +@pytest.mark.asyncio +async def test_list_guardrails_v2_skips_stale_db_backed_in_memory_entries(mocker): + """ + A guardrail that's still in this pod's memory tagged source='db' but is no + longer in the DB result (deleted on another pod, awaiting reconcile) must + NOT surface in the list response — pre-fix it leaked as 'config'. + """ + stale_guardrail = { + "guardrail_id": "stale-db-id", + "guardrail_name": "Stale DB Guardrail", + "litellm_params": {"guardrail": "bedrock", "mode": "pre_call"}, + "guardrail_info": {}, + } + mock_prisma_client = mocker.Mock() + mock_prisma_client.db = mocker.Mock() + mock_prisma_client.db.litellm_guardrailstable = mocker.Mock() + mock_prisma_client.db.litellm_guardrailstable.find_many = AsyncMock(return_value=[]) + + mock_in_memory_handler = mocker.Mock() + mock_in_memory_handler.list_in_memory_guardrails.return_value = [stale_guardrail] + mock_in_memory_handler.get_source.return_value = "db" + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch( + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + + admin_auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + response = await list_guardrails_v2(user_api_key_dict=admin_auth) + + assert response.guardrails == [] + mock_in_memory_handler.get_source.assert_called_with("stale-db-id") + + +@pytest.mark.asyncio +async def test_get_guardrail_info_404s_stale_db_backed_entry( + mocker, mock_prisma_client, mock_in_memory_handler +): + """ + Stale DB-backed entry (in-memory but not in DB) must 404 instead of being + returned as if it were a config-loaded guardrail. + """ + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch( + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + mock_prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock( + return_value=None + ) + # In-memory still has it, but it's tagged as 'db' (stale, awaiting reconcile) + mock_in_memory_handler.get_source.return_value = "db" + + with pytest.raises(HTTPException) as exc_info: + await get_guardrail_info("stale-db-id") + + assert exc_info.value.status_code == 404 + assert "not found" in str(exc_info.value.detail) + + @pytest.mark.asyncio async def test_list_guardrails_v2_masks_sensitive_data_in_db_guardrails(mocker): """Test that sensitive litellm_params are masked for DB guardrails in list response""" @@ -1160,6 +1223,7 @@ async def test_get_guardrail_info_endpoint_config_guardrail(mocker): # Mock IN_MEMORY_GUARDRAIL_HANDLER at its source to return config guardrail mock_in_memory_handler = mocker.Mock() mock_in_memory_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL + mock_in_memory_handler.get_source.return_value = "config" mocker.patch( "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", mock_in_memory_handler, diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 1d70126681d..9f7173383b0 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -60,3 +60,123 @@ def test_update_in_memory_guardrail(): handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call ) + + +def _make_guardrail(guardrail_id: str, name: str = "g") -> Guardrail: + return Guardrail( + guardrail_id=guardrail_id, + guardrail_name=name, + litellm_params=LitellmParams(guardrail=name, mode="pre_call", default_on=False), + ) + + +def test_reconcile_db_guardrails_drops_stale_db_entries_only(): + """ + The reconcile pass must drop in-memory entries marked source='db' that are + missing from the DB result, and never touch source='config' entries. + Models the multi-pod case where another pod deleted a DB-backed guardrail. + """ + handler = InMemoryGuardrailHandler() + + # Two DB-backed entries on this pod (synced from earlier polling cycles) + handler.IN_MEMORY_GUARDRAILS["db-keep"] = _make_guardrail("db-keep") + handler.IN_MEMORY_GUARDRAILS["db-stale"] = _make_guardrail("db-stale") + handler._sources["db-keep"] = "db" + handler._sources["db-stale"] = "db" + + # One config-loaded entry that must survive reconciliation + handler.IN_MEMORY_GUARDRAILS["cfg"] = _make_guardrail("cfg") + handler._sources["cfg"] = "config" + + # The DB now only contains db-keep — db-stale was deleted on another pod. + removed = handler.reconcile_db_guardrails(db_guardrail_ids={"db-keep"}) + + assert removed == ["db-stale"] + assert "db-stale" not in handler.IN_MEMORY_GUARDRAILS + assert "db-stale" not in handler._sources + assert "db-keep" in handler.IN_MEMORY_GUARDRAILS + assert "cfg" in handler.IN_MEMORY_GUARDRAILS + assert handler._sources["cfg"] == "config" + + +def test_reconcile_does_not_drop_config_entries_missing_from_db(): + """A config-only guardrail (no DB row) must never be reconciled away.""" + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["cfg-only"] = _make_guardrail("cfg-only") + handler._sources["cfg-only"] = "config" + + removed = handler.reconcile_db_guardrails(db_guardrail_ids=set()) + + assert removed == [] + assert "cfg-only" in handler.IN_MEMORY_GUARDRAILS + + +def test_get_source_returns_marker_set_at_insert(): + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["a"] = _make_guardrail("a") + handler._sources["a"] = "db" + handler.IN_MEMORY_GUARDRAILS["b"] = _make_guardrail("b") + handler._sources["b"] = "config" + + assert handler.get_source("a") == "db" + assert handler.get_source("b") == "config" + assert handler.get_source("missing") is None + + +def test_delete_in_memory_guardrail_clears_source_marker(): + handler = InMemoryGuardrailHandler() + handler.IN_MEMORY_GUARDRAILS["a"] = _make_guardrail("a") + handler._sources["a"] = "db" + + handler.delete_in_memory_guardrail("a") + + assert "a" not in handler.IN_MEMORY_GUARDRAILS + assert "a" not in handler._sources + assert handler.get_source("a") is None + + +def test_initialize_guardrail_early_return_updates_source_marker(): + """ + When initialize_guardrail is called for a guardrail that already exists + in memory, the early-return path must still honor the caller's source. + Otherwise a racing polling tick that placed a DB entry in memory first + would leave a later config-init call wrongly marked as 'db' (or vice + versa), and the entry would be reconciled with the wrong classification. + """ + handler = InMemoryGuardrailHandler() + # Simulate a polling tick already placing the entry as DB-backed. + handler.IN_MEMORY_GUARDRAILS["collide"] = _make_guardrail("collide", name="bedrock") + handler._sources["collide"] = "db" + + # Config init re-visits the same id (e.g., hot-reload, or UUID collision). + g = Guardrail( + guardrail_id="collide", + guardrail_name="bedrock", + litellm_params=LitellmParams( + guardrail="bedrock", mode="pre_call", default_on=False + ), + ) + handler.initialize_guardrail(guardrail=g, source="config") + + assert handler.get_source("collide") == "config" + + # And the symmetric direction: db sync should override an entry left + # marked as 'config' from a stale init path. + handler.initialize_guardrail(guardrail=g, source="db") + assert handler.get_source("collide") == "db" + + +def test_sync_guardrail_from_db_marks_source_db_when_unchanged(): + """ + sync_guardrail_from_db must enforce source='db' even when params are + unchanged, so a config entry whose UUID happens to collide with a later + DB row gets re-tagged correctly. + """ + handler = InMemoryGuardrailHandler() + g = _make_guardrail("collide") + handler.IN_MEMORY_GUARDRAILS["collide"] = g + handler._sources["collide"] = "config" + + handler.sync_guardrail_from_db(g) + + assert handler.get_source("collide") == "db" diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 2edcb00c967..bcd7fcb37b3 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -466,6 +466,236 @@ async def test_test_model_connection_loads_config_from_router(): assert "result" in result +@pytest.mark.asyncio +async def test_test_model_connection_uses_model_info_id_to_disambiguate_duplicate_model_names(): + """ + When two deployments share the same `model_name` (e.g. wildcard + `openai/*`) but have different `api_base` values, clicking "Test + Connection" on a specific row in the UI must probe THAT row's + `api_base` — not whichever happens to be `deployments[0]`. + + The UI passes `model_info.id` to identify the deployment the user + actually clicked on. The backend must use that id to look up the + specific deployment rather than always grabbing the first match. + + Regression test for: silent fallback to deployments[0] when + multiple deployments share a wildcard model_name. + """ + from litellm.types.router import Deployment, LiteLLM_Params + + mock_request = MagicMock() + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.token = "test-token" + + mock_prisma_client = MagicMock() + + deployment_a = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-A-base.invalid/v1", + "api_key": "fake-key-A", + }, + "model_info": {"id": "deployment-A-id"}, + } + deployment_b = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-B-base.invalid/v1", + "api_key": "fake-key-B", + }, + "model_info": {"id": "deployment-B-id"}, + } + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [deployment_a, deployment_b] + + # Backend uses get_deployment(model_id=...) for O(1) lookup by id. + def _get_deployment_by_id(model_id): + if model_id == "deployment-A-id": + return Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params(**deployment_a["litellm_params"]), + model_info=deployment_a["model_info"], + ) + if model_id == "deployment-B-id": + return Deployment( + model_name="openai/*", + litellm_params=LiteLLM_Params(**deployment_b["litellm_params"]), + model_info=deployment_b["model_info"], + ) + return None + + mock_router.get_deployment.side_effect = _get_deployment_by_id + + mock_can_user_make_model_call = AsyncMock() + + mock_health_check_result = {"status": "healthy", "response_time_ms": 50} + mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) + mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) + + def mock_update_params(model_info, litellm_params): + params = litellm_params.copy() + params["messages"] = [{"role": "user", "content": "test"}] + return params + + def mock_reject_os_environ(params): + return None + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + mock_prisma_client, + ), + patch( + "litellm.proxy.proxy_server.llm_router", + mock_router, + ), + patch( + "litellm.proxy.proxy_server.premium_user", + False, + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + mock_can_user_make_model_call, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check", + mock_ahealth_check, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.run_with_timeout", + mock_run_with_timeout, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._update_litellm_params_for_health_check", + mock_update_params, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._reject_os_environ_references", + mock_reject_os_environ, + ), + ): + # Click "Test Connection" on deployment B (NOT the first one). + # The UI sends only `model` + `model_info.id` — it does NOT + # send `api_base`/`api_key`, so the backend must resolve them + # from the right deployment. + await health_test_model_connection( + request=mock_request, + mode="chat", + litellm_params={"model": "openai/*"}, + model_info={"id": "deployment-B-id"}, + user_api_key_dict=mock_user_api_key_dict, + ) + + # The outbound health check must hit deployment B's api_base. + ahealth_check_call_args = mock_ahealth_check.call_args + assert ahealth_check_call_args is not None + model_params = ahealth_check_call_args.kwargs.get("model_params", {}) + + assert model_params.get("api_base") == ( + "https://deployment-B-base.invalid/v1" + ), ( + "Expected /health/test_connection to probe deployment B's " + "api_base when model_info.id='deployment-B-id' was provided. " + f"Got: {model_params.get('api_base')!r}. This means the " + "backend silently fell back to deployments[0] (A) instead " + "of disambiguating by model_info.id." + ) + assert model_params.get("api_key") == "fake-key-B" + + +@pytest.mark.asyncio +async def test_test_model_connection_falls_back_to_deployments_zero_without_id(): + """ + Backwards-compat: when the request body does NOT include + `model_info.id`, the legacy behavior of using `deployments[0]` + is preserved (single-deployment case, or callers that haven't + been updated to pass an id). + """ + mock_request = MagicMock() + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.token = "test-token" + + mock_prisma_client = MagicMock() + + deployment_a = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-A-base.invalid/v1", + "api_key": "fake-key-A", + }, + "model_info": {"id": "deployment-A-id"}, + } + deployment_b = { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_base": "https://deployment-B-base.invalid/v1", + "api_key": "fake-key-B", + }, + "model_info": {"id": "deployment-B-id"}, + } + + mock_router = MagicMock() + mock_router.get_model_list.return_value = [deployment_a, deployment_b] + + mock_can_user_make_model_call = AsyncMock() + mock_health_check_result = {"status": "healthy"} + mock_ahealth_check = AsyncMock(return_value=mock_health_check_result) + mock_run_with_timeout = AsyncMock(return_value=mock_health_check_result) + + def mock_update_params(model_info, litellm_params): + params = litellm_params.copy() + params["messages"] = [{"role": "user", "content": "test"}] + return params + + def mock_reject_os_environ(params): + return None + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + patch("litellm.proxy.proxy_server.premium_user", False), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + mock_can_user_make_model_call, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.litellm.ahealth_check", + mock_ahealth_check, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints.run_with_timeout", + mock_run_with_timeout, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._update_litellm_params_for_health_check", + mock_update_params, + ), + patch( + "litellm.proxy.health_endpoints._health_endpoints._reject_os_environ_references", + mock_reject_os_environ, + ), + ): + await health_test_model_connection( + request=mock_request, + mode="chat", + litellm_params={"model": "openai/*"}, + model_info={}, # no id provided + user_api_key_dict=mock_user_api_key_dict, + ) + + # Without id, deployments[0] (A) should be used (legacy behavior). + model_params = mock_ahealth_check.call_args.kwargs.get("model_params", {}) + assert model_params.get("api_base") == "https://deployment-A-base.invalid/v1" + assert model_params.get("api_key") == "fake-key-A" + + @pytest.mark.asyncio async def test_health_services_endpoint_datadog_llm_observability(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index b292e8d0cae..66716400c4f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5689,7 +5689,7 @@ async def test_process_single_key_update(): "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): # Create update request - key_update_item = BulkUpdateKeyRequestItem( + update_key_request = UpdateKeyRequest( key="test-key-123", max_budget=100.0, tags=["production"], @@ -5703,7 +5703,7 @@ async def test_process_single_key_update(): # Call the function result = await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=None, prisma_client=mock_prisma_client, @@ -9855,9 +9855,6 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): from litellm.proxy.management_endpoints.key_management_endpoints import ( _process_single_key_update, ) - from litellm.types.proxy.management_endpoints.key_management_endpoints import ( - BulkUpdateKeyRequestItem, - ) token_hash = "abc123def456" @@ -9900,7 +9897,7 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): new_callable=AsyncMock, ), ): - key_update_item = BulkUpdateKeyRequestItem( + update_key_request = UpdateKeyRequest( key=token_hash, max_budget=100.0, ) @@ -9912,7 +9909,7 @@ async def test_process_single_key_update_cache_invalidation_with_token_hash(): ) await _process_single_key_update( - key_update_item=key_update_item, + update_key_request=update_key_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=None, prisma_client=mock_prisma_client, @@ -10019,3 +10016,583 @@ async def test_execute_virtual_key_regeneration_cache_invalidation_with_token_ha call_kwargs = mock_delete_cache.call_args.kwargs # The token hash should be passed as-is, NOT double-hashed assert call_kwargs["hashed_token"] == token_hash + + +# --------------------------------------------------------------------------- +# /team/key/bulk_update tests +# --------------------------------------------------------------------------- + + +_BULK_PKG = "litellm.proxy.management_endpoints.key_management_endpoints" + + +def _make_team_key(token: str, team_id: str = "team-abc") -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token=token, + user_id="user-123", + models=[], + team_id=team_id, + max_budget=None, + ) + + +def _admin() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin" + ) + + +def _internal_user() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-iu", user_id="iu" + ) + + +def _updated(payload): + m = MagicMock() + m.model_dump.return_value = payload + return m + + +def _setup_team_keys_mocks( + monkeypatch, + *, + find_many=None, + find_unique=None, + update_data=None, + hash_identity=True, +): + """Set up mocks for bulk_update_team_keys; returns mock_prisma.""" + mock_prisma = AsyncMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[] if find_many is None else find_many + ) + if find_unique is not None: + mock_prisma.db.litellm_verificationtoken.find_unique = find_unique + if update_data is not None: + mock_prisma.update_data = update_data + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.user_custom_key_update", None) + monkeypatch.setattr( + f"{_BULK_PKG}.prepare_key_update_data", + AsyncMock(return_value={"max_budget": 50.0}), + ) + monkeypatch.setattr(f"{_BULK_PKG}._delete_cache_key_object", AsyncMock()) + monkeypatch.setattr( + f"{_BULK_PKG}.KeyManagementEventHooks.async_key_updated_hook", AsyncMock() + ) + monkeypatch.setattr(f"{_BULK_PKG}.get_team_object", AsyncMock(return_value=None)) + monkeypatch.setattr(f"{_BULK_PKG}._check_team_key_limits", AsyncMock()) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + AsyncMock(), + ) + if hash_identity: + # Tests use already-hashed tokens; the raw-sk regression opts out. + monkeypatch.setattr(f"{_BULK_PKG}._hash_token_if_needed", lambda token: token) + return mock_prisma + + +async def _call_as_admin(data): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + + return await bulk_update_team_keys( + data=data, user_api_key_dict=_admin(), litellm_changed_by=None + ) + + +# ---- happy paths ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_success_with_key_ids(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + keys = [_make_team_key("tok-a"), _make_team_key("tok-b")] + find_unique = AsyncMock(side_effect=keys) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=keys, + find_unique=find_unique, + update_data=AsyncMock( + side_effect=[{"data": _updated({"max_budget": 50.0})}] * 2 + ), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-b"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert len(response.successful_updates) == 2 + assert len(response.failed_updates) == 0 + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + assert where["team_id"] == "team-abc" + assert where["token"] == {"in": ["tok-a", "tok-b"]} + find_unique.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_success_all_keys_in_team(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + keys = [_make_team_key(f"tok-{i}") for i in range(3)] + find_unique = AsyncMock(side_effect=keys) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=keys, + find_unique=find_unique, + update_data=AsyncMock( + side_effect=[{"data": _updated({"max_budget": 50.0})}] * 3 + ), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert len(response.successful_updates) == 3 + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + # `blocked` is Boolean? with no default → /key/generate writes NULL. Prisma's + # NOT excludes NULLs, so the filter has to OR `false` with `null` explicitly. + blocked_or, expires_or = where["AND"][0]["OR"], where["AND"][1]["OR"] + assert {"blocked": False} in blocked_or and {"blocked": None} in blocked_or + assert {"expires": None} in expires_or + assert any( + "gt" in c.get("expires", {}) + for c in expires_or + if isinstance(c.get("expires"), dict) + ) + find_unique.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_key_not_in_team(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + in_team = _make_team_key("tok-a") + _setup_team_keys_mocks( + monkeypatch, + find_many=[in_team], + find_unique=AsyncMock(return_value=in_team), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-foreign"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert [u.key for u in response.successful_updates] == ["tok-a"] + assert [u.key for u in response.failed_updates] == ["tok-foreign"] + assert "not found in team" in response.failed_updates[0].failed_reason + + +# ---- error paths ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_batch_size_cap(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks( + monkeypatch, + find_many=[_make_team_key(f"tok-{i}") for i in range(501)], + ) + + with pytest.raises(HTTPException) as exc: + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert exc.value.status_code == 400 + assert "more than 500" in exc.value.detail["error"] + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_empty_team_returns_404(monkeypatch): + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks(monkeypatch, find_many=[]) + with pytest.raises(HTTPException) as exc: + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-empty", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert exc.value.status_code == 404 + + +# ---- auth ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_team_member_with_permission(monkeypatch): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + key_a = _make_team_key("tok-a") + _setup_team_keys_mocks( + monkeypatch, + find_many=[key_a], + find_unique=AsyncMock(return_value=key_a), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + ) + auth_check = AsyncMock() + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + auth_check, + ) + + response = await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=50.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + assert len(response.successful_updates) == 1 + # Upfront check + per-key check inside _process_single_key_update + assert auth_check.await_count == 2 + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_team_member_no_permission(monkeypatch): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + AsyncMock( + side_effect=ProxyException( + message="not in team", + type="team_member_permission_error", + param="/key/update", + code=401, + ) + ), + ) + + with pytest.raises(ProxyException): + await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields(max_budget=1.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + mock.update_data.assert_not_called() + + +# ---- pydantic-layer validation ------------------------------------------- + + +def test_bulk_update_team_keys_request_validation(): + """Allowlist (extra='forbid'), empty-payload rejection, and selection XOR.""" + from pydantic import ValidationError + + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + forbidden = [ + "key", + "key_alias", + "team_id", + "allowed_routes", + "allowed_passthrough_routes", + "permissions", + "object_permission", + "access_group_ids", + "user_id", + "organization_id", + "blocked", + "key_type", + "models", + "config", + "router_settings", + "spend", + ] + for f in forbidden: + with pytest.raises(ValidationError, match=f): + KeyUpdateFields(**{f: True}) + + with pytest.raises(ValidationError, match="at least one"): + KeyUpdateFields() + + assert KeyUpdateFields(max_budget=50.0, tags=["x"]).max_budget == 50.0 + + valid = KeyUpdateFields(max_budget=10) + with pytest.raises(ValidationError): + BulkUpdateTeamKeysRequest( + team_id="t", key_ids=["k"], all_keys_in_team=True, update_fields=valid + ) + with pytest.raises(ValidationError): + BulkUpdateTeamKeysRequest(team_id="t", update_fields=valid) + + +# ---- security regressions ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_hashes_raw_sk_key_ids(monkeypatch): + """Regression: raw sk-... key_ids must be hashed before the find_many lookup.""" + from litellm.proxy._types import hash_token + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + raw_sk = "sk-rawkey1234567890" + hashed = hash_token(raw_sk) + row = LiteLLM_VerificationToken( + token=hashed, user_id="u", models=[], team_id="team-abc", max_budget=None + ) + mock = _setup_team_keys_mocks( + monkeypatch, + find_many=[row], + find_unique=AsyncMock(return_value=row), + update_data=AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}), + hash_identity=False, + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=[raw_sk], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + where = mock.db.litellm_verificationtoken.find_many.await_args.kwargs["where"] + assert where["token"] == {"in": [hashed]} + # Response reports the user-supplied form, not the hash. + assert response.successful_updates[0].key == raw_sk + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_auth_check_runs_when_no_keys_match(monkeypatch): + """Regression: non-admin with bogus key_ids must still hit the membership gate.""" + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[]) + auth_check = AsyncMock( + side_effect=ProxyException( + message="not in team", + type="team_member_permission_error", + param="/key/update", + code=401, + ) + ) + monkeypatch.setattr( + f"{_BULK_PKG}.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint", + auth_check, + ) + + with pytest.raises(ProxyException): + await bulk_update_team_keys( + data=BulkUpdateTeamKeysRequest( + team_id="victim-team", + key_ids=["bogus-1", "bogus-2"], + update_fields=KeyUpdateFields(max_budget=1.0), + ), + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + # Anchored on data.team_id, not existing_keys[0]. + assert auth_check.await_args.kwargs["existing_key_row"].team_id == "victim-team" + mock.update_data.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_does_not_log_raw_sk_token_on_failure( + monkeypatch, caplog +): + """Regression: per-key failure must not log the raw sk-... (ERROR-level logs persist).""" + import logging + + from litellm.proxy._types import hash_token + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + raw_sk = "sk-supersecret1234567890" + row = LiteLLM_VerificationToken( + token=hash_token(raw_sk), + user_id="u", + models=[], + team_id="team-abc", + max_budget=None, + ) + _setup_team_keys_mocks( + monkeypatch, + find_many=[row], + update_data=AsyncMock(side_effect=RuntimeError("boom")), + hash_identity=False, + ) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=[raw_sk], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + assert len(response.failed_updates) == 1 + log_text = "\n".join(r.getMessage() for r in caplog.records) + assert raw_sk not in log_text + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_propagates_team_id_to_per_key_request(monkeypatch): + """Regression: per-key UpdateKeyRequest carries data.team_id (gates _check_team_key_limits).""" + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + captured = [] + + async def fake_process(*, update_key_request, **kw): + captured.append(update_key_request) + return {"max_budget": update_key_request.max_budget} + + monkeypatch.setattr(f"{_BULK_PKG}._process_single_key_update", fake_process) + + await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a"], + update_fields=KeyUpdateFields( + tpm_limit=10_000, tpm_limit_type="guaranteed_throughput" + ), + ) + ) + assert captured[0].team_id == "team-abc" + assert captured[0].tpm_limit_type == "guaranteed_throughput" + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_dedupes_key_ids(monkeypatch): + """Duplicate key_ids collapse to a single update (no redundant DB writes, no inflated counts).""" + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + key_a = _make_team_key("tok-a") + update_data = AsyncMock(return_value={"data": _updated({"max_budget": 50.0})}) + _setup_team_keys_mocks( + monkeypatch, + find_many=[key_a], + find_unique=AsyncMock(return_value=key_a), + update_data=update_data, + ) + + response = await _call_as_admin( + BulkUpdateTeamKeysRequest( + team_id="team-abc", + key_ids=["tok-a", "tok-a", "tok-a"], + update_fields=KeyUpdateFields(max_budget=50.0), + ) + ) + + assert response.total_requested == 1 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 0 + update_data.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_bulk_update_team_keys_blocks_metadata_allowed_passthrough_routes( + monkeypatch, +): + """Non-admin can't grant passthrough access by smuggling allowed_passthrough_routes through metadata.""" + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + bulk_update_team_keys, + ) + from litellm.types.proxy.management_endpoints.key_management_endpoints import ( + BulkUpdateTeamKeysRequest, + KeyUpdateFields, + ) + + mock = _setup_team_keys_mocks(monkeypatch, find_many=[_make_team_key("tok-a")]) + + request = BulkUpdateTeamKeysRequest( + team_id="team-abc", + all_keys_in_team=True, + update_fields=KeyUpdateFields( + metadata={"allowed_passthrough_routes": ["/admin/*"]} + ), + ) + + with pytest.raises(HTTPException) as exc: + await bulk_update_team_keys( + data=request, + user_api_key_dict=_internal_user(), + litellm_changed_by=None, + ) + + assert exc.value.status_code == 403 + assert "allowed_passthrough_routes" in str(exc.value.detail) + mock.update_data.assert_not_called() diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index f0909afcbf6..30ad84e18b8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -2491,6 +2491,32 @@ class TestMCPApprovalWorkflow: assert exc_info.value.status_code == 400 assert "team" in str(exc_info.value.detail).lower() + @pytest.mark.asyncio + async def test_register_mcp_server_rejects_stdio_transport(self): + # stdio servers spawn a local subprocess on the proxy host. Accepting + # them from the non-admin submission endpoint would let a team member + # propose a config that an admin could rubber-stamp into local code + # execution. Admins use POST /v1/mcp/server or config.yaml instead. + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + register_mcp_server, + ) + + payload = NewMCPServerRequest( + alias="local", + transport=MCPTransport.stdio, + command="python3", + args=["-m", "mcp_server_filesystem", "/tmp"], + ) + user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-123", + user_id="user-abc", + ) + with pytest.raises(HTTPException) as exc_info: + await register_mcp_server(payload=payload, user_api_key_dict=user_auth) + assert exc_info.value.status_code == 400 + assert "stdio" in str(exc_info.value.detail).lower() + @pytest.mark.asyncio async def test_register_mcp_server_sets_pending_review(self): from litellm.proxy._types import MCPApprovalStatus diff --git a/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py b/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py index bd982480d60..a06d79306ab 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py +++ b/tests/test_litellm/proxy/management_endpoints/test_project_org_authz.py @@ -34,7 +34,7 @@ async def test_project_perm_check_uses_current_team_not_caller_supplied(): """The permission check must look at the project's existing team. Even if the caller is admin of an unrelated team, they must not pass when no explicit team_object is forced through.""" - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) @@ -56,7 +56,7 @@ async def test_project_perm_check_uses_current_team_not_caller_supplied(): @pytest.mark.asyncio async def test_project_perm_check_allows_team_admin_of_existing_team(): - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) @@ -76,7 +76,7 @@ async def test_project_perm_check_allows_team_admin_of_existing_team(): @pytest.mark.asyncio async def test_project_perm_check_proxy_admin_always_allowed(): - from enterprise.litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _check_user_permission_for_project, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 39ec6f075d7..76ba0e3dc67 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -380,6 +380,408 @@ async def test_list_tags_no_dynamic_tags(): app.dependency_overrides.clear() +async def test_internal_user_list_tags_only_returns_tags_used_by_their_keys(): + """ + Internal users can view tag usage, but the tag list must be scoped to tags + produced by API keys owned by the caller. + """ + from datetime import datetime + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + api_key="current-owned-key", + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + + mock_db.litellm_dailytagspend.group_by = AsyncMock( + return_value=[ + { + "tag": "stored-owned-tag", + "_min": {"created_at": "2025-02-01T00:00:00Z"}, + "_max": {"updated_at": "2025-03-01T00:00:00Z"}, + }, + { + "tag": "dynamic-owned-tag", + "_min": {"created_at": "2025-02-02T00:00:00Z"}, + "_max": {"updated_at": "2025-03-02T00:00:00Z"}, + }, + ] + ) + + stored_tag = Mock() + stored_tag.tag_name = "stored-owned-tag" + stored_tag.description = "A stored tag used by the caller" + stored_tag.models = ["model-1"] + stored_tag.model_info = {} + stored_tag.spend = 0.0 + stored_tag.budget_id = None + stored_tag.created_at = datetime(2025, 1, 1) + stored_tag.updated_at = datetime(2025, 1, 1) + stored_tag.created_by = "admin-user" + stored_tag.litellm_budget_table = None + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[stored_tag]) + + response = client.get( + "/tag/list", + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + assert [tag["name"] for tag in response.json()] == [ + "stored-owned-tag", + "dynamic-owned-tag", + ] + mock_db.litellm_verificationtoken.find_many.assert_awaited_once_with( + where={"user_id": "internal-user-123"}, + select={"token": True}, + ) + mock_db.litellm_dailytagspend.group_by.assert_awaited_once_with( + by=["tag"], + where={ + "tag": {"not": None}, + "api_key": {"in": ["current-owned-key", "owned-key"]}, + }, + min={"created_at": True}, + max={"updated_at": True}, + ) + mock_db.litellm_tagtable.find_many.assert_awaited_once_with( + where={"tag_name": {"in": ["stored-owned-tag", "dynamic-owned-tag"]}}, + include={"litellm_budget_table": True}, + ) + + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_list_tags_with_date_range_filters_dynamic_tags(): + """ + /tag/list?start_date=...&end_date=... should push the date window into + the dailytagspend group_by WHERE clause so large tables don't get scanned. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + group_by_mock = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = group_by_mock + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get( + "/tag/list?start_date=2026-04-01&end_date=2026-04-29", + headers=headers, + ) + + assert response.status_code == 200 + group_by_mock.assert_awaited_once() + where = group_by_mock.await_args.kwargs["where"] + assert where["tag"] == {"not": None} + assert where["date"] == {"gte": "2026-04-01", "lte": "2026-04-29"} + + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_is_scoped_to_their_keys(): + """ + Internal users must not receive proxy-wide tag spend rows when viewing tag + usage daily activity. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + mock_get_daily_activity.return_value = "daily-activity-response" + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result == "daily-activity-response" + mock_get_daily_activity.assert_awaited_once() + assert mock_get_daily_activity.await_args.kwargs["api_key"] == ["owned-key"] + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_rejects_unowned_api_key_filter(): + """ + If an internal user filters tag usage by an API key they do not own, the + endpoint should return an empty scoped filter instead of exposing that key's + tag spend. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + + owned_key_record = Mock() + owned_key_record.token = "owned-key" + mock_db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[owned_key_record] + ) + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + api_key="unowned-key", + user_api_key_dict=mock_user_auth, + ) + + assert result.results == [] + assert result.metadata.total_spend == 0 + assert result.metadata.total_api_requests == 0 + mock_get_daily_activity.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_scopes_to_current_key_without_user_id(): + """ + If an internal-user token has no user_id, it should still scope tag usage to + the current request key instead of falling back to proxy-wide tag spend. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + api_key="current-owned-key", + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_get_daily_activity.return_value = "daily-activity-response" + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result == "daily-activity-response" + mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + mock_get_daily_activity.assert_awaited_once() + assert mock_get_daily_activity.await_args.kwargs["api_key"] == [ + "current-owned-key" + ] + + +@pytest.mark.asyncio +async def test_internal_user_tag_daily_activity_without_any_scoped_keys_returns_empty(): + """ + If an internal-user token has neither user_id nor api_key, the endpoint must + return an empty response instead of dropping the API key filter. + """ + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints.get_daily_activity", + new_callable=AsyncMock, + ) as mock_get_daily_activity, + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + result = await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert result.results == [] + assert result.metadata.total_spend == 0 + assert result.metadata.total_api_requests == 0 + mock_db.litellm_verificationtoken.find_many.assert_not_awaited() + mock_get_daily_activity.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_tag_daily_activity_requires_database_connection(): + """ + Tag daily activity should fail with the same explicit DB error used by other + tag endpoints instead of raising an AttributeError during scope resolution. + """ + from litellm.proxy.management_endpoints.tag_management_endpoints import ( + get_tag_daily_activity, + ) + + mock_user_auth = UserAPIKeyAuth( + user_id="internal-user-123", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with patch("litellm.proxy.proxy_server.prisma_client", None): + with pytest.raises(HTTPException) as exc_info: + await get_tag_daily_activity( + start_date="2025-01-01", + end_date="2025-01-31", + user_api_key_dict=mock_user_auth, + ) + + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Database not connected" + + +@pytest.mark.asyncio +async def test_list_tags_without_date_range_omits_date_filter(): + """When no date range is passed, the WHERE clause must not carry a date key.""" + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + group_by_mock = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = group_by_mock + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get("/tag/list", headers=headers) + + assert response.status_code == 200 + group_by_mock.assert_awaited_once() + where = group_by_mock.await_args.kwargs["where"] + assert "date" not in where + + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize( + "query, expected_detail_fragment", + [ + ("?start_date=2026-04-01", "must be provided together"), + ("?end_date=2026-04-29", "must be provided together"), + ("?start_date=2026-04-29&end_date=2026-04-01", "on or before end_date"), + ("?start_date=not-a-date&end_date=2026-04-29", "YYYY-MM-DD"), + ], +) +@pytest.mark.asyncio +async def test_list_tags_rejects_invalid_date_range(query, expected_detail_fragment): + from unittest.mock import AsyncMock, Mock + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + mock_user_auth = UserAPIKeyAuth( + user_id="test-user-123", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[user_api_key_auth] = lambda: mock_user_auth + + try: + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_tagtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_dailytagspend.group_by = AsyncMock(return_value=[]) + + headers = {"Authorization": "Bearer sk-1234"} + response = client.get(f"/tag/list{query}", headers=headers) + + assert response.status_code == 400 + assert expected_detail_fragment in response.json()["detail"] + + finally: + app.dependency_overrides.clear() + + @pytest.mark.asyncio async def test_get_deployments_by_model_id(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e668672dd2a..5c7bbc46c95 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3140,6 +3140,121 @@ async def test_list_team_v2_with_invalid_status(): assert "deleted" in str(exc_info.value.detail) +@pytest.mark.asyncio +async def test_list_team_v2_search_builds_or_clause(): + """ + `search` should be passed as a Prisma OR across team_id (exact) and + team_alias (case-insensitive contains), so the UI can hit a single + backend filter with either a UUID or a name fragment. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id=None, + organization_id=None, + team_id=None, + team_alias=None, + search="platform", + user_api_key_dict=mock_admin, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + assert find_many_kwargs["where"] == { + "OR": [ + {"team_id": "platform"}, + {"team_alias": {"contains": "platform", "mode": "insensitive"}}, + ] + } + + +@pytest.mark.asyncio +async def test_list_team_v2_search_composes_with_user_id_filter(): + """ + For non-admin users, `search` must compose with the membership filter: + the resulting where clause should AND `team_id IN ` with + the search OR clause, so users still only see their own teams. + """ + from datetime import datetime + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import ( + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="member_user" + ) + + mock_user = LiteLLM_UserTable( + user_id="member_user", + teams=["team_a", "team_b"], + organization_memberships=[], + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, + patch( + "litellm.proxy.management_endpoints.team_endpoints.get_user_object", + new=AsyncMock(return_value=mock_user), + ), + patch( + "litellm.proxy.management_endpoints.team_endpoints._get_org_admin_org_ids", + new=AsyncMock(return_value=None), + ), + ): + mock_db = Mock() + mock_prisma.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id="member_user", + organization_id=None, + team_id=None, + team_alias=None, + search="team_a", + user_api_key_dict=mock_user_api_key_dict, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + where = find_many_kwargs["where"] + assert where["OR"] == [ + {"team_id": "team_a"}, + {"team_alias": {"contains": "team_a", "mode": "insensitive"}}, + ] + assert where["team_id"] == {"in": ["team_a", "team_b"]} + + @pytest.mark.asyncio async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_auth): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 69798744f7f..83317157847 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -1797,15 +1797,13 @@ class TestCustomUISSO: ): with patch.dict( "sys.modules", - { - "enterprise.litellm_enterprise.proxy.auth.custom_sso_handler": None - }, + {"litellm_enterprise.proxy.auth.custom_sso_handler": None}, ): # Temporarily mock the google_login function call to test the import error path async def mock_google_login(): # This mimics the relevant part of google_login that would trigger the import error try: - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( # noqa: F401 + from litellm_enterprise.proxy.auth.custom_sso_handler import ( # noqa: F401 EnterpriseCustomSSOHandler, ) @@ -1828,7 +1826,7 @@ class TestCustomUISSO: """Test successful custom UI SSO sign-in with valid headers""" from fastapi_sso.sso.base import OpenID - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler @@ -1903,7 +1901,7 @@ class TestCustomUISSO: @pytest.mark.asyncio async def test_handle_custom_ui_sso_sign_in_rejects_untrusted_proxy(self): """Custom UI SSO rejects spoofed identity headers from direct clients.""" - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler @@ -1943,7 +1941,7 @@ class TestCustomUISSO: """ from fastapi_sso.sso.base import OpenID - from enterprise.litellm_enterprise.proxy.auth.custom_sso_handler import ( + from litellm_enterprise.proxy.auth.custom_sso_handler import ( EnterpriseCustomSSOHandler, ) from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler @@ -6129,3 +6127,163 @@ class TestPKCEStateCookieBinding: # State-cookie check passed, so the function got past the early # ProxyException raise and produced an SSO result object. assert result is not None + + +@pytest.mark.asyncio +async def test_debug_sso_callback_renders_full_jwt_claims(): + """ + /sso/debug/callback should render the complete set of claims returned by the + IdP — both the raw userinfo response and the decoded access-token JWT — in + addition to the proxy-parsed OpenID fields. Bearer tokens must be stripped + even if a non-conforming IdP places them in its userinfo response. + """ + from litellm.proxy.management_endpoints.ui_sso import debug_sso_callback + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "http://proxy.example.com/" + mock_request.cookies = {} + mock_request.query_params = {} + + parsed_openid = CustomOpenID( + id="user_123", + email="philip@example.com", + first_name="Philip", + last_name="Schwartz", + display_name="Philip Schwartz", + provider="generic", + team_ids=["ord-engineering-high"], + user_role=None, + ) + + raw_userinfo_with_leaked_token = { + "sub": "user_123", + "email": "philip@example.com", + "team_id": "ord-engineering-high", + "team_alias": "ord-engineering-high", + "teams": ["ord-engineering-high"], + "roles": ["litellm.api.user"], + # Defense-in-depth: a non-conforming IdP could shove a bearer token + # into userinfo. The debug endpoint must strip it before rendering. + "access_token": "should-not-render", + "id_token": "should-not-render-either", + } + + access_token_payload = { + "sub": "user_123", + "scope": "openid profile email", + "groups": ["litellm-users"], + } + + async def fake_get_generic_sso_response(**kwargs): + return parsed_openid, raw_userinfo_with_leaked_token, access_token_payload + + with ( + patch.dict( + os.environ, + {"GENERIC_CLIENT_ID": "test_client_id"}, + clear=False, + ), + patch( + "litellm.proxy.management_endpoints.ui_sso.get_generic_sso_response", + side_effect=fake_get_generic_sso_response, + ), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.jwt_handler", MagicMock(spec=JWTHandler)), + ): + # Microsoft / Google envs may leak in from other tests — ensure only + # the generic path runs. + for var in ("MICROSOFT_CLIENT_ID", "GOOGLE_CLIENT_ID"): + os.environ.pop(var, None) + response = await debug_sso_callback(mock_request) + + body = response.body.decode() + + # The embedded JSON payload drives the rendered page. Extract and parse it + # so we can assert on shape, not on cosmetic HTML details. + marker = "const ssoData = " + start = body.index(marker) + len(marker) + end = body.index(";", start) + while body[end - 1] not in "}]": # handle ';' inside string values + end = body.index(";", end + 1) + payload = json.loads(body[start:end]) + + assert set(payload.keys()) == { + "parsed_by_proxy", + "raw_claims", + "access_token_claims", + } + + # Parsed OpenID fields are shown + assert payload["parsed_by_proxy"]["email"] == "philip@example.com" + assert payload["parsed_by_proxy"]["id"] == "user_123" + + # Raw IdP claims surface fields the OpenID model drops (the original LIT-2838 ask) + assert payload["raw_claims"]["team_id"] == "ord-engineering-high" + assert payload["raw_claims"]["team_alias"] == "ord-engineering-high" + assert payload["raw_claims"]["teams"] == ["ord-engineering-high"] + assert payload["raw_claims"]["roles"] == ["litellm.api.user"] + + # Defense-in-depth: bearer tokens must never appear in the rendered HTML + assert "access_token" not in payload["raw_claims"] + assert "id_token" not in payload["raw_claims"] + assert "should-not-render" not in body + + # Decoded access-token JWT claims are surfaced + assert payload["access_token_claims"]["groups"] == ["litellm-users"] + + +@pytest.mark.asyncio +async def test_debug_sso_callback_handles_missing_raw_response(): + """ + Microsoft and Google paths don't return a raw response or access-token + payload. The debug endpoint must still render successfully with empty + sections instead of crashing. + """ + from litellm.proxy.management_endpoints.ui_sso import debug_sso_callback + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "http://proxy.example.com/" + mock_request.cookies = {} + mock_request.query_params = {} + + parsed_openid = CustomOpenID( + id="user_456", + email="user@example.com", + first_name="Some", + last_name="User", + display_name="Some User", + provider="microsoft", + team_ids=[], + user_role=None, + ) + + async def fake_microsoft_callback(**kwargs): + return parsed_openid + + with ( + patch.dict( + os.environ, + {"MICROSOFT_CLIENT_ID": "test_microsoft_id"}, + clear=False, + ), + patch.object( + MicrosoftSSOHandler, + "get_microsoft_callback_response", + side_effect=fake_microsoft_callback, + ), + patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.jwt_handler", MagicMock(spec=JWTHandler)), + ): + for var in ("GENERIC_CLIENT_ID", "GOOGLE_CLIENT_ID"): + os.environ.pop(var, None) + response = await debug_sso_callback(mock_request) + + assert response.status_code == 200 + body = response.body.decode() + assert '"raw_claims": {}' in body + assert '"access_token_claims": {}' in body + assert "user@example.com" in body diff --git a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py index 6cab5baee9a..1d0c0f90fd1 100644 --- a/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_prometheus_auth_middleware.py @@ -121,6 +121,26 @@ def test_invalid_auth_metrics(app_with_middleware, monkeypatch): assert "Unauthorized access to metrics endpoint" in response.text +def test_invalid_auth_metrics_includes_optout_hint(app_with_middleware, monkeypatch): + """ + The 401 body must tell operators how to restore the previous unauthenticated + behavior, otherwise a Prometheus scraper that worked pre-upgrade just sees + "Malformed API Key" with no actionable migration path. + """ + monkeypatch.setattr(litellm, "require_auth_for_metrics_endpoint", True) + monkeypatch.setattr( + "litellm.proxy.middleware.prometheus_auth_middleware.user_api_key_auth", + fake_invalid_auth, + ) + + client = TestClient(app_with_middleware) + response = client.get("/metrics") + + assert response.status_code == 401, response.text + assert "require_auth_for_metrics_endpoint" in response.text + assert "false" in response.text + + def test_metrics_auth_uses_real_auth_when_route_is_public( app_with_middleware, monkeypatch ): diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 070b232066a..aa0f8d63274 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -585,15 +585,24 @@ async def test_should_cap_known_estimate_to_remaining_budget( @pytest.mark.asyncio -async def test_should_reserve_remaining_budget_when_output_cap_missing( +async def test_should_clamp_reservation_to_default_when_output_cap_missing( spend_counter_state, ): + """When max_tokens is not specified, _estimate_output_tokens falls back to + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK (16K), clamped by the model's + max_output_tokens. Reservation must be a bounded per-request amount + (mirroring parallel_request_limiter_v3's DEFAULT_MAX_TOKENS_ESTIMATE), + not the entire remaining headroom.""" + from litellm.proxy.spend_tracking.budget_reservation import ( + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK, + ) + counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( token="key-budget-uncapped", spend=0.2, - max_budget=1.0, + max_budget=10000.0, ) await key_cache.async_set_cache( key="key-budget-uncapped", @@ -602,22 +611,24 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( request_body = _request_body() request_body.pop("max_tokens") + output_cost_per_token = 1e-5 # roughly Opus 4.5/4.7 output rate + expected_cost = DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK * output_cost_per_token + with patch( "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", return_value={ "input_cost_per_token": 0.0, - "output_cost_per_token": 100.0, - "max_output_tokens": 200000, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": 200000, # well above the 16K fallback }, ): - assert ( - estimate_request_max_cost( - request_body=request_body, - route="/chat/completions", - llm_router=None, - ) - is None + estimated = estimate_request_max_cost( + request_body=request_body, + route="/chat/completions", + llm_router=None, ) + assert estimated == pytest.approx(expected_cost) + reservation = await reserve_budget_for_request( request_body=request_body, route="/chat/completions", @@ -631,47 +642,45 @@ async def test_should_reserve_remaining_budget_when_output_cap_missing( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) @pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_when_counter_advances( +async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state, - monkeypatch, ): + """An adversarial caller sending max_tokens=999_999_999 must not be able + to inflate the per-request reservation up to the entire remaining team + headroom. _estimate_output_tokens clamps the explicit value at the + model's max_output_tokens — the model can only physically emit that + many tokens anyway, so anything more is both wasteful and a DoS surface.""" counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-race", - spend=0.2, - max_budget=1.0, + token="key-budget-overrequest", + spend=0.0, + max_budget=10000.0, ) + await key_cache.async_set_cache( + key="key-budget-overrequest", + value=valid_token, + ) + request_body = _request_body() - request_body.pop("max_tokens") + request_body["max_tokens"] = 999_999_999 - from litellm.proxy.spend_tracking import budget_reservation - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=0.3, - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, - ) + output_cost_per_token = 1e-5 + model_ceiling = 128_000 + expected_cost = model_ceiling * output_cost_per_token with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 0.0, + "output_cost_per_token": output_cost_per_token, + "max_output_tokens": model_ceiling, + }, ): reservation = await reserve_budget_for_request( request_body=request_body, @@ -686,66 +695,91 @@ async def test_should_shrink_uncapped_reservation_when_counter_advances( ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.7) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(1.0) - + assert reservation["reserved_cost"] == pytest.approx(expected_cost) await release_budget_reservation(reservation) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-uncapped-race" - ) == pytest.approx(0.3) - @pytest.mark.asyncio -async def test_should_shrink_uncapped_reservation_multiple_times( +async def test_should_reserve_image_generation_cost_per_image( spend_counter_state, - monkeypatch, ): + """Image-generation requests reserve `n × per-image cost` so concurrent + requests against a depleted budget cannot all bypass the admission gate. + The OpenAI ``dall-e-3`` entry exposes the per-image price as + ``input_cost_per_image`` (a naming quirk), while other providers use + ``output_cost_per_image`` — both must be honored.""" counter_cache, key_cache = spend_counter_state proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) valid_token = UserAPIKeyAuth( - token="key-budget-double-resize", - spend=0.2, - max_budget=1.0, - team_id="team-budget-double-resize", + token="key-image-gen", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-gen", value=valid_token) + + request_body = {"model": "dall-e-3", "prompt": "a cat", "n": 3} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.12) # 3 × $0.04 + await release_budget_reservation(reservation) + + +@pytest.mark.asyncio +async def test_should_reject_concurrent_image_request_against_depleted_budget( + spend_counter_state, +): + """Greptile P1 regression: with image-gen reservation in place, a second + concurrent image request against a budget already pinned at the cap by + the first reservation must raise BudgetExceededError instead of + silently reaching the provider.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-deplete", + spend=0.0, + team_id="team-image-deplete", ) team_object = LiteLLM_TeamTable( - team_id="team-budget-double-resize", - spend=0.2, - max_budget=1.0, + team_id="team-image-deplete", + max_budget=0.04, + spend=0.0, ) - request_body = _request_body() - request_body.pop("max_tokens") - - from litellm.proxy.spend_tracking import budget_reservation - - stale_spend_by_counter_key = { - "spend:key:key-budget-double-resize": 0.3, - "spend:team:team-budget-double-resize": 0.4, - } - - async def stale_counter_read(counter): - await counter_cache.async_increment_cache( - key=counter.counter_key, - value=stale_spend_by_counter_key[counter.counter_key], - ) - return 0.2 - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - stale_counter_read, + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, ) + request_body = {"model": "dall-e-3", "prompt": "a cat"} + with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_image": 0.04, + }, ): - reservation = await reserve_budget_for_request( + first = await reserve_budget_for_request( request_body=request_body, - route="/chat/completions", + route="/v1/images/generations", llm_router=None, valid_token=valid_token, team_object=team_object, @@ -754,32 +788,167 @@ async def test_should_shrink_uncapped_reservation_multiple_times( user_api_key_cache=key_cache, proxy_logging_obj=proxy_logging_obj, ) + assert first is not None + + with pytest.raises(litellm.BudgetExceededError): + await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + await release_budget_reservation(first) + + +@pytest.mark.asyncio +async def test_should_skip_reservation_for_per_pixel_image_model( + spend_counter_state, +): + """DALL-E 2-style per-pixel pricing depends on the requested ``size``, + which we don't decode here. Fall through to read-time enforcement + rather than guess.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-per-pixel", + spend=0.0, + max_budget=1.0, + ) + await key_cache.async_set_cache(key="key-image-per-pixel", value=valid_token) + + request_body = {"model": "dall-e-2", "prompt": "a cat", "size": "256x256"} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_generation", + "input_cost_per_pixel": 2.4414e-07, + "output_cost_per_pixel": 0.0, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/generations", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is None + + +@pytest.mark.asyncio +async def test_should_use_token_pricing_for_chat_model_with_image_cost_field( + spend_counter_state, +): + """Several chat and embedding models carry ``input_cost_per_image`` / + ``output_cost_per_image`` to price multimodal vision *input*, not image + generation (e.g. gemini-3.1-pro-preview, azure/gpt-realtime-*, + amazon.titan-embed-image-v1). _estimate_image_generation_cost must gate + on ``mode`` so these models still go through the token-priced path — + otherwise a long chat reserves a fraction of a cent instead of the true + token cost.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-multimodal-chat", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-multimodal-chat", value=valid_token) + + # Roughly the gemini-3.1-pro-preview shape: chat-mode model that + # carries an output_cost_per_image alongside token pricing. + output_cost_per_token = 1.2e-5 + request_body = { + "model": "gemini-3.1-pro-preview", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1000, + } + expected_cost = 1000 * output_cost_per_token # token-priced path, not 1 × $0.00012 + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": output_cost_per_token, + "output_cost_per_image": 0.00012, + "max_output_tokens": 64000, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.6) - assert [entry["reserved_cost"] for entry in reservation["entries"]] == [ - pytest.approx(0.6), - pytest.approx(0.6), - ] - assert [entry["applied_adjustment"] for entry in reservation["entries"]] == [ - pytest.approx(0.0), - pytest.approx(0.0), - ] - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.9) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(1.0) - + # Token-priced path: reservation ≈ output_tokens × output_cost_per_token, + # plus a small input-token contribution. Must NOT collapse to the + # per-image price ($0.00012) which would indicate the image-gen branch + # incorrectly fired for this chat model. + assert reservation["reserved_cost"] == pytest.approx(expected_cost, rel=0.05) + assert reservation["reserved_cost"] > 0.001 # well above per-image price await release_budget_reservation(reservation) - assert counter_cache.in_memory_cache.get_cache( - key="spend:key:key-budget-double-resize" - ) == pytest.approx(0.3) - assert counter_cache.in_memory_cache.get_cache( - key="spend:team:team-budget-double-resize" - ) == pytest.approx(0.4) + +@pytest.mark.asyncio +async def test_should_reserve_image_edit_cost_per_image( + spend_counter_state, +): + """``image_edit`` models (Flux Kontext, Stability inpaint/outpaint, etc.) + bill per generated image just like ``image_generation`` and must get + the same atomic per-image reservation.""" + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + valid_token = UserAPIKeyAuth( + token="key-image-edit", + spend=0.0, + max_budget=10.0, + ) + await key_cache.async_set_cache(key="key-image-edit", value=valid_token) + + request_body = {"model": "stability/inpaint", "prompt": "a cat", "n": 2} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "mode": "image_edit", + "output_cost_per_image": 0.05, + }, + ): + reservation = await reserve_budget_for_request( + request_body=request_body, + route="/v1/images/edits", + llm_router=None, + valid_token=valid_token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert reservation is not None + assert reservation["reserved_cost"] == pytest.approx(0.10) # 2 × $0.05 + await release_budget_reservation(reservation) def test_should_start_window_without_reset_at_at_duration_boundary(): @@ -1047,62 +1216,6 @@ async def test_should_release_tracked_entry_when_reservation_fails_after_increme ) == pytest.approx(0.0) -@pytest.mark.asyncio -async def test_should_not_re_read_uncapped_budget_after_reservation_fallback( - spend_counter_state, - monkeypatch, -): - _, key_cache = spend_counter_state - proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) - valid_token = UserAPIKeyAuth( - token="key-budget-uncapped-read-once", - spend=0.2, - max_budget=1.0, - ) - - from litellm.proxy.spend_tracking import budget_reservation - - current_counter_reads = [] - - async def mock_get_current_counter_value(counter): - current_counter_reads.append(counter.counter_key) - return counter.fallback_spend - - async def mock_reserve_counter(counter, reservation_cost): - return None - - monkeypatch.setattr( - budget_reservation, - "_get_current_counter_value", - mock_get_current_counter_value, - ) - monkeypatch.setattr( - budget_reservation, - "_reserve_counter", - mock_reserve_counter, - ) - - with patch( - "litellm.proxy.spend_tracking.budget_reservation.estimate_request_max_cost", - return_value=None, - ): - reservation = await reserve_budget_for_request( - request_body=_request_body(), - route="/chat/completions", - llm_router=None, - valid_token=valid_token, - team_object=None, - user_object=None, - prisma_client=None, - user_api_key_cache=key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - assert reservation is not None - assert reservation["reserved_cost"] == pytest.approx(0.8) - assert current_counter_reads == ["spend:key:key-budget-uncapped-read-once"] - - @pytest.mark.asyncio async def test_should_reconcile_reserved_counter_to_actual_spend( spend_counter_state, @@ -1492,4 +1605,94 @@ async def test_should_reserve_all_budgeted_counters(spend_counter_state): counter_cache.in_memory_cache.get_cache(key="spend:team:team-budget-all") == 0.3 ) - await release_budget_reservation(reservation) + +@pytest.mark.asyncio +async def test_should_not_block_concurrent_team_request_when_first_request_lacks_max_tokens( + spend_counter_state, +): + """ + Regression test: a team-bound request with no max_tokens must not pin the + team's spend counter at max_budget for the duration of the request. + + Repro of the integration-test team being falsely budget-blocked at the + $2000 cap while DB spend is $0.144: the first request without max_tokens + used to reserve the entire remaining headroom, leaving any subsequent + request stuck behind a counter sitting at the cap until the success + callback finished reconciling. + """ + counter_cache, key_cache = spend_counter_state + proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache) + + valid_token = UserAPIKeyAuth( + token="key-team-integration-tests", + spend=0.0, + team_id="team-integration-tests", + ) + team_object = LiteLLM_TeamTable( + team_id="team-integration-tests", + max_budget=2000.0, + spend=0.144, + ) + await key_cache.async_set_cache( + key=f"team_id:{team_object.team_id}", + value=team_object, + ) + + request_body = _request_body() + request_body.pop("max_tokens") + + # Realistic Opus 4.7 output pricing — the 16K fallback × $25/M ≈ $0.40 + # reservation per request, leaving ~5000 admittable concurrent requests + # against a $2000 team budget. + with patch( + "litellm.proxy.spend_tracking.budget_reservation._get_model_cost_info", + return_value={ + "input_cost_per_token": 5e-6, + "output_cost_per_token": 2.5e-5, + "max_output_tokens": 128000, + }, + ): + first_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + # The team counter must not be pinned at max_budget while the first + # request is in flight, otherwise concurrent requests false-positive. + team_counter_after_first = ( + counter_cache.in_memory_cache.get_cache( + key=f"spend:team:{team_object.team_id}" + ) + or 0.0 + ) + assert team_counter_after_first < team_object.max_budget, ( + f"Team counter sat at {team_counter_after_first} after one uncapped " + f"reservation against a {team_object.max_budget} budget — concurrent " + "requests will be falsely blocked." + ) + + # Second request — same shape — must succeed without raising. + second_reservation = await reserve_budget_for_request( + request_body=request_body, + route="/chat/completions", + llm_router=None, + valid_token=valid_token, + team_object=team_object, + user_object=None, + prisma_client=None, + user_api_key_cache=key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert second_reservation is not None + + if first_reservation is not None: + await release_budget_reservation(first_reservation) + if second_reservation is not None: + await release_budget_reservation(second_reservation) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 92611431a15..b803dfb709a 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -21,6 +21,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _get_enforced_params, _get_metadata_variable_name, _resolve_credential_from_model_config, + _resolve_provider_from_deployment, _update_model_if_key_alias_exists, add_guardrails_from_policy_engine, add_litellm_data_to_request, @@ -4043,3 +4044,174 @@ def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata(): assert result == [ "my-guardrail" ], f"Expected guardrails from litellm_metadata fallback, got: {result}" + + +# ============================================================================ +# Tests for #27516: provider hint resolution from deployment when the +# user-facing model name has no provider prefix. +# ============================================================================ + + +def test_resolve_provider_from_deployment_uses_litellm_params_model(): + """When custom_llm_provider is unset, fall back to the prefix of model.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "bedrock/us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = None + router.get_deployment_by_model_group_name.return_value = deployment + + assert ( + _resolve_provider_from_deployment(router, "claude-sonnet-4.6") == "bedrock" + ) + + +def test_resolve_provider_from_deployment_prefers_custom_llm_provider(): + """Explicit custom_llm_provider on the deployment wins over model prefix.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = "bedrock" + router.get_deployment_by_model_group_name.return_value = deployment + + assert ( + _resolve_provider_from_deployment(router, "claude-sonnet-4.6") == "bedrock" + ) + + +def test_resolve_provider_from_deployment_no_match(): + """No deployment for the model group -> None.""" + router = MagicMock() + router.get_deployment_by_model_group_name.return_value = None + assert _resolve_provider_from_deployment(router, "unknown-model") is None + + +def test_resolve_provider_from_deployment_router_raises(): + """Router exceptions must not propagate — fall back to None.""" + router = MagicMock() + router.get_deployment_by_model_group_name.side_effect = RuntimeError("boom") + assert _resolve_provider_from_deployment(router, "claude-sonnet-4.6") is None + + +def test_resolve_provider_from_deployment_falls_back_to_pre_alias(): + """If post-alias lookup fails, the pre-alias name is also tried.""" + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "bedrock/anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = None + + def lookup(model_group_name): + if model_group_name == "pre-alias-name": + return deployment + return None + + router.get_deployment_by_model_group_name.side_effect = lookup + + result = _resolve_provider_from_deployment( + router, "post-alias-name", pre_alias_model_name="pre-alias-name" + ) + assert result == "bedrock" + + +def test_apply_overrides_multi_provider_default_picks_correct_provider( + setup_test_credentials, +): + """ + Regression for #27516: when defaultconfig has multiple providers and the + request model has no '/' prefix, the deployment's custom_llm_provider must + drive provider matching instead of falling through to dict insertion order. + """ + litellm.credential_list.append( + CredentialItem( + credential_name="bedrock-team-1", + credential_info={}, + credential_values={"api_key": "ABSK-bedrock-key-for-team-1"}, + ) + ) + litellm.credential_list.append( + CredentialItem( + credential_name="gemini-team-1", + credential_info={}, + credential_values={"api_key": "gemini-key-for-team-1"}, + ) + ) + + data = {"model": "claude-sonnet-4.6"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + # gemini comes first in insertion order — the bug picked it. + "gemini": {"litellm_credentials": "gemini-team-1"}, + "bedrock": {"litellm_credentials": "bedrock-team-1"}, + } + } + }, + ) + + router = MagicMock() + deployment = MagicMock() + deployment.litellm_params.model = "us.anthropic.claude-sonnet-4-6" + deployment.litellm_params.custom_llm_provider = "bedrock" + router.get_deployment_by_model_group_name.return_value = deployment + + _apply_credential_overrides_from_model_config( + data=data, + user_api_key_dict=user_api_key_dict, + llm_router=router, + ) + assert data["api_key"] == "ABSK-bedrock-key-for-team-1" + + +def test_apply_overrides_no_router_keeps_legacy_behaviour(setup_test_credentials): + """ + Without a router, the function still works for the single-provider case + (the historical behaviour). Multi-provider configs with no '/' prefix + keep the legacy first-entry behaviour because there is no way to + disambiguate — this preserves backwards compatibility. + """ + data = {"model": "gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + "azure": {"litellm_credentials": "hotel-azure-eastus"} + } + } + }, + ) + _apply_credential_overrides_from_model_config( + data=data, user_api_key_dict=user_api_key_dict, llm_router=None + ) + assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" + assert data["api_key"] == "key-hotel-eastus" + + +def test_apply_overrides_provider_prefix_in_model_skips_router_lookup( + setup_test_credentials, +): + """ + When the request model already has a 'provider/...' prefix, the router + lookup must be skipped — the explicit prefix is authoritative. + """ + data = {"model": "azure/gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_metadata={ + "model_config": { + "defaultconfig": { + "azure": {"litellm_credentials": "hotel-azure-eastus"}, + "bedrock": {"litellm_credentials": "hotel-rec-azure"}, + } + } + }, + ) + + router = MagicMock() + _apply_credential_overrides_from_model_config( + data=data, user_api_key_dict=user_api_key_dict, llm_router=router + ) + assert data["api_base"] == "https://hotel-eastus.openai.azure.com/" + assert data["api_key"] == "key-hotel-eastus" + router.get_deployment_by_model_group_name.assert_not_called() diff --git a/tests/test_litellm/proxy/test_openapi_schema_validation.py b/tests/test_litellm/proxy/test_openapi_schema_validation.py index 68d537b2593..b44edc8a3bc 100644 --- a/tests/test_litellm/proxy/test_openapi_schema_validation.py +++ b/tests/test_litellm/proxy/test_openapi_schema_validation.py @@ -140,3 +140,110 @@ class TestCredentialEndpointsOpenAPISchema: assert ( "credential_name" in sig.parameters ), "get_credential_by_name must have a credential_name parameter" + + +class TestWebSocketStubInjection: + """ + Regression test for the v1.82.3 bug where adding a WebSocket route on a path + that already had an HTTP route silently dropped the HTTP operation from the + OpenAPI schema. + + Related case: 2026-05-05-madhu-swagger-responses-missing + """ + + def _make_fake_ws_route(self, path: str, name: str = "fake_ws"): + """Minimal stand-in for fastapi.routing.APIWebSocketRoute for the helper's purposes.""" + from types import SimpleNamespace + + return SimpleNamespace(path=path, name=name, dependant=None) + + def test_websocket_stub_does_not_clobber_existing_post(self): + """ + When a WebSocket route shares its path with an existing POST operation, + the POST must survive — the WebSocket stub is added alongside, not on top. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = { + "paths": { + "/v1/responses": { + "post": {"summary": "responses_api", "operationId": "responses_api"} + } + } + } + ws_routes = [self._make_fake_ws_route("/v1/responses", name="responses_ws")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert ( + "post" in result["paths"]["/v1/responses"] + ), "POST operation must be preserved when a WebSocket route shares the path" + assert ( + result["paths"]["/v1/responses"]["post"]["operationId"] == "responses_api" + ) + assert ( + "get" in result["paths"]["/v1/responses"] + ), "WebSocket stub should also be added under 'get'" + assert result["paths"]["/v1/responses"]["get"]["tags"] == ["WebSocket"] + + def test_websocket_stub_added_when_path_is_new(self): + """ + When a WebSocket route's path is not already in the schema, the stub + creates a fresh entry — preserving the original behavior for WebSocket-only + paths. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = {"paths": {}} + ws_routes = [self._make_fake_ws_route("/ws_only", name="ws_only")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert "/ws_only" in result["paths"] + assert "get" in result["paths"]["/ws_only"] + assert result["paths"]["/ws_only"]["get"]["tags"] == ["WebSocket"] + + def test_websocket_stub_skipped_when_existing_get(self): + """ + If a real GET is already documented on the path, the WebSocket stub is + skipped — a real operation always wins over the synthetic stub. This + closes the same trap for future GET-vs-WebSocket collisions. + """ + from litellm.proxy.proxy_server import ( + _inject_websocket_stubs_into_openapi_schema, + ) + + schema = { + "paths": { + "/health": { + "get": {"summary": "health_check", "operationId": "real_get"} + } + } + } + ws_routes = [self._make_fake_ws_route("/health", name="health_ws")] + + result = _inject_websocket_stubs_into_openapi_schema(schema, ws_routes) + + assert ( + result["paths"]["/health"]["get"]["operationId"] == "real_get" + ), "Real GET must take precedence over WebSocket stub" + + def test_responses_post_routes_registered_on_router(self): + """ + Sanity check: the three POST routes for the responses API are still wired + on the responses router. Guards against accidental removal at the source. + """ + from litellm.proxy.response_api_endpoints.endpoints import router + + post_paths = { + route.path + for route in router.routes + if hasattr(route, "methods") + and "POST" in (route.methods or set()) + and route.path in {"/v1/responses", "/responses", "/openai/v1/responses"} + } + assert post_paths == {"/v1/responses", "/responses", "/openai/v1/responses"} diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 59b43330a25..327200a6a95 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -387,6 +387,102 @@ class TestProxyInitializationHelpers: ), f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() + @pytest.mark.parametrize( + "timeout_config,expected_timeout", + [ + ({"database_connection_timeout": 30}, 30), + ({"database_connection_pool_timeout": 45}, 45), + ( + { + "database_connection_timeout": 30, + "database_connection_pool_timeout": 45, + }, + 30, + ), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_db_timeout_settings_are_forwarded_to_pool_timeout( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + timeout_config, + expected_timeout, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_connection_pool_limit": 5, + **timeout_config, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL") + } + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: ( + f"{url}?connection_limit={params['connection_limit']}&pool_timeout={params['pool_timeout']}" + ), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + mock_append_query_params.assert_called() + appended_params = mock_append_query_params.call_args.args[1] + assert appended_params["connection_limit"] == 5 + assert appended_params["pool_timeout"] == expected_timeout + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 3f19db36c3f..859594f7a0b 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1728,6 +1728,67 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): assert call_args.kwargs["query_type"] == "update_data" +@pytest.mark.asyncio +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): + """ + Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional + update_many with `WHERE budget_reset_at IS NULL` to backfill the column on + rows that pre-existed without a reset schedule. Without this, the proxy + admin row stays at NULL and reset_budget_for_litellm_users never matches + it (NULL < now() is unknown in SQL), so the global proxy budget never + resets. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + import litellm + from litellm.proxy.proxy_server import ProxyStartupEvent + + litellm.budget_duration = "30d" + litellm.max_budget = 100.0 + litellm_proxy_budget_name = "litellm-proxy-budget" + + mock_prisma = MagicMock() + mock_prisma.db.litellm_usertable.update_many = AsyncMock(return_value={"count": 1}) + + mock_generate_key_helper = AsyncMock( + return_value={ + "user_id": litellm_proxy_budget_name, + "max_budget": 100.0, + "budget_duration": "30d", + "spend": 0, + "models": [], + } + ) + + with ( + patch( + "litellm.proxy.proxy_server.generate_key_helper_fn", + mock_generate_key_helper, + ), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + ): + await ProxyStartupEvent._upsert_proxy_budget_with_reset_at_backfill( + litellm_proxy_budget_name + ) + + # Upsert ran with the configured budget + mock_generate_key_helper.assert_called_once() + + # Backfill update_many ran with the conditional WHERE + mock_prisma.db.litellm_usertable.update_many.assert_called_once() + backfill_call = mock_prisma.db.litellm_usertable.update_many.call_args + assert backfill_call.kwargs["where"]["user_id"] == litellm_proxy_budget_name + assert backfill_call.kwargs["where"]["budget_reset_at"] is None + + # The backfilled value must be a real future datetime — anything else and + # reset_budget_for_litellm_users would still skip the row. + from datetime import datetime, timezone + + backfilled_reset_at = backfill_call.kwargs["data"]["budget_reset_at"] + assert isinstance(backfilled_reset_at, datetime) + assert backfilled_reset_at > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_custom_ui_sso_sign_in_handler_config_loading(): """ @@ -6513,3 +6574,37 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): finally: ps.spend_counter_cache = orig_counter ps.prisma_client = orig_prisma + + +def test_realtime_websocket_route_aliases_registered(): + """Realtime sessions reach the proxy via three path aliases stacked on + `realtime_websocket_endpoint`. Dropping any of them silently 405s + WebSocket upgrades because the catch-all `/openai/{endpoint:path}` + HTTP passthrough only declares HTTP methods. The aliases must also be + in `LiteLLMRoutes.openai_routes` (so non-admin / team / key-scoped + auth allows them) and in `API_ROUTE_TO_CALL_TYPES` (so call-type-aware + logic such as guardrails can resolve the realtime call type).""" + from starlette.routing import WebSocketRoute + + from litellm.proxy._types import LiteLLMRoutes + from litellm.proxy.proxy_server import app + from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes + + websocket_paths = { + route.path for route in app.routes if isinstance(route, WebSocketRoute) + } + openai_routes = LiteLLMRoutes.openai_routes.value + + for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + assert expected in websocket_paths, ( + f"{expected!r} missing from registered WebSocket routes; the " + f"realtime endpoint will 405 for clients hitting this path." + ) + assert expected in openai_routes, ( + f"{expected!r} missing from LiteLLMRoutes.openai_routes; " + f"non-admin / team / key-scoped users will get 403 on this path." + ) + assert API_ROUTE_TO_CALL_TYPES.get(expected) == [CallTypes.arealtime], ( + f"{expected!r} missing from API_ROUTE_TO_CALL_TYPES; call-type " + f"resolution will return None and break call-type-aware features." + ) diff --git a/tests/test_litellm/proxy/test_shared_health_check.py b/tests/test_litellm/proxy/test_shared_health_check.py index 04099f2634b..1530d336085 100644 --- a/tests/test_litellm/proxy/test_shared_health_check.py +++ b/tests/test_litellm/proxy/test_shared_health_check.py @@ -322,13 +322,13 @@ class TestSharedHealthCheckManager: async def test_perform_shared_health_check_lock_failed_then_cache( self, shared_health_manager, mock_redis_cache ): - """Test performing shared health check when lock fails but cache becomes available""" + """Test performing shared health check when lock fails but cache becomes available during polling""" # First call: no cache, lock fails - # Second call: cache available + # Polling finds cache on first iteration mock_redis_cache.async_get_cache.side_effect = [ - None, # No cache initially + None, # No cache initially (get_cached_health_check_results) json.dumps( - { # Cache available after waiting + { # Cache available on first poll iteration "healthy_endpoints": [{"model": "cached-model"}], "unhealthy_endpoints": [], "healthy_count": 1, @@ -350,18 +350,68 @@ class TestSharedHealthCheckManager: ) ) - # Should wait and then get cached results - mock_sleep.assert_called_once_with(2) + # Should poll once (5s interval) and find cached results + mock_sleep.assert_called_once_with(5) assert healthy == [{"model": "cached-model"}] assert unhealthy == [] @pytest.mark.asyncio - async def test_perform_shared_health_check_fallback( + async def test_perform_shared_health_check_fallback(self, mock_redis_cache): + """Test performing shared health check with fallback to local health check""" + # Use short lock_ttl so the polling loop only runs 2 iterations + manager = SharedHealthCheckManager( + redis_cache=mock_redis_cache, + health_check_ttl=300, + lock_ttl=10, + ) + + # No cache ever, lock always held by another pod + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check + "other_pod", # Iteration 1: lock check (still held) + None, # Iteration 2: cache check + "other_pod", # Iteration 2: lock check (still held) + ] + mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + expected_healthy = [{"model": "test-model", "status": "healthy"}] + expected_unhealthy = [] + + with ( + patch("asyncio.sleep") as mock_sleep, + patch( + "litellm.proxy.health_check_utils.shared_health_check_manager.perform_health_check" + ) as mock_perform, + ): + mock_perform.return_value = (expected_healthy, expected_unhealthy, {}) + + healthy, unhealthy, _ = await manager.perform_shared_health_check( + model_list, details=True + ) + + # Should poll twice (5s * 2 = 10s >= lock_ttl) then fall back + assert mock_sleep.call_count == 2 + mock_sleep.assert_called_with(5) + mock_perform.assert_called_once_with( + model_list=model_list, details=True, max_concurrency=None + ) + assert healthy == expected_healthy + assert unhealthy == expected_unhealthy + + @pytest.mark.asyncio + async def test_perform_shared_health_check_early_exit_orphaned_lock( self, shared_health_manager, mock_redis_cache ): - """Test performing shared health check with fallback to local health check""" - # No cache, lock fails, no cache after waiting - mock_redis_cache.async_get_cache.return_value = None + """Test that polling exits early when the lock disappears without a cache write (crash recovery)""" + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check (still no cache) + None, # Iteration 1: lock check -> lock gone (holder crashed) + ] mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails model_list = [ @@ -384,8 +434,77 @@ class TestSharedHealthCheckManager: ) ) - # Should fall back to local health check - mock_sleep.assert_called_once_with(2) + # Should detect orphaned lock after 1 iteration and fall back immediately + mock_sleep.assert_called_once_with(5) + mock_perform.assert_called_once_with( + model_list=model_list, details=True, max_concurrency=None + ) + assert healthy == expected_healthy + assert unhealthy == expected_unhealthy + + @pytest.mark.asyncio + async def test_perform_shared_health_check_redis_error_during_polling( + self, shared_health_manager, mock_redis_cache + ): + """Test that a transient Redis error during lock polling doesn't crash the loop""" + cached_data = json.dumps( + { + "healthy_endpoints": [{"model": "cached-model"}], + "unhealthy_endpoints": [], + "healthy_count": 1, + "unhealthy_count": 0, + "timestamp": time.time() - 100, + } + ) + mock_redis_cache.async_get_cache.side_effect = [ + None, # Initial cache check + None, # Iteration 1: cache check + Exception("Redis connection lost"), # Iteration 1: lock check errors + cached_data, # Iteration 2: cache check -> found! + ] + mock_redis_cache.async_set_cache.return_value = False # Lock acquisition fails + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + + with patch("asyncio.sleep") as mock_sleep: + healthy, unhealthy, _ = ( + await shared_health_manager.perform_shared_health_check( + model_list, details=True + ) + ) + + # Should survive the Redis error and find cache on iteration 2 + assert mock_sleep.call_count == 2 + assert healthy == [{"model": "cached-model"}] + assert unhealthy == [] + + @pytest.mark.asyncio + async def test_perform_shared_health_check_no_redis_skips_polling(self): + """Test that polling is skipped entirely when redis_cache is None""" + manager = SharedHealthCheckManager(redis_cache=None) + + model_list = [ + {"model_name": "test-model", "litellm_params": {"model": "test-model"}} + ] + expected_healthy = [{"model": "test-model", "status": "healthy"}] + expected_unhealthy = [] + + with ( + patch("asyncio.sleep") as mock_sleep, + patch( + "litellm.proxy.health_check_utils.shared_health_check_manager.perform_health_check" + ) as mock_perform, + ): + mock_perform.return_value = (expected_healthy, expected_unhealthy, {}) + + healthy, unhealthy, _ = await manager.perform_shared_health_check( + model_list, details=True + ) + + # Should NOT sleep at all — falls back to local health check immediately + mock_sleep.assert_not_called() mock_perform.assert_called_once_with( model_list=model_list, details=True, max_concurrency=None ) diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index cbc4a920245..1f6848d3b5b 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -791,3 +791,384 @@ def test_encrypted_content_wrapping_empty_string(): assert extracted_model_id == model_id assert unwrapped == original_content + + +# --------------------------------------------------------------------------- +# LIT-2531: cross-model-group fallback via encryption boundary (api_base + api_key) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_affinity_falls_back_to_same_encryption_boundary_on_model_group_switch(): + """ + LIT-2531: Client starts a session on gpt-5.3-codex, follow-up switches to + gpt-5.4 mid-chat (e.g. via Codex `model_migrations`). Affinity must pin to + the gpt-5.4 deployment on the SAME Azure resource as the originating + gpt-5.3-codex deployment -- otherwise Azure rejects the encrypted_content. + """ + first_resp = _build_mock_response( + output_items=[ + { + "type": "reasoning", + "id": "rs_encrypted_xyz", + "status": "completed", + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + response_id="resp_first", + ) + second_resp = _build_mock_response( + output_items=[ + { + "type": "message", + "id": "msg_ok", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + }, + ], + response_id="resp_second", + ) + + ACCOUNT_A_BASE = "https://account-a.openai.azure.com/" + ACCOUNT_A_KEY = "key-a" + ACCOUNT_B_BASE = "https://account-b.openai.azure.com/" + ACCOUNT_B_KEY = "key-b" + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-a"}, + }, + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-b"}, + }, + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "azure/gpt-5.4", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.4-account-a"}, + }, + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "azure/gpt-5.4", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.4-account-b"}, + }, + ], + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + + def first_call_picks_account_a(seq): + for d in seq: + if d["model_info"]["id"] == "gpt-5.3-codex-account-a": + return d + return seq[0] + + with ( + patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), + patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=first_call_picks_account_a, + ), + ): + r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") + + assert r1._hidden_params["model_id"] == "gpt-5.3-codex-account-a" + encoded_id = _extract_encoded_item_id(r1) + assert encoded_id.startswith("encitem_") + + # simple_shuffle.random.choice NOT patched: prove affinity narrows the + # candidate pool to a single deployment regardless of which one shuffle picks. + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=second_resp, + ): + r2 = await router.aresponses( + model="gpt-5.4", + input=[ + { + "type": "reasoning", + "id": encoded_id, + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + ) + + assert r2._hidden_params["model_id"] == "gpt-5.4-account-a" + + +@pytest.mark.asyncio +async def test_affinity_falls_back_to_same_boundary_on_alias_switch(): + """ + LIT-2531 alias path: gpt-5.2-codex is a LiteLLM alias that points at the + same underlying Azure model as gpt-5.3-codex. Different model_name groups + in the router, so model_id-based pinning misses, but the encryption + boundary (api_base + api_key) is identical -> follow-up must still pin. + """ + first_resp = _build_mock_response( + output_items=[ + { + "type": "reasoning", + "id": "rs_alias_xyz", + "status": "completed", + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + response_id="resp_alias_first", + ) + second_resp = _build_mock_response( + output_items=[ + { + "type": "message", + "id": "msg_alias_ok", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}], + }, + ], + response_id="resp_alias_second", + ) + + ACCOUNT_A_BASE = "https://account-a.openai.azure.com/" + ACCOUNT_A_KEY = "key-a" + ACCOUNT_B_BASE = "https://account-b.openai.azure.com/" + ACCOUNT_B_KEY = "key-b" + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-a"}, + }, + { + "model_name": "gpt-5.3-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.3-codex-account-b"}, + }, + { + "model_name": "gpt-5.2-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_A_BASE, + "api_key": ACCOUNT_A_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.2-codex-account-a"}, + }, + { + "model_name": "gpt-5.2-codex", + "litellm_params": { + "model": "azure/gpt-5.3-codex", + "api_base": ACCOUNT_B_BASE, + "api_key": ACCOUNT_B_KEY, + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": "gpt-5.2-codex-account-b"}, + }, + ], + optional_pre_call_checks=["encrypted_content_affinity"], + num_retries=0, + ) + + def pick_account_a(seq): + for d in seq: + if d["model_info"]["id"] == "gpt-5.3-codex-account-a": + return d + return seq[0] + + with ( + patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=first_resp, + ), + patch( + "litellm.router_strategy.simple_shuffle.random.choice", + side_effect=pick_account_a, + ), + ): + r1 = await router.aresponses(model="gpt-5.3-codex", input="hi") + + encoded_id = _extract_encoded_item_id(r1) + assert encoded_id.startswith("encitem_") + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.async_response_api_handler", + new_callable=AsyncMock, + return_value=second_resp, + ): + r2 = await router.aresponses( + model="gpt-5.2-codex", + input=[ + { + "type": "reasoning", + "id": encoded_id, + "encrypted_content": "gAAAAABpnW_yEYmSNEyOG...", + }, + ], + ) + + assert r2._hidden_params["model_id"] == "gpt-5.2-codex-account-a" + + +def test_boundary_fallback_no_router_ref_returns_empty(): + """ + Standalone use (no router wired in) -> the boundary lookup short-circuits + to ``[]`` instead of crashing on ``None.get_deployment``. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + check = EncryptedContentAffinityCheck(router=None) + healthy = [ + { + "model_info": {"id": "dep-1"}, + "litellm_params": {"api_base": "https://x", "api_key": "k"}, + } + ] + matches = check._find_deployments_on_same_encryption_boundary( + healthy_deployments=healthy, + model_id="dep-2", + ) + assert matches == [] + + +def test_boundary_fallback_originating_deployment_removed_returns_empty(): + """ + If the originating deployment has been removed from the router (e.g. via + /model/delete), ``router.get_deployment`` returns None and we return [] so + the caller falls back to the full healthy_deployments list. + """ + from unittest.mock import MagicMock + + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + mock_router = MagicMock() + mock_router.get_deployment.return_value = None + + check = EncryptedContentAffinityCheck(router=mock_router) + healthy = [ + { + "model_info": {"id": "dep-1"}, + "litellm_params": {"api_base": "https://x", "api_key": "k"}, + } + ] + matches = check._find_deployments_on_same_encryption_boundary( + healthy_deployments=healthy, + model_id="dep-removed", + ) + assert matches == [] + mock_router.get_deployment.assert_called_once_with(model_id="dep-removed") + + +def test_boundary_key_accepts_pydantic_litellm_params_instance(): + """ + Regression: ``_encryption_boundary_key`` must accept any object exposing + dict-style ``.get()`` (incl. ``LiteLLM_Params`` Pydantic instances) — not + just plain dicts. + + A stricter ``isinstance(dict)`` guard would silently return ``None`` for a + ``LiteLLM_Params`` value, drop the deployment from boundary matching, and + fall back to the full pool — which is the exact ``invalid_encrypted_content`` + failure this check exists to prevent. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + from litellm.types.router import LiteLLM_Params + + pydantic_params = LiteLLM_Params( + model="azure/gpt-5.3-codex", + api_base="https://mateo-resource.openai.azure.com", + api_key="fake-azure-resource-key-a", + ) + plain_params = { + "model": "azure/gpt-5.3-codex", + "api_base": "https://mateo-resource.openai.azure.com", + "api_key": "fake-azure-resource-key-a", + } + + pydantic_key = EncryptedContentAffinityCheck._encryption_boundary_key( + pydantic_params + ) + plain_key = EncryptedContentAffinityCheck._encryption_boundary_key(plain_params) + + assert pydantic_key is not None + assert ( + pydantic_key + == plain_key + == ( + "https://mateo-resource.openai.azure.com", + "fake-azure-resource-key-a", + ) + ) + + +def test_boundary_key_rejects_non_dict_like_inputs(): + """ + Inputs that don't expose ``.get()`` (None, lists, strings, ints) -> None. + Guards against accidentally treating a stray non-dict-like value as a + valid boundary. + """ + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + for bad in (None, [], "not a dict", 42, object()): + assert EncryptedContentAffinityCheck._encryption_boundary_key(bad) is None + + assert ( + EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "", "api_key": "k"} + ) + is None + ) + assert ( + EncryptedContentAffinityCheck._encryption_boundary_key( + {"api_base": "https://x"} + ) + is None + ) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ebe175b2503..e53484dd287 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -110,6 +110,25 @@ def test_wandb_model_api_pricing_entries(): assert model_info["output_cost_per_token"] == output_cost +def test_openrouter_qwen36_plus_model_info(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_info = litellm.model_cost.get("openrouter/qwen/qwen3.6-plus") + + assert model_info is not None + assert model_info["litellm_provider"] == "openrouter" + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 1000000 + assert model_info["max_output_tokens"] == 65536 + assert model_info["input_cost_per_token"] == 3.25e-07 + assert model_info["output_cost_per_token"] == 1.95e-06 + assert model_info["supports_function_calling"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_reasoning"] is True + assert model_info["supports_vision"] is True + + def test_cost_calculator_with_usage(monkeypatch): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4358d0dc193..b03579c2dbd 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -757,6 +757,60 @@ def test_responses_api_bridge_check_gpt_5_4_tools_without_reasoning_stays_chat() assert model_info.get("mode") != "responses" +def test_responses_api_bridge_check_gpt_5_4_reasoning_summary_without_tools_routes_to_responses(): + """gpt-5.4+ with reasoning_effort + reasoningSummary but no tools should bridge (AI SDK).""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider="openai", + tools=None, + reasoning_effort="medium", + reasoning_summary="auto", + ) + + assert model == "gpt-5.4" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_reasoning_summary_routes_to_responses(): + """Bare ``gpt-5`` with reasoning_effort + reasoningSummary should bridge (not 5.4+).""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5", + custom_llm_provider="openai", + tools=None, + reasoning_effort="medium", + reasoning_summary="auto", + ) + + assert model == "gpt-5" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_gpt_5_tools_without_summary_stays_chat(): + """gpt-5 with tools + reasoning_effort but no summary should stay on chat.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 128000} + model_info, model = responses_api_bridge_check( + model="gpt-5", + custom_llm_provider="openai", + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort="medium", + reasoning_summary=None, + ) + + assert model == "gpt-5" + assert model_info.get("mode") != "responses" + + @patch("litellm.completion_extras.responses_api_bridge.completion") def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( mock_responses_completion, @@ -794,6 +848,132 @@ def test_gpt_5_4_responses_bridge_preserves_reasoning_summary_dict( } +@pytest.mark.parametrize( + "model, model_info, expected_model_param", + [ + ("gemini/gemini-3.1-pro", None, "gemini-3.1-pro"), + ( + "gemini/gemini-3.1-pro", + {"base_model": "gemini-3.1-pro-preview"}, + "gemini-3.1-pro-preview", + ), + ], +) +def test_completion_optional_params_base_model( + model: str, + model_info: dict | None, + expected_model_param: str, +): + with patch("litellm.main.get_optional_params") as mock_get_optional_params: + mock_get_optional_params.return_value = MagicMock() + + import litellm + + kwargs = { + "model": model, + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "api_key": "fake-key", + "mock_response": "Hey, how's it going?", + } + if model_info is not None: + kwargs["model_info"] = model_info + + litellm.completion(**kwargs) + + assert mock_get_optional_params.called is True + get_optional_params_model_param = mock_get_optional_params.call_args.kwargs[ + "model" + ] + assert get_optional_params_model_param == expected_model_param + + +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_gpt_5_4_responses_bridge_merges_reasoning_summary_kwarg_without_tools( + mock_responses_completion, +): + """reasoningSummary without tools should route and merge into reasoning_effort dict.""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + litellm.completion( + model="gpt-5.4", + messages=[{"role": "user", "content": "ok"}], + reasoning_effort="medium", + reasoningSummary="auto", + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == { + "effort": "medium", + "summary": "auto", + } + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + + +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_responses_bridge_preserves_reasoning_summary_without_effort( + mock_responses_completion, +): + """Reasoning summary should survive responses routing even without effort.""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "ok"}], + reasoningSummary="auto", + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params["reasoning_effort"] == {"summary": "auto"} + assert "reasoningSummary" not in optional_params + assert "reasoning_summary" not in optional_params + + +@patch("litellm.completion_extras.responses_api_bridge.completion") +def test_gpt_5_responses_bridge_tools_and_reasoning_summary( + mock_responses_completion, +): + """Bare gpt-5 with tools + reasoningSummary should bridge (OpenCode-style).""" + mock_responses_completion.return_value = MagicMock() + + import litellm + + litellm.completion( + model="gpt-5", + messages=[{"role": "user", "content": "ok"}], + tools=[ + { + "type": "function", + "function": { + "name": "apply_patch", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + tool_choice="auto", + reasoning_effort="medium", + reasoningSummary="auto", + stream=True, + api_key="fake-key", + ) + + assert mock_responses_completion.called is True + optional_params = mock_responses_completion.call_args.kwargs["optional_params"] + assert optional_params.get("reasoning_effort") == { + "effort": "medium", + "summary": "auto", + } + + def test_responses_api_bridge_check_handles_exception(): """Test that responses_api_bridge_check handles exceptions and still processes responses/ models.""" from litellm.main import responses_api_bridge_check diff --git a/tests/test_litellm/test_main_module_header.py b/tests/test_litellm/test_main_module_header.py new file mode 100644 index 00000000000..a16e14e8c32 --- /dev/null +++ b/tests/test_litellm/test_main_module_header.py @@ -0,0 +1,13 @@ +from pathlib import Path + + +def test_main_py_starts_with_brief_file_description(): + repo_root = Path(__file__).resolve().parents[2] + main_py = repo_root / "litellm" / "main.py" + + first_two_lines = main_py.read_text(encoding="utf-8").splitlines()[:2] + + assert any( + "LiteLLM main module" in line and "entrypoints" in line + for line in first_two_lines + ) diff --git a/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py new file mode 100644 index 00000000000..9df18a9f0f0 --- /dev/null +++ b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py @@ -0,0 +1,162 @@ +"""Cover the Bedrock-ARN dispatch in ``litellm.batches.main.retrieve_batch``. + +The dispatch picks one of two Bedrock handlers depending on the ARN +family in ``batch_id``: + +* ``:async-invoke/`` -> ``_handle_async_invoke_status`` (data plane) +* ``:model-invocation-job/`` -> ``_handle_model_invocation_job_status`` + (control plane, added in this PR) + +Anything else falls through to the generic ``provider_config`` retrieve +flow. We mock the two handlers so the tests don't hit AWS — the focus +here is purely the dispatch logic that lives in ``main.py``. +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm # noqa: E402 + +ASYNC_INVOKE_ARN = "arn:aws:bedrock:us-west-2:123456789012:async-invoke/abc123def456" +MIJ_ARN = "arn:aws:bedrock:us-west-2:123456789012:model-invocation-job/abc1234567" + + +@pytest.fixture +def mock_handlers(): + """Patch both Bedrock retrieve handlers and yield the mocks. + + We patch at the import site (litellm.batches.main) rather than the + definition site so the ``BedrockBatchesHandler`` reference inside + ``retrieve_batch`` resolves to our mocks. + """ + fake_batch = MagicMock(name="LiteLLMBatch") + with ( + patch( + "litellm.batches.main.BedrockBatchesHandler._handle_async_invoke_status", + return_value=fake_batch, + ) as async_invoke, + patch( + "litellm.batches.main.BedrockBatchesHandler._handle_model_invocation_job_status", + return_value=fake_batch, + ) as mij, + ): + yield async_invoke, mij, fake_batch + + +def test_async_invoke_arn_routes_to_async_invoke_handler(mock_handlers): + """``:async-invoke/`` ARNs go to the data-plane handler.""" + async_invoke, mij, fake_batch = mock_handlers + + result = litellm.retrieve_batch( + batch_id=ASYNC_INVOKE_ARN, + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + ) + + assert result is fake_batch + async_invoke.assert_called_once() + mij.assert_not_called() + call_kwargs = async_invoke.call_args.kwargs + assert call_kwargs["batch_id"] == ASYNC_INVOKE_ARN + assert call_kwargs["aws_region_name"] == "us-west-2" + # Region must be stripped from the forwarded kwargs to avoid TypeError + # (it's already an explicit positional/keyword arg). + assert "aws_region_name" not in { + k + for k in call_kwargs + if k not in {"batch_id", "aws_region_name", "logging_obj"} + } + + +def test_async_invoke_arn_falls_back_to_default_region_when_unset(mock_handlers): + """If no ``aws_region_name`` is passed, the data-plane handler defaults + to ``us-east-1`` (preserving prior behavior on this branch).""" + async_invoke, _mij, _ = mock_handlers + + litellm.retrieve_batch( + batch_id=ASYNC_INVOKE_ARN, + custom_llm_provider="bedrock", + ) + + async_invoke.assert_called_once() + assert async_invoke.call_args.kwargs["aws_region_name"] == "us-east-1" + + +def test_model_invocation_job_arn_routes_to_mij_handler(mock_handlers): + """``:model-invocation-job/`` ARNs go to the new control-plane handler.""" + _async_invoke, mij, fake_batch = mock_handlers + + result = litellm.retrieve_batch( + batch_id=MIJ_ARN, + custom_llm_provider="bedrock", + aws_region_name="us-west-2", + ) + + assert result is fake_batch + mij.assert_called_once() + _async_invoke.assert_not_called() + call_kwargs = mij.call_args.kwargs + assert call_kwargs["batch_id"] == MIJ_ARN + assert call_kwargs["aws_region_name"] == "us-west-2" + + +def test_model_invocation_job_arn_with_no_region_passes_none(mock_handlers): + """The MIJ handler is responsible for sniffing region from the ARN + when none is explicitly provided. Dispatch must forward ``None`` + rather than substituting a default — otherwise per-region jobs in + other AWS regions would silently route to ``us-east-1``.""" + _async_invoke, mij, _ = mock_handlers + + litellm.retrieve_batch( + batch_id=MIJ_ARN, + custom_llm_provider="bedrock", + ) + + mij.assert_called_once() + assert mij.call_args.kwargs["aws_region_name"] is None + + +def test_unrelated_bedrock_arn_falls_through_to_provider_config(mock_handlers): + """Bedrock ARNs that aren't async-invoke or model-invocation-job + must NOT hit either special handler — they should fall through to + the existing generic provider_config path. We don't fully exercise + that path here (it requires a real provider config); we just assert + neither special handler is invoked.""" + async_invoke, mij, _ = mock_handlers + + # Use a plausible-but-unsupported Bedrock ARN family. + unrelated_arn = "arn:aws:bedrock:us-west-2:123456789012:provisioned-model/xyz" + + with pytest.raises(Exception): + # Will raise because no provider_config exists for this path — + # that's fine, we just need to assert neither bedrock handler ran + # before the failure. + litellm.retrieve_batch( + batch_id=unrelated_arn, + custom_llm_provider="bedrock", + ) + + async_invoke.assert_not_called() + mij.assert_not_called() + + +def test_non_bedrock_id_skips_bedrock_dispatch_entirely(mock_handlers): + """Plain (non-ARN) batch ids must not even enter the Bedrock dispatch + block — they belong to other providers' retrieve flows.""" + async_invoke, mij, _ = mock_handlers + + with pytest.raises(Exception): + litellm.retrieve_batch( + batch_id="batch_abc123", + custom_llm_provider="openai", + ) + + async_invoke.assert_not_called() + mij.assert_not_called() diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 7a9d5acaa27..c3f93078557 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -18,6 +18,7 @@ sys.path.insert( import litellm from litellm import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo def test_should_not_pollute_shared_key_with_zero_cost_pricing(): @@ -266,3 +267,59 @@ def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): f"Order should not matter. Expected {builtin_output_cost}, " f"got {info_std_2['output_cost_per_token']}" ) + + +def test_responses_prefix_stripped_alias_registered_for_model_list(): + """ + Register ``litellm.model_cost`` under the backend key with ``responses/`` and + under the stripped key (``responses_api_bridge_check`` removes that segment). + """ + uid = "responses-strip-alias-test-a1b2c3d4" + Router( + model_list=[ + { + "model_name": "azure-responses-strip-test", + "litellm_params": { + "model": "responses/gpt-strip-test-a1b2c3d4", + "custom_llm_provider": "azure", + "api_key": "fake-key-strip", + }, + "model_info": { + "id": uid, + "supports_native_streaming": True, + }, + } + ], + ) + assert "azure/responses/gpt-strip-test-a1b2c3d4" in litellm.model_cost + assert "azure/gpt-strip-test-a1b2c3d4" in litellm.model_cost + assert ( + litellm.model_cost["azure/gpt-strip-test-a1b2c3d4"].get( + "supports_native_streaming" + ) + is True + ) + + +def test_responses_prefix_stripped_alias_registered_for_add_deployment(): + """Dynamic ``add_deployment`` must mirror ``_create_deployment`` registration.""" + uid = "add-dep-responses-strip-e5f6a7b8" + router = Router(model_list=[]) + deployment = Deployment( + model_name="dyn-responses-strip", + litellm_params=LiteLLM_Params( + model="responses/gpt-add-strip-e5f6a7b8", + custom_llm_provider="azure", + api_key="fake-key-add", + ), + model_info=ModelInfo(id=uid, supports_native_streaming=True), + ) + router.add_deployment(deployment=deployment) + assert "azure/responses/gpt-add-strip-e5f6a7b8" in litellm.model_cost + assert "azure/gpt-add-strip-e5f6a7b8" in litellm.model_cost + assert ( + litellm.model_cost["azure/gpt-add-strip-e5f6a7b8"].get( + "supports_native_streaming" + ) + is True + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 65305e1a81e..d07af922ea6 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2817,6 +2817,128 @@ def test_generate_gcp_iam_access_token_import_error(): assert "pip install google-cloud-iam" in str(exc_info.value) +def test_generate_azure_ad_redis_token(): + """Test _generate_azure_ad_redis_token with mocked Azure credential.""" + from unittest.mock import Mock, patch + + expected_token = "azure-access-token-12345" + + mock_token = Mock() + mock_token.token = expected_token + + mock_credential = Mock() + mock_credential.get_token.return_value = mock_token + + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock(return_value=mock_credential) + mock_azure_identity.ClientSecretCredential = Mock() + mock_azure_identity.ManagedIdentityCredential = Mock() + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _generate_azure_ad_redis_token + + result = _generate_azure_ad_redis_token() + + assert result == expected_token + mock_credential.get_token.assert_called_once_with( + "https://redis.azure.com/.default" + ) + + +def test_generate_azure_ad_redis_token_service_principal(): + """Test _generate_azure_ad_redis_token with service principal credentials.""" + from unittest.mock import Mock, patch + + expected_token = "sp-access-token-67890" + + mock_token = Mock() + mock_token.token = expected_token + + mock_credential = Mock() + mock_credential.get_token.return_value = mock_token + + mock_client_secret_credential = Mock(return_value=mock_credential) + + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock() + mock_azure_identity.ClientSecretCredential = mock_client_secret_credential + mock_azure_identity.ManagedIdentityCredential = Mock() + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _generate_azure_ad_redis_token + + result = _generate_azure_ad_redis_token( + azure_client_id="test-client-id", + azure_tenant_id="test-tenant-id", + azure_client_secret="test-secret", + ) + + assert result == expected_token + mock_client_secret_credential.assert_called_once_with( + client_id="test-client-id", + tenant_id="test-tenant-id", + client_secret="test-secret", + ) + + +def test_generate_azure_ad_redis_token_import_error(): + """Test that _generate_azure_ad_redis_token raises ImportError when azure-identity is missing.""" + from unittest.mock import patch + from litellm._redis import _generate_azure_ad_redis_token + + with patch.dict("sys.modules", {"azure.identity": None}): + with pytest.raises(ImportError) as exc_info: + _generate_azure_ad_redis_token() + + assert "azure-identity is required" in str(exc_info.value) + + +def test_redis_client_logic_azure_ad_auth(): + """Test that _get_redis_client_logic sets up Azure AD auth when REDIS_AZURE_AD_TOKEN=true. + + Mocks ``azure.identity`` via ``sys.modules`` so the test does not require + the real ``azure-identity`` package to be installed in the CI environment. + """ + from unittest.mock import Mock, patch + + mock_credential = Mock() + mock_azure_identity = Mock() + mock_azure_identity.DefaultAzureCredential = Mock(return_value=mock_credential) + mock_azure_identity.ClientSecretCredential = Mock(return_value=mock_credential) + mock_azure_identity.ManagedIdentityCredential = Mock(return_value=mock_credential) + + with patch.dict( + "sys.modules", {"azure.identity": mock_azure_identity, "azure": Mock()} + ): + from litellm._redis import _get_redis_client_logic + + redis_kwargs = _get_redis_client_logic( + host="myredis.redis.cache.windows.net", + port="6380", + azure_redis_ad_token="true", + ssl=True, + ) + + assert "redis_connect_func" in redis_kwargs + # Marker for async paths to detect Azure AD auth + assert hasattr(redis_kwargs["redis_connect_func"], "_azure_redis_ad_token") + assert redis_kwargs["redis_connect_func"]._azure_redis_ad_token is True + # Live credential object (not raw secret) is exposed for async paths + assert hasattr(redis_kwargs["redis_connect_func"], "_azure_credential") + # Raw credentials must NOT be exposed on the function + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_client_secret") + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_client_id") + assert not hasattr(redis_kwargs["redis_connect_func"], "_azure_tenant_id") + + # Azure-specific kwargs should be removed from the dict passed to Redis + assert "azure_redis_ad_token" not in redis_kwargs + assert "azure_client_id" not in redis_kwargs + + if __name__ == "__main__": # Allow running this test file directly for debugging pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/test_utils_module_docstring.py b/tests/test_litellm/test_utils_module_docstring.py new file mode 100644 index 00000000000..ac99fb63fd4 --- /dev/null +++ b/tests/test_litellm/test_utils_module_docstring.py @@ -0,0 +1,11 @@ +import ast +from pathlib import Path + + +def test_utils_module_has_docstring(): + utils_path = Path(__file__).parents[2] / "litellm" / "utils.py" + module = ast.parse(utils_path.read_text()) + + assert ast.get_docstring(module) == ( + "Utility helpers for LiteLLM core request handling and provider support." + ) diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 024d05e1037..8a3f9361ba1 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -303,7 +303,7 @@ async def test_chat_completion(): api_key=key_gen["key"], api_version="2024-02-15-preview", ) - with pytest.raises(openai.AuthenticationError) as e: + with pytest.raises(openai.PermissionDeniedError) as e: response = await azure_client.chat.completions.create( model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], diff --git a/tests/test_users.py b/tests/test_users.py index 05253a19aa5..57fbb0483e4 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -302,14 +302,14 @@ async def test_user_model_access(): model="good-model", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", ) - with pytest.raises(openai.AuthenticationError): + with pytest.raises(openai.PermissionDeniedError): await chat_completion( session=session, key=key, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index f74a71e901e..b25b6ce393a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -29,6 +29,7 @@ export interface TeamListCallOptions { organizationID?: string | null; teamID?: string | null; team_alias?: string | null; + search?: string | null; userID?: string | null; sortBy?: string | null; sortOrder?: string | null; @@ -52,6 +53,7 @@ export const teamListCall = async ( team_id: options.teamID, organization_id: options.organizationID, team_alias: options.team_alias, + search: options.search, user_id: options.userID, page, page_size: pageSize, @@ -178,6 +180,7 @@ const deletedTeamListCall = async ( team_id: options.teamID, organization_id: options.organizationID, team_alias: options.team_alias, + search: options.search, user_id: options.userID, page, page_size: pageSize, diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx index 6da2f82a2f1..a57b7e2095a 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx @@ -535,6 +535,21 @@ describe("ModelSelect", () => { }); }); + it("should not render an empty optgroup when includeSpecialOptions is omitted", async () => { + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByTestId("model-select")).toBeInTheDocument(); + }); + + const optgroups = document.querySelectorAll("optgroup"); + // Wildcard Options + Models — no blank leading group + expect(optgroups.length).toBe(2); + optgroups.forEach((g) => { + expect(g.getAttribute("label")).toBeTruthy(); + }); + }); + it("should render maxTagPlaceholder when many items are selected", async () => { // Create many models to trigger maxTagCount responsive behavior const manyModels: ProxyModel[] = Array.from({ length: 20 }, (_, i) => ({ diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx index 74b2619f7f3..800fa86b165 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx @@ -141,36 +141,38 @@ export const ModelSelect = (props: ModelSelectProps) => { onChange={handleChange} style={style} options={[ - includeSpecialOptions - ? { - label: Special Options, - title: "Special Options", - options: [ - ...(shouldShowAllProxyModels - ? [ - { - label: All Proxy Models, - value: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - disabled: - value.length > 0 && - value.some( - (v) => isSpecialOption(v) && v !== MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - ), - key: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, - }, - ] - : []), - { - label: No Default Models, - value: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, - disabled: - value.length > 0 && - value.some((v) => isSpecialOption(v) && v !== MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value), - key: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, - }, - ], - } - : [], + ...(includeSpecialOptions + ? [ + { + label: Special Options, + title: "Special Options", + options: [ + ...(shouldShowAllProxyModels + ? [ + { + label: All Proxy Models, + value: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + disabled: + value.length > 0 && + value.some( + (v) => isSpecialOption(v) && v !== MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + ), + key: MODEL_SELECT_ALL_PROXY_MODELS_SPECIAL_VALUE.value, + }, + ] + : []), + { + label: No Default Models, + value: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, + disabled: + value.length > 0 && + value.some((v) => isSpecialOption(v) && v !== MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value), + key: MODEL_SELECT_NO_DEFAULT_MODELS_SPECIAL_VALUE.value, + }, + ], + }, + ] + : []), ...(wildcard.length > 0 ? [ { diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index abc26c4cd44..b9305e4723a 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -80,8 +80,7 @@ interface TeamProps { } interface FilterState { - team_id: string; - team_alias: string; + search: string; organization_id: string; sort_by: string; sort_order: "asc" | "desc"; @@ -200,8 +199,7 @@ const Teams: React.FC = ({ const [currentOrg, setCurrentOrg] = useState(null); const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null); const [filters, setFilters] = useState({ - team_id: "", - team_alias: "", + search: "", organization_id: "", sort_by: "created_at", sort_order: "desc", @@ -215,7 +213,7 @@ const Teams: React.FC = ({ sortBy?: string; sortOrder?: string; organizationID?: string; - teamAlias?: string; + search?: string; } = {}) => { if (!accessToken) return; const page = opts.page ?? currentPage; @@ -223,7 +221,7 @@ const Teams: React.FC = ({ const sortBy = opts.sortBy ?? filters.sort_by; const sortOrder = opts.sortOrder ?? filters.sort_order; const organizationID = opts.organizationID ?? filters.organization_id; - const teamAlias = opts.teamAlias ?? filters.team_alias; + const search = opts.search ?? filters.search; setIsLoading(true); setFetchError(null); @@ -234,7 +232,7 @@ const Teams: React.FC = ({ size, { organizationID: organizationID || null, - team_alias: teamAlias || null, + search: search || null, userID: userRole !== "Admin" && userRole !== "Admin Viewer" ? userID : null, sortBy: sortBy || null, sortOrder: sortOrder || null, @@ -632,9 +630,9 @@ const Teams: React.FC = ({ setIsSearching(true); searchDebounceRef.current = setTimeout(async () => { try { - setFilters((prev) => ({ ...prev, team_alias: value })); + setFilters((prev) => ({ ...prev, search: value })); setCurrentPage(1); - await fetchTeamsV2({ page: 1, teamAlias: value }); + await fetchTeamsV2({ page: 1, search: value }); } finally { setIsSearching(false); } @@ -653,7 +651,7 @@ const Teams: React.FC = ({ pageSize, { organizationID: newFilters.organization_id || null, - team_alias: newFilters.team_alias || null, + search: newFilters.search || null, userID: userRole !== "Admin" && userRole !== "Admin Viewer" ? userID : null, sortBy: newFilters.sort_by || null, sortOrder: newFilters.sort_order || null, @@ -670,15 +668,14 @@ const Teams: React.FC = ({ if (searchDebounceRef.current) clearTimeout(searchDebounceRef.current); setIsSearching(false); const resetFilters: FilterState = { - team_id: "", - team_alias: "", + search: "", organization_id: "", sort_by: "created_at", sort_order: "desc", }; setFilters(resetFilters); setCurrentPage(1); - fetchTeamsV2({ page: 1, organizationID: "", teamAlias: "", sortBy: "created_at", sortOrder: "desc" }); + fetchTeamsV2({ page: 1, organizationID: "", search: "", sortBy: "created_at", sortOrder: "desc" }); }; const { token } = theme.useToken(); @@ -945,7 +942,7 @@ const Teams: React.FC = ({ } suffix={isSearching ? : null} - placeholder="Search teams by name..." + placeholder="Search teams by name or ID..." onChange={(e) => handleSearchChange(e.target.value)} allowClear style={{ maxWidth: 400 }} diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx index bbcddd572cd..b95f63368c7 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.test.tsx @@ -57,7 +57,10 @@ vi.mock("./EndpointUsage/EndpointUsage", () => ({ vi.mock("./UsageViewSelect/UsageViewSelect", async () => { const React = await import("react"); - const UsageViewSelect = ({ value, onChange }: any) => { + const UsageViewSelect = ({ value, onChange, canViewTagUsage = false }: any) => { + const tagOption = canViewTagUsage + ? React.createElement("option", { value: "tag" }, "Tag Usage") + : null; return React.createElement( "select", { @@ -70,7 +73,7 @@ vi.mock("./UsageViewSelect/UsageViewSelect", async () => { React.createElement("option", { value: "team" }, "Team Usage"), React.createElement("option", { value: "organization" }, "Organization Usage"), React.createElement("option", { value: "customer" }, "Customer Usage"), - React.createElement("option", { value: "tag" }, "Tag Usage"), + tagOption, React.createElement("option", { value: "agent" }, "Agent Usage"), React.createElement("option", { value: "user-agent-activity" }, "User Agent Activity"), ); @@ -639,6 +642,29 @@ describe("UsagePage", () => { }); }); + it("should show tag usage selector option for internal users", async () => { + mockUseAuthorized.mockReturnValue({ + isLoading: false, + isAuthorized: true, + token: "mock-token", + accessToken: "test-token", + userId: "user-123", + userEmail: "test@example.com", + userRole: "internal_user", + premiumUser: true, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }); + + renderWithProviders(); + + await waitFor(() => { + expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); + }); + + expect(screen.getByRole("option", { name: "Tag Usage" })).toBeInTheDocument(); + }); + it("should show organization usage banner and view for admins", async () => { renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index 809f1d4e17b..50dd5e78f37 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -31,7 +31,7 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { all_admin_roles } from "../../../utils/roles"; +import { all_admin_roles, internalUserRoles } from "../../../utils/roles"; import { ActivityMetrics, processActivityData } from "../../activity_metrics"; import CloudZeroExportModal from "../../cloudzero_export_modal"; import EntityUsageExportModal from "../../EntityUsageExport"; @@ -84,6 +84,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { console.log(`currentUser: ${JSON.stringify(currentUser)}`); console.log(`currentUser max budget: ${currentUser?.max_budget}`); const isAdmin = all_admin_roles.includes(userRole || ""); + const canViewTagUsage = isAdmin || internalUserRoles.includes(userRole || ""); // Debounced search for user selector const [userSearchInput, setUserSearchInput] = useState(""); @@ -145,23 +146,6 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const [topKeysLimit, setTopKeysLimit] = useState(5); const [topModelsLimit, setTopModelsLimit] = useState(5); const [showTokenBreakdown, setShowTokenBreakdown] = useState(false); - const getAllTags = async () => { - if (!accessToken) { - return; - } - const tags = await tagListCall(accessToken); - setAllTags( - Object.values(tags).map((tag: Tag) => ({ - label: tag.name, - value: tag.name, - })), - ); - }; - - useEffect(() => { - getAllTags(); - }, [accessToken]); - // Sync selectedUserId when auth state settles (isAdmin/userID may be null on initial render) useEffect(() => { if (!isAdmin && userID) { @@ -175,6 +159,30 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const startTime = useMemo(() => (dateValue.from ? new Date(dateValue.from) : null), [dateValue.from]); const endTime = useMemo(() => (dateValue.to ? new Date(dateValue.to) : null), [dateValue.to]); + useEffect(() => { + if (!accessToken) return; + let cancelled = false; + (async () => { + try { + const tags = await tagListCall(accessToken, startTime, endTime); + if (cancelled) return; + setAllTags( + Object.values(tags).map((tag: Tag) => ({ + label: tag.name, + value: tag.name, + })), + ); + } catch (e) { + if (!cancelled) { + console.error("Failed to fetch tag list", e); + } + } + })(); + return () => { + cancelled = true; + }; + }, [accessToken, startTime, endTime]); + // Try aggregated endpoint first, fall back to paginated on failure const aggregatedFetchIdRef = useRef(0); useEffect(() => { @@ -437,7 +445,12 @@ const UsagePage: React.FC = ({ teams, organizations }) => {
- setUsageView(value)} isAdmin={isAdmin} /> + setUsageView(value)} + isAdmin={isAdmin} + canViewTagUsage={canViewTagUsage} + />
{paginatedResult.isFetchingMore && ( diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx index 7bb80b424b5..b33129fcdb4 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.test.tsx @@ -110,4 +110,16 @@ describe("UsageViewSelect", () => { expect(mockOnChange).toHaveBeenCalledWith("team"); }); + + it("should show Tag Usage for non-admin users with tag usage permission", () => { + render(); + + expect(screen.getByRole("option", { name: "Tag Usage" })).toBeInTheDocument(); + }); + + it("should hide Tag Usage for non-admin users without tag usage permission", () => { + render(); + + expect(screen.queryByRole("option", { name: "Tag Usage" })).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx index 11fdb8a7cf5..184fc000274 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsageViewSelect/UsageViewSelect.tsx @@ -16,6 +16,7 @@ export interface UsageViewSelectProps { value: UsageOption; onChange: (value: UsageOption) => void; isAdmin: boolean; + canViewTagUsage?: boolean; title?: string; description?: string; "data-id"?: string; @@ -106,12 +107,16 @@ export const UsageViewSelect: React.FC = ({ value, onChange, isAdmin, + canViewTagUsage = false, title = "Usage View", description = "Select the usage data you want to view", "data-id": dataId, }) => { const getFilteredOptions = () => { return OPTIONS.filter((option) => { + if (option.value === "tag" && canViewTagUsage) { + return true; + } if (option.adminOnly && !isAdmin) { return false; } diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index c91a6e85cdd..16c1c6efecd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -5,6 +5,7 @@ import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUI import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { choiceToSkipSystemForCreate, + choiceToSkipToolForCreate, getGuardrailProviders, guardrail_provider_map, guardrailLogoMap, @@ -188,6 +189,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a mode: preset.mode, default_on: preset.defaultOn, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }; if (preset.provider === "BlockCodeExecution") { baseValues.confidence_threshold = 0.5; @@ -433,6 +435,11 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; } + const skipToolForCreate = choiceToSkipToolForCreate(values.skip_tool_message_choice); + if (skipToolForCreate !== undefined) { + guardrailData.litellm_params.skip_tool_message_in_guardrail = skipToolForCreate; + } + // For Presidio PII, add the entity and action configurations if (values.provider === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -804,6 +811,18 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a + + + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && !shouldRenderLLMJudgeFields(selectedProvider) && ( = ({ visible, onClose, a mode: "pre_call", default_on: false, skip_system_message_choice: "inherit", + skip_tool_message_choice: "inherit", }} > {stepConfigs.map((step, index) => { diff --git a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx index ad823df53fc..8ba9b0b312f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx @@ -6,6 +6,7 @@ import { guardrailLogoMap, getGuardrailProviders, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking"; import PiiConfiguration from "./pii_configuration"; @@ -29,6 +30,7 @@ interface EditGuardrailFormProps { default_on: boolean; pii_entities_config?: { [key: string]: string }; skip_system_message_choice?: SkipSystemMessageChoice; + skip_tool_message_choice?: SkipToolMessageChoice; [key: string]: any; }; } @@ -138,6 +140,15 @@ const EditGuardrailForm: React.FC = ({ delete litellm_params.skip_system_message_in_guardrail; } + const skipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (skipToolChoice === "yes") { + litellm_params.skip_tool_message_in_guardrail = true; + } else if (skipToolChoice === "no") { + litellm_params.skip_tool_message_in_guardrail = false; + } else { + delete litellm_params.skip_tool_message_in_guardrail; + } + let guardrail_info: any = {}; // For Presidio PII, add the entity and action configurations @@ -432,6 +443,18 @@ const EditGuardrailForm: React.FC = ({ + + + + {renderProviderSpecificFields()}
diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index 60400443d5c..53aebcff0de 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -29,7 +29,9 @@ import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice, + skipToolMessageToChoice, type SkipSystemMessageChoice, + type SkipToolMessageChoice, } from "./guardrail_info_helpers"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; @@ -214,12 +216,16 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (guardrailData && form) { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; form.setFieldsValue({ guardrail_name: guardrailData.guardrail_name, ...lp, skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", // Include any optional_params if they exist ...(guardrailData.litellm_params?.optional_params && { @@ -302,6 +308,20 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } } + const prevSkipToolChoice = skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ); + const nextSkipToolChoice = values.skip_tool_message_choice as SkipToolMessageChoice | undefined; + if (nextSkipToolChoice !== undefined && nextSkipToolChoice !== prevSkipToolChoice) { + if (nextSkipToolChoice === "inherit") { + updateData.litellm_params.skip_tool_message_in_guardrail = null; + } else if (nextSkipToolChoice === "yes") { + updateData.litellm_params.skip_tool_message_in_guardrail = true; + } else { + updateData.litellm_params.skip_tool_message_in_guardrail = false; + } + } + // Only include guardrail_info if it has changed const originalGuardrailInfo = guardrailData.guardrail_info; const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined; @@ -674,11 +694,15 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, ...(() => { const lp = { ...(guardrailData.litellm_params || {}) }; delete lp.skip_system_message_in_guardrail; + delete lp.skip_tool_message_in_guardrail; return lp; })(), skip_system_message_choice: skipSystemMessageToChoice( guardrailData.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + guardrailData.litellm_params?.skip_tool_message_in_guardrail, + ), guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -716,6 +740,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, + + + + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx index dfda86c1e4a..1fc62f94cf1 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx @@ -12,6 +12,8 @@ import { GuardrailProviders, skipSystemMessageToChoice, choiceToSkipSystemForCreate, + skipToolMessageToChoice, + choiceToSkipToolForCreate, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -215,4 +217,18 @@ describe("guardrail_info_helpers", () => { expect(choiceToSkipSystemForCreate("no")).toBe(false); }); }); + + describe("skipToolMessageToChoice / choiceToSkipToolForCreate", () => { + it("maps API values to form choices and back for create", () => { + expect(skipToolMessageToChoice(undefined)).toBe("inherit"); + expect(skipToolMessageToChoice(null)).toBe("inherit"); + expect(skipToolMessageToChoice(true)).toBe("yes"); + expect(skipToolMessageToChoice(false)).toBe("no"); + + expect(choiceToSkipToolForCreate("inherit")).toBeUndefined(); + expect(choiceToSkipToolForCreate(undefined)).toBeUndefined(); + expect(choiceToSkipToolForCreate("yes")).toBe(true); + expect(choiceToSkipToolForCreate("no")).toBe(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index ac4b787e96a..54b16b81765 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -179,3 +179,19 @@ export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | un if (choice === "no") return false; return undefined; } + +/** Tri-state UI value for `litellm_params.skip_tool_message_in_guardrail` (inherit = use global). */ +export type SkipToolMessageChoice = "inherit" | "yes" | "no"; + +export function skipToolMessageToChoice(v: boolean | null | undefined): SkipToolMessageChoice { + if (v === true) return "yes"; + if (v === false) return "no"; + return "inherit"; +} + +/** Create flow: omit key when inheriting global default. */ +export function choiceToSkipToolForCreate(choice: SkipToolMessageChoice | undefined): boolean | undefined { + if (choice === "yes") return true; + if (choice === "no") return false; + return undefined; +} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 5bb2da78fa2..ecf6ce48fde 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -11,7 +11,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers"; +import { + getGuardrailLogoAndName, + guardrail_provider_map, + skipSystemMessageToChoice, + skipToolMessageToChoice, +} from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; import { Guardrail, GuardrailDefinitionLocation } from "./types"; @@ -304,6 +309,9 @@ const GuardrailTable: React.FC = ({ skip_system_message_choice: skipSystemMessageToChoice( selectedGuardrail.litellm_params?.skip_system_message_in_guardrail, ), + skip_tool_message_choice: skipToolMessageToChoice( + selectedGuardrail.litellm_params?.skip_tool_message_in_guardrail, + ), ...selectedGuardrail.guardrail_info, }} /> diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index f410a049762..6b3c65aaf7b 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -53,6 +53,7 @@ export interface KeyResponse { organization_id: string | null; org_id?: string | null; created_at: string; + created_by?: string; updated_at: string; last_active: string | null; team_spend: number; diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index 29eb8e0019b..54144d02eda 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -250,6 +250,33 @@ describe("ModelInfoView", () => { }); }); + it("should pass model_info.id to disambiguate duplicate model_name deployments", async () => { + // Regression test: when two deployments share `model_name` (e.g. + // wildcard `openai/*` with different `api_base` values), the UI + // must forward the clicked row's `model_info.id` to the backend. + // Otherwise /health/test_connection silently probes deployments[0] + // instead of the deployment the user actually selected. + const user = userEvent.setup(); + render(, { wrapper }); + + await waitFor(() => { + expect(screen.getByText("Model Settings")).toBeInTheDocument(); + }); + + const testButton = screen.getByRole("button", { name: /test connection/i }); + await user.click(testButton); + + await waitFor(() => { + expect(mockTestConnectionRequest).toHaveBeenCalled(); + }); + + const callArgs = mockTestConnectionRequest.mock.calls[0]; + // Signature: (accessToken, litellm_params, model_info, mode) + const modelInfoArg = callArgs[2] as Record; + expect(modelInfoArg).toBeDefined(); + expect(modelInfoArg.id).toBe("123"); + }); + it("should display error notification when connection test fails", async () => { const user = userEvent.setup(); mockTestConnectionRequest.mockRejectedValue(new Error("Connection failed")); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index ea8af2fcd62..95a43862de5 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -379,6 +379,12 @@ export default function ModelInfoView({ model: localModelData.litellm_model_name, }, { + // `id` is required to disambiguate when multiple deployments + // share the same model_name (e.g. wildcard `openai/*` with two + // different `api_base` values for failover). Without it the + // backend silently falls back to deployments[0] and probes + // the wrong endpoint. + id: localModelData.model_info?.id, mode: localModelData.model_info?.mode, }, localModelData.model_info?.mode, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 8bfe7c98691..7f28dbe6da9 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -7288,10 +7288,29 @@ export const tagInfoCall = async (accessToken: string, tagNames: string[]): Prom } }; -export const tagListCall = async (accessToken: string): Promise => { +const formatYmd = (value: Date): string => { + const year = value.getFullYear(); + const month = String(value.getMonth() + 1).padStart(2, "0"); + const day = String(value.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +}; + +export const tagListCall = async ( + accessToken: string, + startTime?: Date | null, + endTime?: Date | null, +): Promise => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/tag/list` : `/tag/list`; + if (startTime && endTime) { + const params = new URLSearchParams({ + start_date: formatYmd(startTime), + end_date: formatYmd(endTime), + }); + url = `${url}?${params.toString()}`; + } + const response = await fetch(url, { method: "GET", headers: { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx index 3ad59cb3693..bd139df4833 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.test.tsx @@ -155,6 +155,15 @@ vi.mock("antd", () => { const Button = ({ children, htmlType, ...props }: { children?: any; htmlType?: string }) => React.createElement("button", { ...props, type: htmlType ?? props.type }, children); + const Typography = ({ children, ...props }: { children?: any }) => + React.createElement("div", props, children); + Typography.Text = ({ children, ...props }: { children?: any }) => + React.createElement("span", props, children); + Typography.Paragraph = ({ children, ...props }: { children?: any }) => + React.createElement("p", props, children); + Typography.Title = ({ children, ...props }: { children?: any }) => + React.createElement("h1", props, children); + return { Button, Form, @@ -171,6 +180,7 @@ vi.mock("antd", () => { Switch, Tag, Tooltip, + Typography, }; }); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 29050907503..7d3f077dafc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -9,7 +9,7 @@ import { formatNumberWithCommas } from "@/utils/dataUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { Accordion, AccordionBody, AccordionHeader, Button, Col, Grid, Text, TextInput, Title } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip } from "antd"; +import { Button as Button2, Form, Input, Modal, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; import debounce from "lodash/debounce"; import React, { useCallback, useEffect, useState } from "react"; import { rolesWithWriteAccess } from "../../utils/roles"; @@ -979,28 +979,28 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } }} > - + diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index a7e48ff12bc..cfc08e8c4ef 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -25,7 +25,8 @@ import { Text, } from "@tremor/react"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip } from "antd"; +import { Popover, Skeleton, Tooltip, Typography } from "antd"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; @@ -339,18 +340,59 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi size: 70, enableSorting: false, cell: (info) => { - const value = info.getValue() as string | null; - const displayValue = value === "default_user_id" ? "Default Proxy Admin" : value; + const userId = info.getValue() as string | null; + if (!userId) return "-"; + const { created_by_user } = info.row.original; + const userAlias = created_by_user?.user_alias ?? null; + const userEmail = created_by_user?.user_email ?? null; + const isDefaultAdmin = userId === "default_user_id"; + const displayValue = userAlias || userEmail || userId; const width = info.cell.column.getSize(); + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias }, + { label: "User Email", value: userEmail }, + { label: "User ID", value: userId }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + {value} + + ) : ( + - + )} +
+ ))} +
+ ); + + if (isDefaultAdmin && !userAlias && !userEmail) { + return ( + + + + + + ); + } + return ( - + - {displayValue ?? "-"} + {displayValue} - + ); }, }, diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx index c9c5856129f..14750787609 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.test.tsx @@ -8,10 +8,12 @@ const MOCK_DATA: KeyInfoData = { keyId: "sk-1234567890abcdef", userId: "user-abc-123", userEmail: "test@example.com", + userAlias: null, createdBy: "admin@example.com", createdAt: "Oct 29, 2025 at 1:26 AM", lastUpdated: "Oct 29, 2025 at 1:47 AM", lastActive: "Oct 29, 2025 at 2:00 AM", + expires: "Never", }; describe("KeyInfoHeader", () => { @@ -28,12 +30,11 @@ describe("KeyInfoHeader", () => { it("should render all metadata fields", () => { render(); - expect(screen.getByText("User Email")).toBeInTheDocument(); + expect(screen.getByText("User")).toBeInTheDocument(); expect(screen.getByText("test@example.com")).toBeInTheDocument(); - expect(screen.getByText("User ID")).toBeInTheDocument(); - expect(screen.getByText("user-abc-123")).toBeInTheDocument(); expect(screen.getByText("Created At")).toBeInTheDocument(); expect(screen.getByText("Created By")).toBeInTheDocument(); + expect(screen.getByText("Expires")).toBeInTheDocument(); expect(screen.getByText("Last Updated")).toBeInTheDocument(); expect(screen.getByText("Last Active")).toBeInTheDocument(); }); @@ -122,8 +123,8 @@ describe("KeyInfoHeader", () => { }); describe("default_user_id handling", () => { - it("should show Default Proxy Admin tag for User ID when value is default_user_id", () => { - const data = { ...MOCK_DATA, userId: "default_user_id" }; + it("should show Default Proxy Admin tag for User when userId is default_user_id and no alias/email", () => { + const data = { ...MOCK_DATA, userId: "default_user_id", userEmail: "", userAlias: null }; render(); expect(screen.getAllByText("Default Proxy Admin").length).toBeGreaterThanOrEqual(1); }); @@ -135,9 +136,27 @@ describe("KeyInfoHeader", () => { }); }); - describe("empty value handling", () => { - it("should show '-' for User Email when value is empty", () => { - const data = { ...MOCK_DATA, userEmail: "" }; + describe("User field fallbacks", () => { + it("should display userAlias as primary when set, overriding email and userId", () => { + const data = { ...MOCK_DATA, userAlias: "alice" }; + render(); + expect(screen.getByText("alice")).toBeInTheDocument(); + expect(screen.queryByText("test@example.com")).not.toBeInTheDocument(); + }); + + it("should display userEmail when alias is null", () => { + render(); + expect(screen.getByText("test@example.com")).toBeInTheDocument(); + }); + + it("should fall back to userId when alias and email are missing", () => { + const data = { ...MOCK_DATA, userEmail: "", userAlias: null }; + render(); + expect(screen.getByText("user-abc-123")).toBeInTheDocument(); + }); + + it("should show '-' when alias, email, and userId are all empty", () => { + const data = { ...MOCK_DATA, userId: "", userEmail: "", userAlias: null }; render(); expect(screen.getByText("-")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx index 93ebae9c4be..d39b46a5cdd 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoHeader.tsx @@ -1,19 +1,20 @@ import React from "react"; -import { Button, Typography, Tooltip, Space, Divider, Flex } from "antd"; +import { Button, Typography, Tooltip, Space, Divider, Flex, Popover } from "antd"; import { ArrowLeftOutlined, SyncOutlined, DeleteOutlined, PlusOutlined, UserOutlined, - MailOutlined, CalendarOutlined, ClockCircleOutlined, ThunderboltOutlined, SafetyCertificateOutlined, TransactionOutlined, + FieldTimeOutlined, } from "@ant-design/icons"; import LabeledField from "../common_components/LabeledField"; +import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; const { Title, Text } = Typography; @@ -22,10 +23,12 @@ export interface KeyInfoData { keyId: string; userId: string; userEmail: string; + userAlias?: string | null; createdBy: string; createdAt: string; lastUpdated: string; lastActive: string; + expires: string; } interface KeyInfoHeaderProps { @@ -41,6 +44,94 @@ interface KeyInfoHeaderProps { regenerateTooltip?: string; } +function UserField({ + userAlias, + userEmail, + userId, +}: { + userAlias?: string | null; + userEmail: string; + userId: string; +}) { + const labelEl = ( + + + + User + + + ); + + const isEmpty = !userAlias && !userEmail && !userId; + if (isEmpty) { + return ( +
+ {labelEl} +
-
+
+ ); + } + + const isDefaultAdmin = userId === "default_user_id"; + const displayValue = userAlias || userEmail || userId; + + const popoverContent = ( +
+ {[ + { label: "User Alias", value: userAlias ?? null }, + { label: "User Email", value: userEmail || null }, + { label: "User ID", value: userId || null }, + ].map(({ label, value }) => ( +
+ {label} + {value ? ( + + {value} + + ) : ( + - + )} +
+ ))} +
+ ); + + if (isDefaultAdmin && !userAlias && !userEmail) { + return ( +
+ {labelEl} +
+ + + +
+
+ ); + } + + return ( +
+ {labelEl} +
+ + + {displayValue} + + +
+
+ ); +} + export function KeyInfoHeader({ data, onBack, @@ -101,15 +192,8 @@ export function KeyInfoHeader({ - } /> - } - truncate - copyable - defaultUserIdCheck - /> + + } /> diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 2e4d0d97e4c..1886075a9d9 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -158,8 +158,8 @@ describe("KeyEditView", () => { const { getByText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -176,8 +176,8 @@ describe("KeyEditView", () => { const { getByText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -194,8 +194,8 @@ describe("KeyEditView", () => { const { getByLabelText } = renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -219,7 +219,7 @@ describe("KeyEditView", () => { { }} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -241,8 +241,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -259,8 +259,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -277,8 +277,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -295,8 +295,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -314,7 +314,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -344,8 +344,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -367,8 +367,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={""} userID={""} userRole={""} @@ -385,8 +385,8 @@ describe("KeyEditView", () => { renderWithProviders( { }} - onSubmit={async () => { }} + onCancel={() => {}} + onSubmit={async () => {}} accessToken={"test-token"} userID={""} userRole={""} @@ -404,7 +404,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -434,10 +434,14 @@ describe("KeyEditView", () => { it("should handle empty allowed routes string on submit", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataWithRoutes = { + ...MOCK_KEY_DATA, + allowed_routes: ["llm_api_routes"], + }; renderWithProviders( { }} + keyData={keyDataWithRoutes} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -463,6 +467,101 @@ describe("KeyEditView", () => { }); }); + it("should omit allowed_routes from submit when value is unchanged", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const aiApisKeyData = { + ...MOCK_KEY_DATA, + allowed_routes: ["llm_api_routes"], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); + + it("should omit allowed_routes from submit when keyData.allowed_routes is null and form is untouched", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataNullRoutes = { + ...MOCK_KEY_DATA, + allowed_routes: null as unknown as string[], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); + + it("should omit allowed_routes from submit when server returned routes in a different order", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataReordered = { + ...MOCK_KEY_DATA, + allowed_routes: ["beta_routes", "alpha_routes"], + }; + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /save changes/i })).toBeInTheDocument(); + }); + + const submitButton = screen.getByRole("button", { name: /save changes/i }); + await userEvent.click(submitButton); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalled(); + const callArgs = onSubmitMock.mock.calls[0][0]; + expect("allowed_routes" in callArgs).toBe(false); + }); + }); it("should pass access_group_ids to onSubmit when saving key with access groups", async () => { const onSubmitMock = vi.fn().mockResolvedValue(undefined); @@ -554,7 +653,7 @@ describe("KeyEditView", () => { renderWithProviders( { }} + onCancel={() => {}} onSubmit={onSubmitMock} accessToken={"test-token"} userID={"test-user"} @@ -576,10 +675,13 @@ describe("KeyEditView", () => { }); // Wait for the cancel button to actually be disabled (state update may take a moment) - await waitFor(() => { - const cancelButton = screen.getByRole("button", { name: /cancel/i }); - expect(cancelButton).toBeDisabled(); - }, { timeout: 3000 }); + await waitFor( + () => { + const cancelButton = screen.getByRole("button", { name: /cancel/i }); + expect(cancelButton).toBeDisabled(); + }, + { timeout: 3000 }, + ); // Clean up: resolve the promise to allow the form to complete if (resolveSubmit) { diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 9b38d930a50..d5e410029a7 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -78,7 +78,6 @@ const getKeyTypeFromRoutes = (allowedRoutes: string[] | null | undefined): strin return "default"; }; - export function KeyEditView({ keyData, onCancel, @@ -106,7 +105,7 @@ export function KeyEditView({ const [neverExpire, setNeverExpire] = useState(!keyData.expires); const [isKeySaving, setIsKeySaving] = useState(false); const [budgetLimits, setBudgetLimits] = useState( - Array.isArray(keyData.budget_limits) ? keyData.budget_limits : [] + Array.isArray(keyData.budget_limits) ? keyData.budget_limits : [], ); const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations(); const { data: projects } = useProjects(); @@ -116,9 +115,7 @@ export function KeyEditView({ const projectDisplay = (() => { if (!keyData.project_id) return null; const project = projects?.find((p) => p.project_id === keyData.project_id); - return project?.project_alias - ? `${project.project_alias} (${keyData.project_id})` - : keyData.project_id; + return project?.project_alias ? `${project.project_alias} (${keyData.project_id})` : keyData.project_id; })(); useEffect(() => { @@ -198,9 +195,10 @@ export function KeyEditView({ access_group_ids: keyData.access_group_ids || [], auto_rotate: keyData.auto_rotate || false, ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", + allowed_routes: + Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 + ? keyData.allowed_routes.join(", ") + : "", }; useEffect(() => { @@ -226,9 +224,10 @@ export function KeyEditView({ access_group_ids: keyData.access_group_ids || [], auto_rotate: keyData.auto_rotate || false, ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", + allowed_routes: + Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 + ? keyData.allowed_routes.join(", ") + : "", }); }, [keyData, form]); @@ -275,12 +274,25 @@ export function KeyEditView({ } // If it's already an array (shouldn't happen, but handle it), keep as is + // Backend rejects non-empty allowed_routes from non-admins, so re-sending + // an unchanged value 403s a team admin. Set compare tolerates reorder. + const originalRoutesSet = new Set(Array.isArray(keyData.allowed_routes) ? keyData.allowed_routes : []); + const submittedRoutesSet = new Set(Array.isArray(values.allowed_routes) ? values.allowed_routes : []); + const allowedRoutesUnchanged = + originalRoutesSet.size === submittedRoutesSet.size && + [...submittedRoutesSet].every((r) => originalRoutesSet.has(r)); + if (allowedRoutesUnchanged) { + delete values.allowed_routes; + } + if (neverExpire) { values.duration = null; } // Include multi-window budget limits (filter out incomplete entries) - const validWindows = budgetLimits.filter((w) => w.budget_duration && w.max_budget !== null && w.max_budget !== undefined); + const validWindows = budgetLimits.filter( + (w) => w.budget_duration && w.max_budget !== null && w.max_budget !== undefined, + ); values.budget_limits = validWindows.length > 0 ? validWindows : undefined; await onSubmit(values); @@ -305,9 +317,13 @@ export function KeyEditView({ {({ getFieldValue, setFieldValue }) => { const allowedRoutesValue = getFieldValue("allowed_routes") || ""; // Convert string to array for checking - const allowedRoutes = typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" - ? allowedRoutesValue.split(",").map((r: string) => r.trim()).filter((r: string) => r.length > 0) - : []; + const allowedRoutes = + typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" + ? allowedRoutesValue + .split(",") + .map((r: string) => r.trim()) + .filter((r: string) => r.length > 0) + : []; const isDisabled = allowedRoutes.includes("management_routes") || allowedRoutes.includes("info_routes"); const models = getFieldValue("models") || []; @@ -348,9 +364,13 @@ export function KeyEditView({ {({ getFieldValue, setFieldValue }) => { const allowedRoutesValue = getFieldValue("allowed_routes") || ""; // Convert string to array for getKeyTypeFromRoutes - const allowedRoutes = typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" - ? allowedRoutesValue.split(",").map((r: string) => r.trim()).filter((r: string) => r.length > 0) - : []; + const allowedRoutes = + typeof allowedRoutesValue === "string" && allowedRoutesValue.trim() !== "" + ? allowedRoutesValue + .split(",") + .map((r: string) => r.trim()) + .filter((r: string) => r.length > 0) + : []; const keyTypeValue = getKeyTypeFromRoutes(allowedRoutes); return ( @@ -415,9 +435,7 @@ export function KeyEditView({ } name="allowed_routes" > - + @@ -442,10 +460,7 @@ export function KeyEditView({ } > - + @@ -579,7 +594,7 @@ export function KeyEditView({ !premiumUser ? "Premium feature - Upgrade to set allowed pass through routes by key" : Array.isArray(keyData.metadata?.allowed_passthrough_routes) && - keyData.metadata.allowed_passthrough_routes.length > 0 + keyData.metadata.allowed_passthrough_routes.length > 0 ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` : "Select or enter allowed pass through routes" } @@ -690,14 +705,13 @@ export function KeyEditView({ return team.team_alias?.toLowerCase().includes(input.toLowerCase()) ?? false; }} > - {(selectedOrganizationId - ? teams?.filter((t) => t.organization_id === selectedOrganizationId) - : teams - )?.map((team) => ( - - {`${team.team_alias} (${team.team_id})`} - - ))} + {(selectedOrganizationId ? teams?.filter((t) => t.organization_id === selectedOrganizationId) : teams)?.map( + (team) => ( + + {`${team.team_alias} (${team.team_id})`} + + ), + )} {enableProjectsUI && hasProject && ( diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 492e43cbc81..f8897780090 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -396,17 +396,23 @@ export default function KeyInfoView({ }; return ( -
+
setIsRegenerateModalOpen(true)} @@ -614,7 +620,7 @@ export default function KeyInfoView({ {/* Settings Panel */} - +
Key Settings {!isEditing && canModifyKey && ( diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 57155feae23..5b0b1d0fee3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -19,6 +19,7 @@ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; export const AGENT_CALL_TYPES = ["asend_message"]; export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ + { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" }, { label: "Last Hour", value: 1, unit: "hours" }, { label: "Last 4 Hours", value: 4, unit: "hours" }, diff --git a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx index 937844e1c1d..7a9a541d3e0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx @@ -1,10 +1,12 @@ import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import moment from "moment"; import { beforeEach, describe, expect, it, vi } from "vitest"; import SpendLogsTable, { RequestViewer } from "./index"; import type { LogEntry } from "./columns"; import type { Row } from "@tanstack/react-table"; import { renderWithProviders } from "../../../tests/test-utils"; +import { uiSpendLogsCall } from "../networking"; const mockHandleFilterResetFromHook = vi.fn(); vi.mock("./log_filter_logic", async (importOriginal) => { @@ -238,4 +240,52 @@ describe("SpendLogsTable", () => { expect(inputsAfterReset.length).toBe(0); }); }); + + describe("Quick Select time range", () => { + const waitForWindowSeconds = async (minMinutes: number) => { + let diff = -1; + await waitFor(() => { + const lastCall = vi.mocked(uiSpendLogsCall).mock.calls.at(-1)?.[0]; + if (!lastCall) throw new Error("uiSpendLogsCall was not called"); + diff = moment + .utc(lastCall.end_date, "YYYY-MM-DD HH:mm:ss") + .diff(moment.utc(lastCall.start_date, "YYYY-MM-DD HH:mm:ss"), "seconds"); + // start_date is rounded down to the minute boundary; end_date is current time + expect(diff).toBeGreaterThanOrEqual(minMinutes * 60); + expect(diff).toBeLessThan((minMinutes + 1) * 60); + }); + return diff; + }; + + it("should pass a ~1-minute window to uiSpendLogsCall when 'Last Minute' is selected", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last Minute" })); + + await waitForWindowSeconds(1); + }); + + it("should pass a ~15-minute window to uiSpendLogsCall when 'Last 15 Minutes' is selected", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last 15 Minutes" })); + + await waitForWindowSeconds(15); + }); + + it("should update the time-range button label to 'Last Minute' after selecting it", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /Last 24 Hours/i })); + await user.click(await screen.findByRole("button", { name: "Last Minute" })); + + expect(screen.getByRole("button", { name: "Last Minute" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Last 24 Hours/i })).not.toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/utils/roles.ts b/ui/litellm-dashboard/src/utils/roles.ts index 11e31fd369c..bb23985d7c2 100644 --- a/ui/litellm-dashboard/src/utils/roles.ts +++ b/ui/litellm-dashboard/src/utils/roles.ts @@ -5,7 +5,7 @@ export const old_admin_roles = ["Admin", "Admin Viewer"]; export const v2_admin_role_names = ["proxy_admin", "proxy_admin_viewer", "org_admin"]; export const all_admin_roles = [...old_admin_roles, ...v2_admin_role_names]; -export const internalUserRoles = ["Internal User", "Internal Viewer"]; +export const internalUserRoles = ["Internal User", "Internal Viewer", "internal_user", "internal_user_viewer"]; export const rolesAllowedToSeeUsage = ["Admin", "Admin Viewer", "Internal User", "Internal Viewer"]; export const rolesWithWriteAccess = ["Internal User", "Admin", "proxy_admin"]; // Admin-tier read parity: Admin Viewer sees Models + Endpoints, Agents, and diff --git a/uv.lock b/uv.lock index f8d78fe8794..ab9aba1e38f 100644 --- a/uv.lock +++ b/uv.lock @@ -3405,7 +3405,7 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, - { name = "jinja2", specifier = ">=3.1.0,<4.0" }, + { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" },