diff --git a/.github/scripts/close_duplicate_issues.py b/.github/scripts/close_duplicate_issues.py deleted file mode 100755 index ec522af4f88..00000000000 --- a/.github/scripts/close_duplicate_issues.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python3 -""" -Detect and close duplicate GitHub issues using title similarity. - -Modes: - --scan Compare all open issues against each other (batch) - --issue-number N Check a single issue against older open issues - -Requires the `gh` CLI to be authenticated. -""" - -import argparse -import difflib -import json -import re -import subprocess -import sys - - -def normalize_title(title: str) -> str: - """Strip common prefixes, lowercase, and collapse whitespace.""" - title = re.sub( - r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*", - "", - title, - flags=re.IGNORECASE, - ) - return " ".join(title.lower().split()) - - -def gh(*args: str) -> str: - """Run a gh CLI command and return stdout.""" - result = subprocess.run( - ["gh", *args], - capture_output=True, - text=True, - check=True, - ) - return result.stdout - - -def fetch_open_issues(repo: str | None) -> list[dict]: - """Fetch all open issues (excluding PRs) via gh api --paginate.""" - if repo: - endpoint = ( - f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - ) - else: - endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - cmd = ["api", "--paginate", endpoint] - - raw = gh(*cmd) - # gh --paginate concatenates JSON arrays, so we may get multiple arrays - issues = [] - for line in raw.strip().splitlines(): - line = line.strip() - if not line: - continue - parsed = json.loads(line) - if isinstance(parsed, list): - issues.extend(parsed) - else: - issues.append(parsed) - - # Filter out pull requests (they also appear in the issues endpoint) - return [i for i in issues if "pull_request" not in i] - - -def close_as_duplicate( - issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool -) -> None: - """Close an issue as duplicate of another, adding a comment and label.""" - repo_args = ["--repo", repo] if repo else [] - - if dry_run: - print( - f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}" - ) - return - - # Add comment - comment_body = ( - f"Closing as duplicate of #{duplicate_of}.\n\n" - "If you believe this is not a duplicate, please reopen and add context " - "explaining how this differs." - ) - gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args) - - # Add label - gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args) - - # Close with not_planned reason - gh( - "api", - f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}", - "-X", - "PATCH", - "-f", - "state=closed", - "-f", - "state_reason=not_planned", - ) - - print(f" Closed #{issue_number} as duplicate of #{duplicate_of}") - - -def find_duplicate( - issue: dict, candidates: list[dict], threshold: float -) -> dict | None: - """Return the first candidate whose normalized title is above threshold.""" - norm = normalize_title(issue["title"]) - for candidate in candidates: - if candidate["number"] == issue["number"]: - continue - cand_norm = normalize_title(candidate["title"]) - ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio() - if ratio >= threshold: - return candidate - return None - - -def scan_all( - issues: list[dict], threshold: float, repo: str | None, dry_run: bool -) -> int: - """Compare every issue against all older issues. Returns count of duplicates found.""" - # Sort oldest first - issues.sort(key=lambda i: i["number"]) - closed_count = 0 - - for idx, issue in enumerate(issues): - older = issues[:idx] - if not older: - continue - dup = find_duplicate(issue, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(issue["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{issue['number']}: \"{issue['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue["number"], dup["number"], repo, dry_run) - closed_count += 1 - - return closed_count - - -def check_single( - issue_number: int, - issues: list[dict], - threshold: float, - repo: str | None, - dry_run: bool, -) -> bool: - """Check a single issue against all older open issues. Returns True if duplicate found.""" - target = None - for i in issues: - if i["number"] == issue_number: - target = i - break - - if target is None: - print(f"Issue #{issue_number} not found among open issues.") - return False - - older = [i for i in issues if i["number"] < issue_number] - dup = find_duplicate(target, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(target["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{target['number']}: \"{target['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue_number, dup["number"], repo, dry_run) - return True - - print(f"#{issue_number}: no duplicate found above threshold {threshold}") - return False - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Detect and close duplicate GitHub issues" - ) - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--scan", action="store_true", help="Scan all open issues") - mode.add_argument("--issue-number", type=int, help="Check a single issue number") - parser.add_argument( - "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)" - ) - parser.add_argument( - "--close", - action="store_true", - help="Actually close duplicates (default is dry-run)", - ) - parser.add_argument( - "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted." - ) - args = parser.parse_args() - - dry_run = not args.close - - if dry_run: - print("=== DRY RUN MODE (pass --close to actually close issues) ===\n") - - print("Fetching open issues...") - issues = fetch_open_issues(args.repo) - print(f"Found {len(issues)} open issues.\n") - - if args.scan: - count = scan_all(issues, args.threshold, args.repo, dry_run) - print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}") - else: - found = check_single( - args.issue_number, issues, args.threshold, args.repo, dry_run - ) - sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.yml new file mode 100644 index 00000000000..d8256917805 --- /dev/null +++ b/.github/workflows/auto-close-duplicates.yml @@ -0,0 +1,69 @@ +name: Auto-close duplicate issues + +on: + schedule: + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Log which issues would close without closing anything + type: boolean + default: true + grace_period_days: + description: Days a duplicate notice must go unanswered before the close + type: number + default: 3 + pull_request: + paths: + - .github/workflows/auto-close-duplicates.yml + - scripts/auto-close-duplicates.ts + - scripts/auto-close-duplicates.test.ts + +permissions: {} + +jobs: + test: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + + - name: Test the sweep + run: bun test scripts/auto-close-duplicates.test.ts + + sweep: + if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + # Exact version, never latest: the next step holds an issues: write token + bun-version: "1.4.0" + + - name: Close unanswered duplicates, reopen ones the reporter answered + run: bun run scripts/auto-close-duplicates.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ inputs.dry_run == true }} + GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }} diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 78198b2c7bb..41ec43a1d9b 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -1,12 +1,19 @@ name: Check Duplicate Issues +# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later, +# and only when its title is identical to an older open issue and nobody replied. +# The HTML marker below is the handshake between the two, so keep it in the template. + on: issues: types: [opened, edited] +permissions: {} + jobs: check-duplicate: runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write contents: read @@ -19,35 +26,12 @@ jobs: threshold: 0.6 reaction: eyes comment: | - **⚠️ Potential duplicate detected** + + **Potential duplicate detected** - This issue appears similar to existing issue(s): + This looks similar to: {{#issues}} - - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar) + - #{{number}} - {{title}} {{/issues}} - Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. - - - name: Checkout close script - if: github.event.action == 'opened' - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - sparse-checkout: .github/scripts - persist-credentials: false - - - name: Set up Python - if: github.event.action == 'opened' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Auto-close if high-confidence duplicate - if: github.event.action == 'opened' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - python3 .github/scripts/close_duplicate_issues.py \ - --issue-number ${{ github.event.issue.number }} \ - --repo ${{ github.repository }} \ - --threshold 0.85 \ - --close + If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open. diff --git a/CLAUDE.md b/CLAUDE.md index 930825aeb89..d9e9e8f1586 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,8 @@ When adding new features, add meaningful tests. Don't add tests that don't check Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression) +Never test structure of code only function of it + `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md` diff --git a/Dockerfile b/Dockerfile index 700b0d6525e..b3ee85e9ed1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ gcc \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ rust \ openssl \ openssl-dev \ @@ -51,6 +51,7 @@ RUN apk add --no-cache \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching @@ -65,7 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -86,7 +87,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -101,7 +102,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 4ca40944606..aa01b9fba8b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -57,7 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -71,7 +71,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 45af8e1d7c9..72b8066ade6 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,9 +1,9 @@ { "reportAny": { - "limit": 15468 + "limit": 14765 }, "reportArgumentType": { - "limit": 2218 + "limit": 2216 }, "reportAssignmentType": { "limit": 319 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 4846 + "limit": 4493 }, "reportFunctionMemberAccess": { "limit": 7 @@ -42,7 +42,7 @@ "limit": 12 }, "reportIndexIssue": { - "limit": 30 + "limit": 25 }, "reportInvalidTypeForm": { "limit": 34 @@ -54,10 +54,10 @@ "limit": 0 }, "reportMissingParameterType": { - "limit": 5609 + "limit": 5607 }, "reportMissingTypeArgument": { - "limit": 15330 + "limit": 15310 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38418 + "limit": 38368 }, "reportUnknownParameterType": { - "limit": 19649 + "limit": 19633 }, "reportUnknownVariableType": { - "limit": 29987 + "limit": 29908 }, "reportUnnecessaryCast": { "limit": 111 @@ -141,6 +141,6 @@ "limit": 543 }, "reportUnusedVariable": { - "limit": 138 + "limit": 137 } } diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index f0d6d02fccf..c1348f68231 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ gcc \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ openssl \ openssl-dev \ nodejs \ @@ -49,6 +49,7 @@ RUN apk add --no-cache \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching @@ -63,7 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -84,7 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -98,7 +99,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4a5df6ecd69..2221435a83a 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. @@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN for i in 1 2 3; do \ apk add --no-cache \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ gcc \ rust \ bash \ @@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" \ LITELLM_NON_ROOT=true \ XDG_CACHE_HOME=/app/.cache @@ -69,7 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -96,7 +97,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 \ + --python python3.13 \ --no-sources-package litellm-proxy-extras; \ else \ uv sync --frozen --no-default-groups --no-editable \ @@ -105,7 +106,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3; \ + --python python3.13; \ fi RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ @@ -124,7 +125,7 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ done # Copy only what runtime needs. The application is installed inside the venv; diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 4a2e32e186e..308d70a6b26 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --extra semantic-router \ --extra bedrock-realtime \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --extra semantic-router \ --extra bedrock-realtime \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/litellm/constants.py b/litellm/constants.py index cc6db6c10cc..9872783bfab 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1728,6 +1728,7 @@ SENTRY_DENYLIST: Final = [ "jwt_token", "private_key", "SLACK_WEBHOOK_URL", + "ALERTING_WEBHOOK_URL", "webhook_url", "LANGFUSE_SECRET_KEY", # Email Configuration diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index d7d06387d85..94d734546be 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1485,9 +1485,9 @@ Model Info: elif self.default_webhook_url is not None: _digest_webhook = self.default_webhook_url else: - _digest_webhook = os.getenv("SLACK_WEBHOOK_URL", None) + _digest_webhook = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL") if _digest_webhook is None: - raise ValueError("Missing SLACK_WEBHOOK_URL from environment") + raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment") digest_key: Final = f"{alert_type_name_str}:{request_model or ''}:{api_base or ''}" @@ -1516,10 +1516,10 @@ Model Info: elif self.default_webhook_url is not None: slack_webhook_url = self.default_webhook_url else: - slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None) + slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL") if slack_webhook_url is None: - raise ValueError("Missing SLACK_WEBHOOK_URL from environment") + raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment") payload: Final = {"text": formatted_message} headers: Final = {"Content-type": "application/json"} diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index b09498f9292..3ac92b04c27 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -62,6 +62,8 @@ class GenAIMapper: GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds, GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, + GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens, + GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens, Error.TYPE: lambda d: d.error.error_type if d.error else None, Server.ADDRESS: lambda d: d.server.address if d.server else None, Server.PORT: lambda d: d.server.port if d.server else None, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index f70c777e1a7..d35405538f6 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -95,6 +95,22 @@ class LLMUsage: input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + + @classmethod + def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage: + # Cache token counts only exist on the raw provider usage object under metadata + metadata: Final[Mapping[str, object]] = payload.get("metadata") or {} + raw_usage: Final = metadata.get("usage_object") + usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {} + return cls( + input_tokens=as_int(payload.get("prompt_tokens")), + output_tokens=as_int(payload.get("completion_tokens")), + total_tokens=as_int(payload.get("total_tokens")), + cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")), + cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")), + ) @dataclass(frozen=True) @@ -363,11 +379,7 @@ class LLMCallSpanData: response_model=context.response_model, response_id=as_str(response.get("id")), request_params=LLMRequestParams.from_model_parameters(params), - usage=LLMUsage( - input_tokens=as_int(payload.get("prompt_tokens")), - output_tokens=as_int(payload.get("completion_tokens")), - total_tokens=as_int(payload.get("total_tokens")), - ), + usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), response_cost=as_float(payload.get("response_cost")), diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 4ad0cb5d1b4..f7a6280f95b 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -110,6 +110,8 @@ class GenAI: # usage USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" + USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens" + USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens" # content (opt-in, gated by capture mode) INPUT_MESSAGES: Final = "gen_ai.input.messages" OUTPUT_MESSAGES: Final = "gen_ai.output.messages" diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 9e8976940e6..3732ffd734c 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -698,6 +698,26 @@ def _count_document_tokens( ) +def _count_file_tokens( + file_value: object, + count_function: TokenCounterFunction, + use_default_image_token_count: bool, +) -> int: + """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" + if not isinstance(file_value, Mapping): + return 0 + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 + if not isinstance(file_data, str) or not file_data: + return name_tokens + return name_tokens + calculate_img_tokens( + data=file_data, + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + + def _count_anthropic_content( content: Mapping[str, object], count_function: TokenCounterFunction, @@ -783,6 +803,12 @@ def _count_content_list( use_default_image_token_count, default_token_count, ) + elif c["type"] == "file": + num_tokens += _count_file_tokens( + c.get("file"), + count_function, + use_default_image_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -812,7 +838,7 @@ def _count_content_list( raise ValueError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)." + f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index db9c8a5cedd..395d99a4caa 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1631,6 +1631,11 @@ class AmazonConverseConfig(BaseConfig): bedrock_tool_config["toolChoice"] = tool_choice_values self._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params) + config_block_entries: Final = tuple( + (config_name, config_class, inference_params.pop(config_name, None)) + for config_name, config_class in self.get_config_blocks().items() + ) + data: Final[CommonRequestObject] = { "inferenceConfig": self._transform_inference_params(inference_params=inference_params), } @@ -1641,9 +1646,7 @@ class AmazonConverseConfig(BaseConfig): if system_content_blocks: data["system"] = system_content_blocks - # Handle all config blocks - for config_name, config_class in self.get_config_blocks().items(): - config_value = inference_params.pop(config_name, None) + for config_name, config_class, config_value in config_block_entries: if config_value is not None: data[config_name] = config_class(**config_value) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 1b1ab80e85d..4d774f6f165 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -268,6 +268,7 @@ class BaseOpenAILLM: "max_retries", "organization", "api_base", + "workload_identity_config", ) openai_client_fields: Final = ( BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type=client_type) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 6e66c998acf..16fa0017b23 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -51,6 +51,7 @@ from .common_utils import ( drop_params_from_unprocessable_entity_error, is_output_token_limit_error, ) +from .workload_identity import resolve_openai_workload_identity_config openaiOSeriesConfig: Final = OpenAIOSeriesConfig() openAIGPT5Config: Final = OpenAIGPT5Config() @@ -349,6 +350,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client: OpenAI | AsyncOpenAI | None = None, shared_session: Optional["ClientSession"] = None, ) -> OpenAI | AsyncOpenAI | None: + workload_identity_config: Final = resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base) client_initialization_params: Final[dict] = locals() if client is None: if not isinstance(max_retries, int): @@ -364,28 +366,49 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client - http_client: Final[httpx.Client | httpx.AsyncClient | None] = ( - OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) - if is_async - else OpenAIChatCompletion._get_sync_http_client() - ) if is_async: - _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=http_client, - timeout=timeout, - max_retries=max_retries, - organization=organization, + async_http_client: Final = OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) + http_client: httpx.Client | httpx.AsyncClient | None = async_http_client + _new_client: OpenAI | AsyncOpenAI = ( + AsyncOpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + http_client=async_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + if workload_identity_config is not None + else AsyncOpenAI( + api_key=api_key, + base_url=api_base, + http_client=async_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) ) else: - _new_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=http_client, - timeout=timeout, - max_retries=max_retries, - organization=organization, + sync_http_client: Final = OpenAIChatCompletion._get_sync_http_client() + http_client = sync_http_client + _new_client = ( + OpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + http_client=sync_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + if workload_identity_config is not None + else OpenAI( + api_key=api_key, + base_url=api_base, + http_client=sync_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) ) ## SAVE CACHE KEY diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 6b2f4535df1..88f04c59e01 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic. This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint. """ -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal + +from typing_extensions import ReadOnly, TypedDict + + +class ResponsesInputTextPart(TypedDict): + type: ReadOnly[Literal["input_text"]] + text: ReadOnly[str] + + +class ResponsesInputImagePart(TypedDict): + type: ReadOnly[Literal["input_image"]] + image_url: ReadOnly[str] + detail: ReadOnly[str] + + +class ResponsesInputFilePart(TypedDict): + type: ReadOnly[Literal["input_file"]] + filename: ReadOnly[str] + file_data: ReadOnly[str] + + +ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart + +ResponsesContentRole = Literal["user", "assistant"] + + +def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None: + url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url + if not isinstance(url, str) or not url: + return None + detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None + part: Final[ResponsesInputImagePart] = { + "type": "input_image", + "image_url": url, + "detail": detail if isinstance(detail, str) and detail else "auto", + } + return part + + +def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None: + """Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it.""" + if not isinstance(file_value, Mapping): + return None + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data: + return None + part: Final[ResponsesInputFilePart] = { + "type": "input_file", + "filename": filename, + "file_data": file_data, + } + return part + + +def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None: + if isinstance(block, str): + bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block} + return bare + if not isinstance(block, Mapping): + return None + match block.get("type"): + case "text": + text_value: Final = block.get("text") + text: Final[ResponsesInputTextPart] = { + "type": "input_text", + "text": text_value if isinstance(text_value, str) else "", + } + return text + case "image_url" if role == "user": + return _chat_image_block_to_responses_part(block.get("image_url")) + case "file" if role == "user": + return _chat_file_block_to_responses_part(block.get("file")) + case _: + return None + + +def chat_content_blocks_to_responses_content( + content: Sequence[object], + role: ResponsesContentRole, +) -> str | tuple[ResponsesInputPart, ...]: + """Text-only content collapses to a joined string, which every role accepts and counts identically. + + Only a user turn may carry an image or file part: the Responses API rejects any part but + output_text and refusal inside an assistant turn. + """ + parts: Final = tuple( + part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None + ) + if any(part["type"] != "input_text" for part in parts): + return parts + return "\n".join(part["text"] for part in parts if part["type"] == "input_text") class OpenAICountTokensConfig: @@ -120,18 +213,13 @@ class OpenAICountTokensConfig: instructions_parts.append("\n".join(text_parts)) elif role == "user": if isinstance(content, list): - # Extract text from content blocks for Responses API - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - content = "\n".join(text_parts) + content = chat_content_blocks_to_responses_content(content, "user") input_items.append({"role": "user", "content": content}) elif role == "assistant": # Map tool_calls to Responses API function_call items tool_calls = msg.get("tool_calls") + if isinstance(content, list): + content = chat_content_blocks_to_responses_content(content, "assistant") if content: input_items.append({"role": "assistant", "content": content}) if tool_calls: diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index eac844a790d..eadc087383a 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -21,6 +21,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from ..common_utils import OpenAIError +from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -392,6 +393,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm_params = litellm_params or GenericLiteLLMParams() api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") headers.setdefault("Content-Type", "application/json") + workload_identity_config: Final = ( + resolve_openai_workload_identity_config(api_key=api_key, api_base=litellm_params.api_base) + if self.custom_llm_provider is LlmProviders.OPENAI + else None + ) + if workload_identity_config is not None: + headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}" + return headers headers["Authorization"] = f"Bearer {api_key}" return headers diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py new file mode 100644 index 00000000000..ecec161ed46 --- /dev/null +++ b/litellm/llms/openai/workload_identity.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import litellm +from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str + +from .common_utils import OpenAIError + +if TYPE_CHECKING: + from collections.abc import Callable + + from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth + +OPENAI_WIF_CLIENT_ID: Final = "litellm" +_OPENAI_API_HOST: Final = "api.openai.com" +_SDK_UPGRADE_MESSAGE: Final = ( + "OpenAI workload identity federation requires openai>=2.32.0. " + "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / " + "OPENAI_SERVICE_ACCOUNT_ID / OPENAI_IDENTITY_TOKEN_FILE." +) + + +@dataclass(frozen=True, slots=True) +class OpenAIWorkloadIdentityConfig: + identity_provider_id: str + service_account_id: str + token_file: str + + def to_sdk_workload_identity(self) -> WorkloadIdentity: + k8s_token_provider: Final = _load_sdk_k8s_token_provider() + workload_identity: Final[WorkloadIdentity] = { + "client_id": OPENAI_WIF_CLIENT_ID, + "identity_provider_id": self.identity_provider_id, + "service_account_id": self.service_account_id, + "provider": k8s_token_provider(self.token_file), + } + return workload_identity + + +def resolve_openai_workload_identity_config( + api_key: str | None, + api_base: str | None, +) -> OpenAIWorkloadIdentityConfig | None: + static_api_key: Final = normalize_nonempty_secret_str(api_key) or normalize_nonempty_secret_str( + get_secret_str("OPENAI_API_KEY") + ) + if static_api_key is not None: + return None + effective_api_base: Final = ( + api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") + ) + if not _targets_openai_api(effective_api_base): + return None + identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID") + service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID") + token_file: Final = get_secret_str("OPENAI_IDENTITY_TOKEN_FILE") + if not identity_provider_id or not service_account_id or not token_file: + return None + return OpenAIWorkloadIdentityConfig( + identity_provider_id=identity_provider_id, + service_account_id=service_account_id, + token_file=token_file, + ) + + +def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) -> str: + return _workload_identity_auth(config).get_token() + + +def _targets_openai_api(api_base: str | None) -> bool: + if api_base is None: + return True + parsed: Final = urlparse(api_base) + return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST + + +@lru_cache(maxsize=16) +def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth: + sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth() + return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity()) + + +def _load_sdk_workload_identity_auth() -> type[WorkloadIdentityAuth]: + try: + from openai.auth import WorkloadIdentityAuth as sdk_workload_identity_auth + except ImportError as e: + raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e + return sdk_workload_identity_auth + + +def _load_sdk_k8s_token_provider() -> Callable[[str], SubjectTokenProvider]: + try: + from openai.auth import k8s_service_account_token_provider + except ImportError as e: + raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e + return k8s_service_account_token_provider diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 75098515deb..aca257dc095 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -27,6 +27,15 @@ from .common_utils import ( get_vertex_base_url, ) + +def _graft_default_vertex_path(api_base: str, default_url: str) -> str: + parsed_api_base: Final = urlparse(api_base) + default_segments: Final = urlparse(default_url).path.lstrip("/").split("/") + graft_segments: Final = default_segments[1:] if default_segments[0] in ("v1", "v1beta1") else default_segments + grafted_path: Final = parsed_api_base.path.rstrip("/") + "/" + "/".join(graft_segments) + return parsed_api_base._replace(path=grafted_path).geturl() + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -621,8 +630,9 @@ class VertexBase: Handles custom api_base for: 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} - 2. Vertex AI with standard proxies - constructs {api_base}:{endpoint}; - if api_base has no path (bare host), grafts the default vertex URL path onto it + 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the + api_base when its path is empty or only an API version (/v1, /v1beta1); + otherwise constructs {api_base}:{endpoint} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -669,10 +679,14 @@ class VertexBase: ) elif urlparse(api_base).path in ("", "/"): url = api_base.rstrip("/") + urlparse(url).path + elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: + url = _graft_default_vertex_path(api_base=api_base, default_url=url) else: url = f"{api_base}:{endpoint}" if stream is True: - url = url + "?alt=sse" + parsed_stream_url: Final = urlparse(url) + stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" + url = parsed_stream_url._replace(query=stream_query).geturl() return auth_header, url def _get_token_and_url( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 05c1cfd3179..7071eaa0807 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3726,7 +3726,7 @@ "output_cost_per_token": 0, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-services/", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure/eu/gpt-4o-2024-08-06": { @@ -5328,7 +5328,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/gpt-realtime-whisper", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -9107,7 +9107,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/Cohere-embed-v3-multilingual": { @@ -9118,7 +9118,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/FLUX-1.1-pro": { @@ -9134,7 +9134,7 @@ "litellm_provider": "azure_ai", "mode": "image_generation", "output_cost_per_image": 0.04, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/images/generations" ] @@ -9467,7 +9467,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9481,7 +9481,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.04e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9494,7 +9494,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -9543,7 +9543,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -9554,7 +9554,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { @@ -9566,7 +9566,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.1e-07, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -9756,7 +9756,7 @@ "supported_endpoints": [ "/v1/ocr" ], - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", @@ -9967,7 +9967,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/embeddings" ], @@ -10151,7 +10151,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://azure.microsoft.com/en-us/products/ai-services/ai-foundry/models/jais-30b-chat" + "source": "https://ai.azure.com/catalog/models/jais-30b-chat" }, "azure_ai/jamba-instruct": { "input_cost_per_token": 5e-07, @@ -10208,7 +10208,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10231,7 +10231,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10243,7 +10243,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10280,7 +10280,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", "supports_function_calling": true }, "azure_ai/mistral-small": { @@ -12315,7 +12315,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "claude-haiku-4-5-20251001": { "deprecation_date": "2026-10-15", @@ -17767,7 +17768,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "deprecation_date": "2027-01-08" }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -22843,7 +22845,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22897,7 +22899,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22962,7 +22964,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23021,7 +23023,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23226,7 +23228,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23310,7 +23312,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23373,7 +23375,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23430,7 +23432,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25327,7 +25329,7 @@ "supports_vision": true }, "gpt-4o-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -25650,7 +25652,7 @@ "supports_vision": true }, "gpt-4o-mini-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -25688,7 +25690,7 @@ "gpt-4o-mini-realtime-preview": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -25787,7 +25789,8 @@ "output_cost_per_token": 5e-06, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-4o-mini-tts": { "input_cost_per_token": 2.5e-06, @@ -25809,7 +25812,7 @@ }, "gpt-4o-realtime-preview": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25828,7 +25831,7 @@ }, "gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25847,7 +25850,7 @@ }, "gpt-4o-realtime-preview-2025-06-03": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25926,7 +25929,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, @@ -27077,7 +27081,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/gpt-5.6-cyber", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-cyber", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27116,7 +27120,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27156,7 +27160,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -27168,7 +27172,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "source": "https://platform.openai.com/docs/models/chat-latest", + "source": "https://developers.openai.com/api/docs/models/chat-latest", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -30463,7 +30467,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30504,7 +30508,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30545,7 +30549,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30578,7 +30582,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30594,7 +30598,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30610,7 +30614,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -30627,7 +30631,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -32572,7 +32576,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 164000, @@ -32584,7 +32588,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { "max_tokens": 128000, @@ -32595,7 +32599,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { "max_tokens": 128000, @@ -32606,7 +32610,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 128000, @@ -32617,7 +32621,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { "max_tokens": 128000, @@ -32629,7 +32633,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -32640,7 +32644,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { "max_tokens": 128000, @@ -32650,7 +32654,7 @@ "output_cost_per_token": 6e-08, "litellm_provider": "nebius", "mode": "chat", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 128000, @@ -32661,7 +32665,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { "max_tokens": 128000, @@ -32672,7 +32676,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { "max_tokens": 128000, @@ -32683,7 +32687,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { "max_tokens": 128000, @@ -32694,7 +32698,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 128000, @@ -32705,7 +32709,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { "max_tokens": 128000, @@ -32716,7 +32720,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { "max_tokens": 131072, @@ -32727,7 +32731,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { "max_tokens": 262144, @@ -32738,7 +32742,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { "max_tokens": 32768, @@ -32749,7 +32753,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -32760,7 +32764,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { "max_tokens": 32768, @@ -32771,7 +32775,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { "max_tokens": 32768, @@ -32782,7 +32786,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { "max_tokens": 32768, @@ -32794,7 +32798,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { "max_tokens": 128000, @@ -32805,7 +32809,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { "max_tokens": 128000, @@ -32816,7 +32820,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { "max_tokens": 32768, @@ -32827,7 +32831,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { "max_tokens": 131072, @@ -32839,7 +32843,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -32851,7 +32855,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { "max_tokens": 131072, @@ -32862,7 +32866,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, @@ -32871,7 +32875,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { "max_tokens": 8192, @@ -32880,7 +32884,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { "max_tokens": 32768, @@ -32889,7 +32893,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, @@ -33630,7 +33634,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33643,7 +33647,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33656,7 +33660,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33713,7 +33717,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true, @@ -33727,7 +33731,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": false, "supports_response_schema": false, "supports_native_streaming": true @@ -33738,7 +33742,7 @@ "max_input_tokens": 512, "mode": "embedding", "output_vector_size": 1024, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -34635,7 +34639,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", + "source": "https://openrouter.ai/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { @@ -34870,7 +34874,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -34911,7 +34915,7 @@ "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -35996,7 +36000,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/deepseek-r1-distill-llama-70b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36010,7 +36014,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llama-3-1-8b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36023,7 +36027,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-1-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": false, "supports_tool_choice": false @@ -36036,7 +36040,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-3-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36049,7 +36053,7 @@ "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-7b-instruct-v0-3", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36062,7 +36066,7 @@ "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-nemo-instruct-2407", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36075,7 +36079,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-small-3-2-24b-instruct-2506", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -36089,7 +36093,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mixtral-8x7b-instruct-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36102,7 +36106,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-coder-32b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36115,7 +36119,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-vl-72b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36129,7 +36133,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen3-32b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36143,7 +36147,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-120b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36157,7 +36161,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-20b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36171,7 +36175,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llava-next-mistral-7b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36185,7 +36189,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mamba-codestral-7b-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -38321,7 +38325,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { - "deprecation_date": "2026-01-14", + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38946,7 +38950,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, - "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", + "source": "https://www.together.ai/models/qwen3-5-397b-a17b", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -43183,7 +43187,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43199,7 +43203,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43216,7 +43220,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43232,7 +43236,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43352,7 +43356,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43366,7 +43370,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43381,7 +43385,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43396,7 +43400,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -44100,7 +44104,8 @@ "output_cost_per_second": 0.0001, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "xai/grok-3": { "cache_read_input_token_cost": 2e-07, @@ -45088,7 +45093,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2", "supported_modalities": [ "text" ], @@ -45100,7 +45105,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -45112,7 +45117,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -49322,7 +49327,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-realtime-whisper", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -50574,7 +50579,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50612,7 +50617,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50650,7 +50655,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50688,7 +50693,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -51437,7 +51442,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { "max_tokens": 131072, @@ -51450,7 +51455,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { "max_tokens": 131072, @@ -51463,7 +51468,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { "max_tokens": 131072, @@ -51476,7 +51481,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { "max_tokens": 163840, @@ -51489,7 +51494,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { "max_tokens": 1000192, @@ -51502,7 +51507,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, @@ -51606,7 +51611,7 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-transcribe", "supported_endpoints": [ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" @@ -51624,7 +51629,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-live-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-live-transcribe", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -51645,7 +51650,7 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", - "source": "https://platform.openai.com/docs/models/gpt-realtime-translate", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-translate", "supported_modalities": [ "audio" ], @@ -51673,7 +51678,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -51712,7 +51717,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_assistant_prefill": false, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ba5e8fa1aa..7075fa5a31a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum): "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", "/openai/v1/responses/{response_id}/cancel", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", # vector stores "/vector_stores", "/v1/vector_stores", @@ -2541,7 +2544,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ) alerting: list | None = Field( None, - description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", + description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL", ) alert_types: list[AlertType] | None = Field( None, @@ -3549,6 +3552,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ) +class SpendLogsRouterMetadata(TypedDict): + """ + Router provenance stamped on spend logs for deployments flagged with + model_info.internal_router_model, correlating the requested model group + with the provider deployment that served the call + """ + + requested_model: ReadOnly[str | None] + selected_model: ReadOnly[str | None] + selected_provider: ReadOnly[str | None] + router_correlation_id: ReadOnly[str | None] + + class SpendLogsMetadata(TypedDict): """ Specific metadata k,v pairs logged to spendlogs for easier cost tracking @@ -3591,6 +3607,7 @@ class SpendLogsMetadata(TypedDict): compression_savings: CompressionSavingsMetadata | None autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed litellm_gateway_injected_cache: ReadOnly[str | None] + router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index fe417396317..3ddce35b53d 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -489,7 +489,7 @@ lite codex exec "summarize the repo" Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process. -The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). +The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). Options (these belong to the wrapper, so put them before the agent's own flags): @@ -505,7 +505,7 @@ The credential is short-lived by design (default 24h, configurable via `LITELLM_ ### Route Every Claude Code Session Through the Proxy -`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. +`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` when that key is missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it. Two things need to already be true: you've run `lite login` (or `lite login --pkce`, whose key the helper renews on its own), since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you. @@ -529,7 +529,7 @@ Cursor is not supported: it has no equivalent file-based config to hot-patch thi lite --base-url https://your-proxy.example.com login --config-claude ``` -It writes the same two settings `lite up` does, `env.ANTHROPIC_BASE_URL` and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. +It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. Because the credential is reached through `apiKeyHelper` rather than copied into the file, a later `lite login` refreshes it with no further action: Claude Code re-runs the helper on every request and picks up whatever token the most recent login stored. Nothing secret is written to `settings.json`. diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index e05e85ae483..45e05d353fb 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -13,6 +13,8 @@ from .auth import context_secret_vault, get_stored_api_key, login ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL" ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_API_KEY_ENV: Final = "ANTHROPIC_API_KEY" +ENABLE_TOOL_SEARCH_ENV: Final = "ENABLE_TOOL_SEARCH" +ENABLE_TOOL_SEARCH_VALUE: Final = "true" OPENAI_BASE_URL_ENV: Final = "OPENAI_BASE_URL" OPENAI_API_KEY_ENV: Final = "OPENAI_API_KEY" @@ -61,7 +63,10 @@ def build_agent_env( Anthropic clients (Claude Code) append /v1/messages to ANTHROPIC_BASE_URL, so it stays the bare proxy root; OpenAI clients (Codex, OpenCode) expect the /v1 suffix on OPENAI_BASE_URL. ANTHROPIC_API_KEY is dropped so a stray - Anthropic key cannot win over the bearer token we set. + Anthropic key cannot win over the bearer token we set. ENABLE_TOOL_SEARCH + defaults to true because Claude Code turns tool search off when + ANTHROPIC_BASE_URL is not a first-party Anthropic host; a value already in + the environment is left alone. """ env: Final = dict(base_env) root: Final = base_url.rstrip("/") @@ -69,6 +74,8 @@ def build_agent_env( env[ANTHROPIC_BASE_URL_ENV] = root env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key env.pop(ANTHROPIC_API_KEY_ENV, None) + if ENABLE_TOOL_SEARCH_ENV not in env: + env[ENABLE_TOOL_SEARCH_ENV] = ENABLE_TOOL_SEARCH_VALUE if PROFILE_OPENAI in profiles: env[OPENAI_BASE_URL_ENV] = root + "/v1" env[OPENAI_API_KEY_ENV] = api_key diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py index 9fcb11a585b..60729b5410d 100644 --- a/litellm/proxy/client/cli/commands/autoroute/settings.py +++ b/litellm/proxy/client/cli/commands/autoroute/settings.py @@ -9,6 +9,8 @@ API_KEY_HELPER_KEY: Final = "apiKeyHelper" ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" +ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" +ENABLE_TOOL_SEARCH_VALUE: Final = "true" # Force every one of Claude Code's own model tiers to request the auto-router by name. # Router's auto-router registry is keyed by the literal requested model string # (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*" @@ -34,6 +36,7 @@ def merge_claude_settings_static_token( raw_env: Final = settings.get(ENV_KEY, {}) base_env: Final = raw_env if isinstance(raw_env, dict) else {} env: Final[dict[str, JsonValue]] = { + ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, **base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), ANTHROPIC_AUTH_TOKEN_KEY: auth_token, diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index e18e5b1b7ee..ea1fa019c83 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -21,6 +21,8 @@ ENV_KEY: Final = "env" API_KEY_HELPER_KEY: Final = "apiKeyHelper" ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" +ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH" +ENABLE_TOOL_SEARCH_VALUE: Final = "true" CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" @@ -70,12 +72,15 @@ def merge_claude_settings( Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued - token (same reasoning as build_agent_env in agents.py). Every other key is - preserved untouched. + token (same reasoning as build_agent_env in agents.py). ENABLE_TOOL_SEARCH + defaults to true because Claude Code turns tool search off when + ANTHROPIC_BASE_URL is not a first-party Anthropic host; an existing value is + left alone. Every other key is preserved untouched. """ raw_env: Final = settings.get(ENV_KEY, {}) base_env: Final = raw_env if isinstance(raw_env, dict) else {} env: Final = { + ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE, **{key: value for key, value in base_env.items() if key != ANTHROPIC_API_KEY_KEY}, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), } @@ -144,6 +149,8 @@ __all__ = ( "AUTOROUTE_BACKUP_PATH", "BACKUP_PATH", "CLAUDE_SETTINGS_PATH", + "ENABLE_TOOL_SEARCH_KEY", + "ENABLE_TOOL_SEARCH_VALUE", "ENV_KEY", "SETTINGS_FILE_OWNERS", "ClaudeSettingsError", diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index a6ea2e09583..68914a1989e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -156,6 +156,31 @@ def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: return headers.get(key, default) +def _is_image_part(item: object) -> bool: + """Whether a structured-message content part carries an image rather than text.""" + + if not isinstance(item, Mapping): + return False + + part: Final[Mapping[object, object]] = item + return part.get("type") == "image_url" + + +def _scannable_text(content: object) -> str: + """Flatten a structured message's content into the single string the v1 detection endpoint takes. + + Image parts are dropped: the endpoint accepts one string, so an image would only reach it as + its stringified source (a base64 blob or a URL), which is not text the scanner can evaluate. + """ + + if not isinstance(content, list): + return str(content or "") + + parts: Final[Sequence[object]] = content + text_parts: Final = [item for item in parts if not _is_image_part(item)] # mutable-ok: sent as a list repr + return str(text_parts or "") + + def is_saas(host: str) -> bool: """Checks whether the connection is to the SaaS platform""" @@ -270,7 +295,7 @@ class HiddenlayerGuardrail(CustomGuardrail): "messages": [ { "role": last_msg.get("role", "user"), - "content": str(last_msg.get("content", "")), + "content": _scannable_text(last_msg.get("content")), } ] }, diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py index fa1f9f3d36d..0aaba4016cd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py @@ -20,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, default_on=litellm_params.default_on, + file_sanitization_fail_open=getattr(litellm_params, "file_sanitization_fail_open", None), ) litellm.logging_callback_manager.add_litellm_callback(_prompt_security_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 809d5e0fb31..84c4f118b00 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -4,10 +4,12 @@ import os from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional +import httpx from fastapi import HTTPException from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.exceptions import Timeout as LiteLLMTimeout from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -24,6 +26,9 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS: Final = 30.0 + + class PromptSecurityGuardrailMissingSecrets(Exception): pass @@ -63,6 +68,13 @@ class _SanitizeStatusResponse(TypedDict, total=False): metadata: ReadOnly[_SanitizeMetadata] +class _SanitizeResult(TypedDict): + action: ReadOnly[str] + content: ReadOnly[str | None] + metadata: ReadOnly[_SanitizeMetadata] + violations: ReadOnly[Sequence[str]] + + class PromptSecurityGuardrail(CustomGuardrail): @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: @@ -79,6 +91,8 @@ class PromptSecurityGuardrail(CustomGuardrail): user: str | None = None, system_prompt: str | None = None, check_tool_results: bool | None = None, + file_sanitization_timeout: float = _SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS, + file_sanitization_fail_open: bool | None = None, **kwargs, ): kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) @@ -108,6 +122,8 @@ class PromptSecurityGuardrail(CustomGuardrail): # Configuration for file sanitization self.max_poll_attempts = 30 # Maximum number of polling attempts self.poll_interval = 2 # Seconds between polling attempts + self.file_sanitization_timeout = file_sanitization_timeout + self.file_sanitization_fail_open = file_sanitization_fail_open is not False super().__init__(**kwargs) @@ -397,6 +413,39 @@ class PromptSecurityGuardrail(CustomGuardrail): Sanitize file content using Prompt Security API. Returns: dict with keys 'action', 'content', 'metadata' """ + try: + return await asyncio.wait_for( + self._sanitize_file_content(file_data, filename, user_api_key_alias), + timeout=self.file_sanitization_timeout, + ) + except (asyncio.TimeoutError, httpx.TimeoutException, LiteLLMTimeout) as exc: + if not self.file_sanitization_fail_open: + verbose_proxy_logger.error( + "Prompt Security Guardrail: file sanitization for %s timed out with %s; failing closed", + filename, + type(exc).__name__, + ) + raise HTTPException(status_code=408, detail="File sanitization timeout") from exc + + verbose_proxy_logger.error( + "Prompt Security Guardrail: file sanitization for %s timed out with %s; failing open", + filename, + type(exc).__name__, + ) + fail_open_result: Final[_SanitizeResult] = { + "action": "allow", + "content": None, + "metadata": {}, + "violations": (), + } + return fail_open_result + + async def _sanitize_file_content( + self, + file_data: bytes, + filename: str, + user_api_key_alias: str | None, + ) -> _SanitizeResult: headers: Final = {"APP-ID": self.api_key} if user_api_key_alias: headers["X-LiteLLM-Key-Alias"] = user_api_key_alias diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 9ef3d2defef..d2d87331d55 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -626,11 +626,7 @@ async def update_end_user( # get non default values for key non_default_values: Final = dict[str, object]() for k, v in data_json.items(): - if v is not None and v not in ( - [], - {}, - 0, - ): # models default to [], spend defaults to 0, we should not reset these values + if v is not None and ((isinstance(v, bool) and k in data.fields_set()) or v not in ([], {}, 0)): non_default_values[k] = v ## Get end user table data ## diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 9c4c9948c10..c08ca5b7783 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1027,6 +1027,14 @@ async def user_info_v2( This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem where the old endpoint loaded all keys and teams into memory. + Note on `spend`: this is the user's running budget counter, which the budget reset job + resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default, + or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT + lifetime or per-period historical spend. For historical spend over a date range, use + `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend + records that only ever accumulate and are never reset. The two values are expected to + diverge once a budget reset has occurred within the queried period. + Access control: - Proxy admins can query any user - Team admins can query users within their teams @@ -2726,6 +2734,11 @@ async def get_user_daily_activity( Meant to optimize querying spend data for analytics for a user. + Reads daily spend records that only ever accumulate and are never affected by budget + resets. Their total can legitimately exceed the `spend` field returned by + `/v2/user/info`, which is a running budget counter that every budget reset sets back + to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). + Returns: (by date) - spend @@ -2839,6 +2852,11 @@ async def get_user_daily_activity_aggregated( """ Aggregated analytics for a user's daily activity without pagination. Returns the same response shape as the paginated endpoint with page metadata set to single-page. + + Reads daily spend records that only ever accumulate and are never affected by budget + resets. Their total can legitimately exceed the `spend` field returned by + `/v2/user/info`, which is a running budget counter that every budget reset sets back + to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). """ from litellm.proxy.proxy_server import prisma_client diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 08346983f32..c2f5dbb4032 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -252,7 +252,7 @@ async def add_team_callbacks( Use this if if you want different teams to have different success/failure callbacks Parameters: - - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add + - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of: - "success": Callback for successful LLM calls - "failure": Callback for failed LLM calls @@ -268,6 +268,8 @@ async def add_team_callbacks( - langsmith_api_key: The API key for the Langsmith callback - langsmith_project: The project for the Langsmith callback - langsmith_base_url: The base URL for the Langsmith callback + - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400 + - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key Example curl: ``` diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 887716a383a..da2e09bd7f6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1363,7 +1363,7 @@ _OPENAPI_HTTP_METHODS: Final = { # the UI. Kept here at module scope to match the analogous descriptor # `is_secret` flags in litellm.proxy.config_resolvers and the # `_CACHE_SENSITIVE_FIELDS` constant in the cache endpoint file. -_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"} +_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"ALERTING_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SMTP_PASSWORD"} def _strip_operation_id_method_suffix(operation_id: str) -> str: @@ -16566,6 +16566,7 @@ async def create_config_audit_log( _EXTRA_SECRET_CALLBACK_ENV_VARS: Final = frozenset( { + "ALERTING_WEBHOOK_URL", "GALILEO_USERNAME", "GENERIC_LOGGER_HEADERS", "OTEL_HEADERS", diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 5e56e822484..aa7595ed13d 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,14 +1,18 @@ import asyncio import json import time -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Mapping +from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args from uuid import uuid4 import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import JSONResponse +from openai.types.responses.response_create_params import ResponseInputParam from starlette.websockets import WebSocket, WebSocketDisconnect +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ModifyResponseException @@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_set_request_parsed_body, ) -from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.llms.openai import ( + REASONING_EFFORT, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, +) from litellm.types.responses.main import DeleteResponseResult +from litellm.types.utils import TokenCountResponse if TYPE_CHECKING: from litellm.router import Router @@ -35,7 +44,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _user_api_key_auth_dep: Final = Depends(user_api_key_auth) -_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags +_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags _TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( { @@ -1017,6 +1026,152 @@ async def compact_response( ) +class _ResponsesApiErrorDetail(TypedDict): + message: ReadOnly[str] + type: ReadOnly[str] + param: ReadOnly[str | None] + code: ReadOnly[str | None] + + +class _ResponsesApiErrorBody(TypedDict): + error: ReadOnly[_ResponsesApiErrorDetail] + + +class _ResponsesInputTokensResult(TypedDict): + object: ReadOnly[str] + input_tokens: ReadOnly[int] + + +class _TokenCountPayload(TypedDict): + model: ReadOnly[str] + messages: ReadOnly[tuple[Mapping[str, object], ...]] + tools: ReadOnly[object] + + +class _TokenCounter(Protocol): + def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ... + + +def _proxy_token_counter() -> _TokenCounter: + from litellm.proxy.proxy_server import token_counter + + return token_counter + + +_token_counter_dep: Final = Depends(_proxy_token_counter) + + +def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: + body: Final[_ResponsesApiErrorBody] = { + "error": { + "message": message, + "type": "invalid_request_error", + "param": param, + "code": code, + } + } + return JSONResponse(status_code=400, content=body) + + +def _missing_responses_param_response(param: str) -> JSONResponse: + return _responses_invalid_request_response( + message=f"Missing required parameter: '{param}'.", + param=param, + code="missing_required_parameter", + ) + + +def _responses_input_as_token_count_messages( + input_value: str | ResponseInputParam, + instructions: str | None, +) -> tuple[Mapping[str, object], ...]: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions} + transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=input_value, + responses_api_request=request_params, + ) + return tuple( + message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed + ) + + +@router.post( + "/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/openai/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +async def responses_input_tokens( + request: Request, + token_counter: _TokenCounter = _token_counter_dep, +): + """ + Count the input tokens of a Responses API request without calling the model. + + Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + + ```bash + curl -X POST http://localhost:4000/v1/responses/input_tokens \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": "Hello, how are you?" + }' + ``` + + Returns: `{"object": "response.input_tokens", "input_tokens": }` + """ + data: Final = await _read_request_body(request=request) + model_name: Final = data.get("model") + input_value: Final = data.get("input") + if not isinstance(model_name, str) or not model_name: + return _missing_responses_param_response("model") + if input_value is None: + return _missing_responses_param_response("input") + if isinstance(input_value, (str, list)) and not input_value: + return _responses_invalid_request_response( + message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + param=None, + code="missing_required_parameter", + ) + + try: + payload: Final[_TokenCountPayload] = { + "model": model_name, + "messages": _responses_input_as_token_count_messages( + input_value=input_value, + instructions=data.get("instructions"), + ), + "tools": data.get("tools"), + } + token_request: Final = TokenCountRequest.model_validate(payload) + except Exception as e: + return _responses_invalid_request_response( + message=f"Invalid request for token counting: {e}", param=None, code=None + ) + + token_response: Final = await token_counter(request=token_request, call_endpoint=True) + result: Final[_ResponsesInputTokensResult] = { + "object": "response.input_tokens", + "input_tokens": token_response.total_tokens, + } + return result + + @router.post( "/v1/responses/{response_id}/cancel", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 4a63a8e25fe..f4d8fd7d906 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -173,7 +173,14 @@ async def reserve_budget_for_request( ) -> dict | None: if valid_token is None or not RouteChecks.is_llm_api_route(route=route): return None - if route in {"/models", "/v1/models", "/utils/token_counter"}: + if route in { + "/models", + "/v1/models", + "/utils/token_counter", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + }: return None if get_model_from_request(request_body, route, llm_router=llm_router) is None: return None diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ffde44e0f9e..7442d71bd96 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -30,7 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import ( request_model_access_groups_from_litellm_params, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes -from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload +from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsRouterMetadata from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token from litellm.types.utils import ( @@ -93,6 +93,24 @@ def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) return hash_token(stripped) +def _get_router_metadata_for_spend_log( + metadata: Mapping[str, object] | None, + requested_model: str | None, + selected_model: str | None, + selected_provider: str | None, + router_correlation_id: str | None, +) -> SpendLogsRouterMetadata | None: + model_info: Final = metadata.get("model_info") if metadata is not None else None + if not isinstance(model_info, Mapping) or model_info.get("internal_router_model") is not True: + return None + return SpendLogsRouterMetadata( + requested_model=requested_model or None, + selected_model=selected_model or None, + selected_provider=selected_provider or None, + router_correlation_id=router_correlation_id, + ) + + def _get_spend_logs_metadata( metadata: dict | None, applied_guardrails: list[str] | None = None, @@ -109,6 +127,7 @@ def _get_spend_logs_metadata( cost_breakdown: CostBreakdown | None = None, litellm_call_id: str | None = None, autorouter_savings: float | None = None, + router_metadata: SpendLogsRouterMetadata | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -148,13 +167,17 @@ def _get_spend_logs_metadata( autorouter_savings=autorouter_savings, litellm_gateway_injected_cache=None, litellm_call_id=litellm_call_id, + router_metadata=router_metadata, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " + str(list(metadata.keys())) ) # Filter the metadata dictionary to include only the specified keys - clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__}) + clean_metadata: Final = SpendLogsMetadata( + **{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"}, + router_metadata=router_metadata, + ) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") _already_redacted: Final = ( @@ -394,6 +417,20 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs hidden_params: Final = standard_logging_payload.get("hidden_params", {}) litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + custom_llm_provider: Final = ( + kwargs.get("custom_llm_provider") + or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") + or None + ) + raw_model: Final = cast(str, kwargs.get("model") or "") + model_name: Final = ( + standard_logging_payload.get("model") if standard_logging_payload is not None else None + ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + litellm_call_id: Final = cast( + str | None, + kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), + ) + # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( metadata, @@ -452,9 +489,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs autorouter_savings=( standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None ), - litellm_call_id=cast( - str | None, - kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), + litellm_call_id=litellm_call_id, + router_metadata=_get_router_metadata_for_spend_log( + metadata=metadata, + requested_model=_model_group, + selected_model=model_name, + selected_provider=custom_llm_provider, + router_correlation_id=litellm_call_id, ), ) @@ -499,15 +540,6 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs # Extract agent_id for A2A requests (set directly on model_call_details) agent_id: Final[str | None] = kwargs.get("agent_id") or metadata.get("agent_id") - custom_llm_provider: Final = ( - kwargs.get("custom_llm_provider") - or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") - or None - ) - raw_model: Final = cast(str, kwargs.get("model") or "") - model_name: Final = ( - standard_logging_payload.get("model") if standard_logging_payload is not None else None - ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) try: payload: Final[SpendLogsPayload] = SpendLogsPayload( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cf56fc0b1dd..051d36c4d0f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -645,7 +645,7 @@ class ProxyLogging: self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(self.internal_usage_cache) self.max_budget_limiter = _PROXY_MaxBudgetLimiter() self.cache_control_check = _PROXY_CacheControlCheck() - self.alerting: list | None = None + self.alerting: list[str] | None = None self.alerting_threshold: float = 300 # default to 5 min. threshold self.alert_types: list[AlertType] = DEFAULT_ALERT_TYPES self.alert_to_webhook_url: dict | None = None @@ -2364,7 +2364,9 @@ class ProxyLogging: # do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails) return - if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting): + if self.alerting is not None and ( + "slack" in self.alerting or "ms_teams" in self.alerting or "webhook" in self.alerting + ): if self.slack_alerting_instance is not None: await self.slack_alerting_instance.budget_alerts( type=type, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index f39df38d069..4a416f61e1b 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig: file_dict["file_id"] = file_id if item.get("file_data"): file_dict["file_data"] = item["file_data"] + if item.get("filename"): + file_dict["filename"] = item["filename"] new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict} if "cache_control" in item: diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py b/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py index 6e64f0f47a5..94f8161f44e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py @@ -12,6 +12,10 @@ class PromptSecurityGuardrailConfigModel(GuardrailConfigModel): default=None, description="The API base for the Prompt Security guardrail. If not provided, the `PROMPT_SECURITY_API_BASE` environment variable is used.", ) + file_sanitization_fail_open: bool = Field( + default=True, + description="Whether file sanitization timeouts allow the original file through instead of blocking the request.", + ) @staticmethod def ui_friendly_name() -> str: diff --git a/litellm/types/router.py b/litellm/types/router.py index 97bd93f3f47..ab6c807ba20 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -189,6 +189,11 @@ class ModelInfo(MirroredPricingParams): # router-wide default. enable_tag_filtering: bool | None = None + # when True, calls routed to this deployment persist a router_metadata block + # (requested model group, selected model + provider, router correlation id) + # in the spend log row's metadata. Set it on every deployment of the group. + internal_router_model: bool | None = None + def __init__(self, id: str | int | None = None, **params) -> None: if id is None: id = str(uuid.uuid4()) # Generate a UUID if id is None or not provided diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 6335e6f6bd8..c6d1b0cc46e 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -35,7 +35,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -56,7 +56,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ --extra extra_proxy \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -65,7 +65,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ --extra extra_proxy \ - --python python3 + --python python3.13 COPY migrations/run.py /app/run.py @@ -87,7 +87,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 nodejs libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 05c1cfd3179..7071eaa0807 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3726,7 +3726,7 @@ "output_cost_per_token": 0, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-services/", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure/eu/gpt-4o-2024-08-06": { @@ -5328,7 +5328,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/gpt-realtime-whisper", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -9107,7 +9107,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/Cohere-embed-v3-multilingual": { @@ -9118,7 +9118,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/FLUX-1.1-pro": { @@ -9134,7 +9134,7 @@ "litellm_provider": "azure_ai", "mode": "image_generation", "output_cost_per_image": 0.04, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/images/generations" ] @@ -9467,7 +9467,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9481,7 +9481,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.04e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9494,7 +9494,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -9543,7 +9543,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -9554,7 +9554,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { @@ -9566,7 +9566,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.1e-07, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -9756,7 +9756,7 @@ "supported_endpoints": [ "/v1/ocr" ], - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", @@ -9967,7 +9967,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/embeddings" ], @@ -10151,7 +10151,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://azure.microsoft.com/en-us/products/ai-services/ai-foundry/models/jais-30b-chat" + "source": "https://ai.azure.com/catalog/models/jais-30b-chat" }, "azure_ai/jamba-instruct": { "input_cost_per_token": 5e-07, @@ -10208,7 +10208,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10231,7 +10231,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10243,7 +10243,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10280,7 +10280,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", "supports_function_calling": true }, "azure_ai/mistral-small": { @@ -12315,7 +12315,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "claude-haiku-4-5-20251001": { "deprecation_date": "2026-10-15", @@ -17767,7 +17768,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "deprecation_date": "2027-01-08" }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -22843,7 +22845,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22897,7 +22899,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22962,7 +22964,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23021,7 +23023,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23226,7 +23228,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23310,7 +23312,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23373,7 +23375,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23430,7 +23432,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -25327,7 +25329,7 @@ "supports_vision": true }, "gpt-4o-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -25650,7 +25652,7 @@ "supports_vision": true }, "gpt-4o-mini-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -25688,7 +25690,7 @@ "gpt-4o-mini-realtime-preview": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -25787,7 +25789,8 @@ "output_cost_per_token": 5e-06, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-4o-mini-tts": { "input_cost_per_token": 2.5e-06, @@ -25809,7 +25812,7 @@ }, "gpt-4o-realtime-preview": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25828,7 +25831,7 @@ }, "gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25847,7 +25850,7 @@ }, "gpt-4o-realtime-preview-2025-06-03": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25926,7 +25929,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, @@ -27077,7 +27081,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/gpt-5.6-cyber", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-cyber", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27116,7 +27120,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27156,7 +27160,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -27168,7 +27172,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "source": "https://platform.openai.com/docs/models/chat-latest", + "source": "https://developers.openai.com/api/docs/models/chat-latest", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -30463,7 +30467,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30504,7 +30508,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30545,7 +30549,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30578,7 +30582,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30594,7 +30598,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30610,7 +30614,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -30627,7 +30631,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -32572,7 +32576,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 164000, @@ -32584,7 +32588,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { "max_tokens": 128000, @@ -32595,7 +32599,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { "max_tokens": 128000, @@ -32606,7 +32610,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 128000, @@ -32617,7 +32621,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { "max_tokens": 128000, @@ -32629,7 +32633,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -32640,7 +32644,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { "max_tokens": 128000, @@ -32650,7 +32654,7 @@ "output_cost_per_token": 6e-08, "litellm_provider": "nebius", "mode": "chat", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 128000, @@ -32661,7 +32665,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { "max_tokens": 128000, @@ -32672,7 +32676,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { "max_tokens": 128000, @@ -32683,7 +32687,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { "max_tokens": 128000, @@ -32694,7 +32698,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 128000, @@ -32705,7 +32709,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { "max_tokens": 128000, @@ -32716,7 +32720,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { "max_tokens": 131072, @@ -32727,7 +32731,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { "max_tokens": 262144, @@ -32738,7 +32742,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { "max_tokens": 32768, @@ -32749,7 +32753,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -32760,7 +32764,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { "max_tokens": 32768, @@ -32771,7 +32775,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { "max_tokens": 32768, @@ -32782,7 +32786,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { "max_tokens": 32768, @@ -32794,7 +32798,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { "max_tokens": 128000, @@ -32805,7 +32809,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { "max_tokens": 128000, @@ -32816,7 +32820,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { "max_tokens": 32768, @@ -32827,7 +32831,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { "max_tokens": 131072, @@ -32839,7 +32843,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -32851,7 +32855,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { "max_tokens": 131072, @@ -32862,7 +32866,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, @@ -32871,7 +32875,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { "max_tokens": 8192, @@ -32880,7 +32884,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { "max_tokens": 32768, @@ -32889,7 +32893,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, @@ -33630,7 +33634,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33643,7 +33647,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33656,7 +33660,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33713,7 +33717,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true, @@ -33727,7 +33731,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": false, "supports_response_schema": false, "supports_native_streaming": true @@ -33738,7 +33742,7 @@ "max_input_tokens": 512, "mode": "embedding", "output_vector_size": 1024, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -34635,7 +34639,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", + "source": "https://openrouter.ai/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { @@ -34870,7 +34874,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -34911,7 +34915,7 @@ "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -35996,7 +36000,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/deepseek-r1-distill-llama-70b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36010,7 +36014,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llama-3-1-8b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36023,7 +36027,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-1-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": false, "supports_tool_choice": false @@ -36036,7 +36040,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-3-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36049,7 +36053,7 @@ "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-7b-instruct-v0-3", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36062,7 +36066,7 @@ "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-nemo-instruct-2407", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36075,7 +36079,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-small-3-2-24b-instruct-2506", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -36089,7 +36093,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mixtral-8x7b-instruct-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36102,7 +36106,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-coder-32b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36115,7 +36119,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-vl-72b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36129,7 +36133,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen3-32b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36143,7 +36147,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-120b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36157,7 +36161,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-20b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36171,7 +36175,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llava-next-mistral-7b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36185,7 +36189,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mamba-codestral-7b-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -38321,7 +38325,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { - "deprecation_date": "2026-01-14", + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38946,7 +38950,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, - "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", + "source": "https://www.together.ai/models/qwen3-5-397b-a17b", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -43183,7 +43187,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43199,7 +43203,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43216,7 +43220,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43232,7 +43236,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43352,7 +43356,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43366,7 +43370,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43381,7 +43385,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43396,7 +43400,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -44100,7 +44104,8 @@ "output_cost_per_second": 0.0001, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "xai/grok-3": { "cache_read_input_token_cost": 2e-07, @@ -45088,7 +45093,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2", "supported_modalities": [ "text" ], @@ -45100,7 +45105,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -45112,7 +45117,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -49322,7 +49327,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-realtime-whisper", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -50574,7 +50579,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50612,7 +50617,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50650,7 +50655,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50688,7 +50693,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -51437,7 +51442,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { "max_tokens": 131072, @@ -51450,7 +51455,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { "max_tokens": 131072, @@ -51463,7 +51468,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { "max_tokens": 131072, @@ -51476,7 +51481,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { "max_tokens": 163840, @@ -51489,7 +51494,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { "max_tokens": 1000192, @@ -51502,7 +51507,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, @@ -51606,7 +51611,7 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-transcribe", "supported_endpoints": [ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" @@ -51624,7 +51629,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-live-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-live-transcribe", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -51645,7 +51650,7 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", - "source": "https://platform.openai.com/docs/models/gpt-realtime-translate", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-translate", "supported_modalities": [ "audio" ], @@ -51673,7 +51678,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -51712,7 +51717,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_assistant_prefill": false, diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index caa8eef252a..569c23cd03f 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -1,6 +1,6 @@ { "ANN001": { - "limit": 2993 + "limit": 2991 }, "ANN002": { "limit": 71 @@ -12,10 +12,10 @@ "limit": 2002 }, "ANN202": { - "limit": 843 + "limit": 841 }, "ANN204": { - "limit": 696 + "limit": 694 }, "ANN205": { "limit": 112 @@ -24,7 +24,7 @@ "limit": 133 }, "ANN401": { - "limit": 487 + "limit": 387 }, "ASYNC230": { "limit": 11 @@ -117,7 +117,7 @@ "limit": 1 }, "PERF102": { - "limit": 22 + "limit": 21 }, "PERF401": { "limit": 12 @@ -168,7 +168,7 @@ "limit": 3 }, "RET504": { - "limit": 174 + "limit": 173 }, "RUF012": { "limit": 239 @@ -198,7 +198,7 @@ "limit": 56 }, "SIM102": { - "limit": 312 + "limit": 310 }, "SIM103": { "limit": 119 @@ -231,7 +231,7 @@ "limit": 5 }, "TID251": { - "limit": 1096 + "limit": 1084 }, "TRY002": { "limit": 524 diff --git a/scripts/auto-close-duplicates.test.ts b/scripts/auto-close-duplicates.test.ts new file mode 100644 index 00000000000..b49bf05cbc2 --- /dev/null +++ b/scripts/auto-close-duplicates.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, test } from "bun:test"; + +import { + CLOSED_MARKER, + REOPEN_COMMENT, + candidateNumbers, + duplicateTarget, + normalizeTitle, + pendingNotice, + readConfig, + reopenTarget, + sweepClosedIssue, + sweepIssue, + type Comment, + type GitHubApi, + type Issue, + type Reaction, + type SweepConfig, +} from "./auto-close-duplicates"; + +const NOW = new Date("2026-09-04T09:00:00Z"); +const DAY_MS = 24 * 60 * 60 * 1000; +const daysAgo = (days: number): string => new Date(NOW.getTime() - days * DAY_MS).toISOString(); + +const issue = (number: number, title: string, overrides: Partial = {}): Issue => ({ + number, + title, + state: "open", + user: { login: "reporter" }, + ...overrides, +}); + +const notice = (candidates: readonly number[], createdAt: string, overrides: Partial = {}): Comment => ({ + id: 900, + body: `\n**Potential duplicate detected**`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + ...overrides, +}); + +const humanComment = (createdAt: string, body = "It is not the same thing", login = "reporter"): Comment => ({ + id: 901, + body, + created_at: createdAt, + user: { type: "User", login }, +}); + +const config: SweepConfig = { repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }; + +describe("normalizeTitle", () => { + test("drops the template prefix, case, and punctuation", () => { + expect(normalizeTitle("[Bug]: Gemma 4-e4b fails on Vertex!")).toBe("gemma 4 e4b fails on vertex"); + expect(normalizeTitle("[Feature]: ")).toBe(""); + }); +}); + +describe("candidateNumbers", () => { + test("reads only the marker field, keeps older issues, sorted ascending and deduplicated", () => { + const body = "\n- #1 - see #1 (100% similar)"; + expect(candidateNumbers(body, 35)).toEqual([10, 30]); + }); + + test("returns nothing without the marker", () => { + expect(candidateNumbers("- #1 - looks like #1", 35)).toEqual([]); + }); +}); + +describe("pendingNotice", () => { + test("waits out the grace period from the latest notice", () => { + const fresh = pendingNotice(issue(35, "t"), [notice([10], daysAgo(2.9))], config); + expect(fresh.kind).toBe("skip"); + const aged = pendingNotice(issue(35, "t"), [notice([10], daysAgo(3.1))], config); + expect(aged.kind).toBe("pending"); + const reposted = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(6)), notice([10], daysAgo(1), { id: 902 })], + config, + ); + expect(reposted.kind).toBe("skip"); + }); + + test("an objection posted before a re-posted notice still keeps the issue open", () => { + const verdict = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(10)), humanComment(daysAgo(7)), notice([10], daysAgo(4), { id: 902 })], + config, + ); + expect(verdict).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + }); + + test("a zero-day grace period acts on the notice at once", () => { + const verdict = pendingNotice(issue(35, "t"), [notice([10], daysAgo(0.01))], { ...config, graceDays: 0 }); + expect(verdict.kind).toBe("pending"); + }); + + test("a human reply after the notice keeps the issue open, a bot reply does not", () => { + const human = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5)), humanComment(daysAgo(4))], config); + expect(human).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + const bot = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(5)), { id: 903, body: "triage", created_at: daysAgo(4), user: { type: "Bot", login: "triage[bot]" } }], + config, + ); + expect(bot.kind).toBe("pending"); + }); + + test("a human quoting the marker is not a notice", () => { + const quoted = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5), { user: { type: "User", login: "reporter" } })], config); + expect(quoted).toEqual({ kind: "skip", reason: "carries no duplicate notice" }); + }); + + test("never closes an issue twice: a reopened issue is left alone", () => { + const reopened = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(9)), { id: 904, body: `Closed automatically\n\n${CLOSED_MARKER}`, created_at: daysAgo(5), user: { type: "Bot", login: "github-actions[bot]" } }], + config, + ); + expect(reopened).toEqual({ kind: "skip", reason: "was reopened after an automatic close" }); + }); + + test("skips pull requests and issues whose only candidates are newer", () => { + expect(pendingNotice(issue(35, "t", { pull_request: {} }), [notice([10], daysAgo(5))], config).kind).toBe("skip"); + expect(pendingNotice(issue(35, "t"), [notice([40], daysAgo(5))], config)).toEqual({ + kind: "skip", + reason: "no candidate is older than this issue", + }); + }); +}); + +describe("duplicateTarget", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + + test("closes only against the earliest open issue with the identical normalized title", () => { + const verdict = duplicateTarget( + reporter, + [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex"), issue(20, "[bug]: gemma 4-e4b fails on vertex"), issue(30, "[Bug]: Gemma 4-e4b fails on Vertex")], + [], + ); + expect(verdict).toEqual({ kind: "close", duplicateOf: 20 }); + }); + + test("a near miss in the title is not a duplicate", () => { + const verdict = duplicateTarget(reporter, [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex")], []); + expect(verdict).toEqual({ kind: "skip", reason: "no older open issue has the identical title" }); + }); + + test("bare template titles never match each other", () => { + const verdict = duplicateTarget(issue(35, "[Bug]: "), [issue(10, "[Bug]: ")], []); + expect(verdict.kind).toBe("skip"); + expect(verdict.kind === "skip" && verdict.reason).toContain("too short"); + }); + + test("a closed candidate or a pull request is never the target", () => { + expect(duplicateTarget(reporter, [issue(10, reporter.title, { state: "closed" })], []).kind).toBe("skip"); + expect(duplicateTarget(reporter, [issue(10, reporter.title, { pull_request: {} })], []).kind).toBe("skip"); + }); + + test("a thumbs down on the notice keeps the issue open", () => { + const verdict = duplicateTarget(reporter, [issue(10, reporter.title)], [{ content: "+1" }, { content: "-1" }]); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + }); +}); + +describe("sweepIssue", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + const original = issue(10, "[Bug]: Gemma 4-e4b fails on Vertex"); + + function fakeApi( + comments: readonly Comment[] = [notice([10], daysAgo(5))], + reactionsByNotice: Readonly> = {}, + ): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + const reactionsPath = path.match(/^\/repos\/BerriAI\/litellm\/issues\/comments\/(\d+)\/reactions/); + if (reactionsPath) { + return (reactionsByNotice[Number(reactionsPath[1])] ?? []) as T; + } + if (path === "/repos/BerriAI/litellm/issues/10") { + return original as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + test("a dry run reports the close and writes nothing", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, { ...config, dryRun: true }, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes).toEqual([]); + }); + + test("a thumbs down on an earlier notice still keeps the issue open", async () => { + const { api, writes } = fakeApi([notice([10], daysAgo(9)), notice([10], daysAgo(5), { id: 902 })], { 900: [{ content: "-1" }] }); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + expect(writes).toEqual([]); + }); + + test("a real run comments, labels, then closes with the duplicate reason", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([ + "POST /repos/BerriAI/litellm/issues/35/comments", + "POST /repos/BerriAI/litellm/issues/35/labels", + "PATCH /repos/BerriAI/litellm/issues/35", + ]); + expect(writes[0]).toContain("duplicate of #10"); + expect(writes[0]).toContain("unanswered for 3 days"); + expect(writes[0]).toContain(CLOSED_MARKER); + expect(writes[1]).toContain('{"labels":["duplicate"]}'); + expect(writes[2]).toContain('{"state":"closed","state_reason":"duplicate"}'); + }); +}); + +describe("reopenTarget", () => { + const closedByBot = (overrides: Partial = {}): Issue => + issue(35, "t", { state: "closed", closed_by: { type: "Bot" }, ...overrides }); + const closeMarker = (createdAt: string): Comment => ({ + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + }); + + test("a reporter reply after the automatic close reopens", () => { + const verdict = reopenTarget(closedByBot(), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "reopen" }); + }); + + test("an issue closed by a person stays closed", () => { + const verdict = reopenTarget(closedByBot({ closed_by: { type: "User" } }), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1)), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "was closed by a person" }); + }); + + test("without the automatic-close marker nothing reopens", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "carries no automatic-close marker" }); + }); + + test("a maintainer reply alone does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1), "Confirmed duplicate", "maintainer"), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a reporter comment from before the close does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(3)), closeMarker(daysAgo(2))]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a pull request never reopens", () => { + const verdict = reopenTarget(closedByBot({ pull_request: {} }), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "is a pull request" }); + }); +}); + +describe("sweepClosedIssue", () => { + function fakeApi(issueBody: Issue, comments: readonly Comment[]): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + if (path === "/repos/BerriAI/litellm/issues/35") { + return issueBody as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + const closedByBot = issue(35, "t", { state: "closed", closed_by: { type: "Bot" } }); + const closeMarker: Comment = { + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: daysAgo(2), + user: { type: "Bot", login: "github-actions[bot]" }, + }; + + test("a real run unlabels, reopens, then explains", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, config, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([ + "DELETE /repos/BerriAI/litellm/issues/35/labels/duplicate undefined", + 'PATCH /repos/BerriAI/litellm/issues/35 {"state":"open"}', + `POST /repos/BerriAI/litellm/issues/35/comments {"body":"${REOPEN_COMMENT}"}`, + ]); + }); + + test("a dry run reports the reopen and writes nothing", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, { ...config, dryRun: true }, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([]); + }); +}); + +describe("readConfig", () => { + test("defaults to a real run with a 3-day grace period", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm" }, NOW); + expect(parsed).toEqual({ token: "t", repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }); + }); + + test("honors DRY_RUN and GRACE_PERIOD_DAYS overrides", () => { + const parsed = readConfig( + { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", DRY_RUN: "true", GRACE_PERIOD_DAYS: "0" }, + NOW, + ); + expect(parsed.dryRun).toBe(true); + expect(parsed.graceDays).toBe(0); + }); + + test("an empty GRACE_PERIOD_DAYS, as a schedule run renders it, means the default", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "" }, NOW); + expect(parsed.graceDays).toBe(3); + }); + + test("refuses a missing token, a malformed repository, or a bad grace period", () => { + expect(() => readConfig({ GITHUB_REPOSITORY: "o/r" }, NOW)).toThrow("GITHUB_TOKEN"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "litellm" }, NOW)).toThrow("owner/repo"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "-1" }, NOW)).toThrow( + "GRACE_PERIOD_DAYS", + ); + }); +}); diff --git a/scripts/auto-close-duplicates.ts b/scripts/auto-close-duplicates.ts new file mode 100644 index 00000000000..c595104d886 --- /dev/null +++ b/scripts/auto-close-duplicates.ts @@ -0,0 +1,300 @@ +#!/usr/bin/env bun + +declare const process: { readonly env: Readonly> }; + +export interface Issue { + readonly number: number; + readonly title: string; + readonly state: string; + readonly user: { readonly login: string }; + readonly closed_by?: { readonly type: string } | null; + readonly pull_request?: unknown; +} + +export interface Comment { + readonly id: number; + readonly body: string; + readonly created_at: string; + readonly user: { readonly type: string; readonly login: string }; +} + +export interface Reaction { + readonly content: string; +} + +export interface GitHubApi { + readonly request: (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object) => Promise; +} + +export interface SweepConfig { + readonly repo: string; + readonly graceDays: number; + readonly dryRun: boolean; + readonly now: Date; +} + +export type NoticeVerdict = + | { readonly kind: "pending"; readonly notices: readonly Comment[]; readonly candidates: readonly number[] } + | { readonly kind: "skip"; readonly reason: string }; + +export type CloseVerdict = + | { readonly kind: "close"; readonly duplicateOf: number } + | { readonly kind: "skip"; readonly reason: string }; + +export type ReopenVerdict = + | { readonly kind: "reopen" } + | { readonly kind: "skip"; readonly reason: string }; + +export const FLAG_LABEL = "potential-duplicate"; +export const CLOSED_MARKER = ""; +export const DEFAULT_GRACE_DAYS = 3; +export const REOPEN_COMMENT = + "Reopened automatically: the reporter replied after the duplicate close, so this needs a human look."; +const NOTICE_MARKER = //; +const MIN_TITLE_WORDS = 3; +const PAGE_SIZE = 100; +const DAY_MS = 24 * 60 * 60 * 1000; +const REOPEN_LOOKBACK_DAYS = 30; + +const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason }); + +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/^\s*\[[^\]]*\]\s*:?/, "") + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +export function candidateNumbers(noticeBody: string, issueNumber: number): readonly number[] { + const field = noticeBody.match(NOTICE_MARKER); + if (!field) { + return []; + } + const older = field[1] + .split(",") + .filter((value) => value !== "") + .map(Number) + .filter((candidate) => candidate < issueNumber); + return [...new Set(older)].sort((a, b) => a - b); +} + +export function pendingNotice( + issue: Issue, + comments: readonly Comment[], + config: Pick, +): NoticeVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (comments.some((comment) => comment.body.includes(CLOSED_MARKER))) { + return skip("was reopened after an automatic close"); + } + const notices = comments.filter((comment) => comment.user.type === "Bot" && NOTICE_MARKER.test(comment.body)); + const first = notices[0]; + const latest = notices[notices.length - 1]; + if (first === undefined || latest === undefined) { + return skip("carries no duplicate notice"); + } + const ageDays = (config.now.getTime() - new Date(latest.created_at).getTime()) / DAY_MS; + if (ageDays < config.graceDays) { + return skip(`notice is ${ageDays.toFixed(1)} days old, grace period is ${config.graceDays}`); + } + const firstNoticeAt = new Date(first.created_at); + if (comments.some((comment) => comment.user.type !== "Bot" && new Date(comment.created_at) > firstNoticeAt)) { + return skip("someone replied after the notice"); + } + const candidates = candidateNumbers(latest.body, issue.number); + if (candidates.length === 0) { + return skip("no candidate is older than this issue"); + } + return { kind: "pending", notices, candidates }; +} + +export function duplicateTarget( + issue: Issue, + candidates: readonly Issue[], + reactions: readonly Reaction[], +): CloseVerdict { + if (reactions.some((reaction) => reaction.content === "-1")) { + return skip("someone gave the notice a thumbs down"); + } + const title = normalizeTitle(issue.title); + if (title.split(" ").length < MIN_TITLE_WORDS) { + return skip(`title "${issue.title}" is too short to match on`); + } + const original = candidates.find( + (candidate) => + candidate.state === "open" && candidate.pull_request === undefined && normalizeTitle(candidate.title) === title, + ); + if (original === undefined) { + return skip("no older open issue has the identical title"); + } + return { kind: "close", duplicateOf: original.number }; +} + +export function reopenTarget(issue: Issue, comments: readonly Comment[]): ReopenVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (issue.closed_by?.type !== "Bot") { + return skip("was closed by a person"); + } + const marker = comments.find((comment) => comment.body.includes(CLOSED_MARKER)); + if (marker === undefined) { + return skip("carries no automatic-close marker"); + } + const markerAt = new Date(marker.created_at); + if (!comments.some((comment) => comment.user.login === issue.user.login && new Date(comment.created_at) > markerAt)) { + return skip("the reporter has not replied since the close"); + } + return { kind: "reopen" }; +} + +export function closingComment(duplicateOf: number, graceDays: number): string { + return `Closed automatically as a duplicate of #${duplicateOf}. Its title is identical to that older open issue and the duplicate notice above went unanswered for ${graceDays} days. If this is wrong, comment here with how it differs from #${duplicateOf} and this issue will be reopened automatically within a day. + +${CLOSED_MARKER}`; +} + +async function listAll(api: GitHubApi, path: string, page = 1): Promise { + const separator = path.includes("?") ? "&" : "?"; + const batch = await api.request("GET", `${path}${separator}per_page=${PAGE_SIZE}&page=${page}`); + return batch.length < PAGE_SIZE ? batch : [...batch, ...(await listAll(api, path, page + 1))]; +} + +async function closeAsDuplicate( + api: GitHubApi, + config: SweepConfig, + issueNumber: number, + duplicateOf: number, +): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("POST", `${issuePath}/comments`, { body: closingComment(duplicateOf, config.graceDays) }); + await api.request("POST", `${issuePath}/labels`, { labels: ["duplicate"] }); + await api.request("PATCH", issuePath, { state: "closed", state_reason: "duplicate" }); +} + +async function reopenForReporter(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("DELETE", `${issuePath}/labels/duplicate`); + await api.request("PATCH", issuePath, { state: "open" }); + await api.request("POST", `${issuePath}/comments`, { body: REOPEN_COMMENT }); +} + +export async function sweepClosedIssue(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issue = await api.request("GET", `/repos/${config.repo}/issues/${issueNumber}`); + const comments = await listAll(api, `/repos/${config.repo}/issues/${issueNumber}/comments`); + const verdict = reopenTarget(issue, comments); + if (verdict.kind === "reopen" && !config.dryRun) { + await reopenForReporter(api, config, issueNumber); + } + return verdict; +} + +export async function sweepIssue(api: GitHubApi, config: SweepConfig, issue: Issue): Promise { + const comments = await listAll(api, `/repos/${config.repo}/issues/${issue.number}/comments`); + const pending = pendingNotice(issue, comments, config); + if (pending.kind === "skip") { + return pending; + } + const reactions = ( + await Promise.all( + pending.notices.map((notice) => listAll(api, `/repos/${config.repo}/issues/comments/${notice.id}/reactions`)), + ) + ).flat(); + const candidates = await Promise.all( + pending.candidates.map((candidate) => api.request("GET", `/repos/${config.repo}/issues/${candidate}`)), + ); + const verdict = duplicateTarget(issue, candidates, reactions); + if (verdict.kind === "close" && !config.dryRun) { + await closeAsDuplicate(api, config, issue.number, verdict.duplicateOf); + } + return verdict; +} + +function describe(issue: Issue, verdict: CloseVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issue.number}: skipped, ${verdict.reason}`; + } + return `#${issue.number}: ${dryRun ? "would close" : "closed"} as a duplicate of #${verdict.duplicateOf}`; +} + +export async function sweep(api: GitHubApi, config: SweepConfig): Promise { + const issues = await listAll(api, `/repos/${config.repo}/issues?state=open&labels=${FLAG_LABEL}`); + console.log(`${issues.length} open issues carry the ${FLAG_LABEL} label in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepIssue(api, config, issue); + console.log(describe(issue, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +function describeReopen(issueNumber: number, verdict: ReopenVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issueNumber}: skipped, ${verdict.reason}`; + } + return `#${issueNumber}: ${dryRun ? "would reopen" : "reopened"} for the reporter's reply`; +} + +export async function reopenSweep(api: GitHubApi, config: SweepConfig): Promise { + const since = new Date(config.now.getTime() - REOPEN_LOOKBACK_DAYS * DAY_MS).toISOString(); + const closedPath = `/repos/${config.repo}/issues?state=closed&labels=duplicate,${FLAG_LABEL}&since=${encodeURIComponent(since)}`; + const issues = await listAll(api, closedPath); + console.log(`${issues.length} recently closed issues carry the duplicate and ${FLAG_LABEL} labels in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepClosedIssue(api, config, issue.number); + console.log(describeReopen(issue.number, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +export function readConfig(env: Readonly>, now: Date): SweepConfig & { readonly token: string } { + const token = env.GITHUB_TOKEN; + const repo = env.GITHUB_REPOSITORY; + if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) { + throw new Error("GITHUB_TOKEN and GITHUB_REPOSITORY (owner/repo) are required"); + } + const rawGraceDays = env.GRACE_PERIOD_DAYS?.trim(); + const graceDays = rawGraceDays === undefined || rawGraceDays === "" ? DEFAULT_GRACE_DAYS : Number(rawGraceDays); + if (!Number.isFinite(graceDays) || graceDays < 0) { + throw new Error(`GRACE_PERIOD_DAYS must be a non-negative number, got "${env.GRACE_PERIOD_DAYS}"`); + } + return { token, repo, graceDays, dryRun: env.DRY_RUN === "true", now }; +} + +export function githubApi(token: string): GitHubApi { + return { + request: async (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object): Promise => { + const response = await fetch(`https://api.github.com${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "litellm-auto-close-duplicates", + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`${method} ${path} failed: ${response.status} ${response.statusText}`); + } + return (await response.json()) as T; + }, + }; +} + +if (import.meta.main) { + const { token, ...config } = readConfig(process.env, new Date()); + const api = githubApi(token); + const closeVerdicts = await sweep(api, config); + const reopenVerdicts = await reopenSweep(api, config); + const closed = closeVerdicts.filter((verdict) => verdict.kind === "close").length; + const reopened = reopenVerdicts.filter((verdict) => verdict.kind === "reopen").length; + console.log( + `${config.dryRun ? "Would close" : "Closed"} ${closed} of ${closeVerdicts.length} flagged issues, ${config.dryRun ? "would reopen" : "reopened"} ${reopened}`, + ); +} diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 3bde72ccd49..35de9961054 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1317,6 +1317,62 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch): assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test" +@pytest.mark.asyncio +async def test_default_team_settings_newrelic_resolves_traces_and_metrics(): + """Static `default_team_settings` is the config-file twin of POST /team/callback. + + A team pinned to New Relic through `default_team_settings` must reach the + same two loggers the dynamic path does: the per-team metrics logger (cost + and usage) and the trace logger (LLM/agent spans). This proves the static + path resolves both, not just one, so the config-file customer gets the + same per-team routing as the API customer. + """ + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc.config = { + "litellm_settings": { + "default_team_settings": [ + { + "team_id": "team-a", + "success_callback": ["newrelic"], + "newrelic_api_key": "team-a-ingest-key", + "newrelic_region": "eu", + } + ] + } + } + + callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config( + team_id="team-a", + proxy_config=pc, + ) + + assert callback_metadata is not None + assert callback_metadata.success_callback == ["newrelic"] + assert callback_metadata.callback_vars == { + "newrelic_api_key": "team-a-ingest-key", + "newrelic_region": "eu", + } + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="static-nr-1", + function_id="static-nr-1", + ) + logging_obj._trusted_callback_vars = tuple(callback_metadata.callback_vars.items()) + + resolved = logging_obj._resolve_dynamic_callback_string("newrelic") + resolved_names = {type(logger).__name__ for logger in resolved} + assert resolved_names == {"NewRelicMetricsLogger", "NewRelicLogger"} + + def test_proxy_config_state_get_config_state_error(): """ Ensures that get_config_state does not raise an error when the config is not a valid dictionary diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index cfbd3e76a88..55e2dcdc270 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -12,7 +12,7 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import CallInfo, Litellm_EntityType -from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys +from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys class TestSlackAlerting(unittest.TestCase): @@ -366,3 +366,56 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through(): _, kwargs = slack_alerting._run_scheduler_helper.await_args assert kwargs["pod_lock_manager"] is pod_lock_manager + + +def _slack_alerting_with_env_resolution() -> SlackAlerting: + slack_alerting: Final = SlackAlerting(alerting=["slack"], internal_usage_cache=DualCache()) + slack_alerting.periodic_started = True + return slack_alerting + + +@pytest.mark.asyncio +async def test_send_alert_falls_back_to_alerting_webhook_url_env(monkeypatch): + monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False) + monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc") + slack_alerting: Final = _slack_alerting_with_env_resolution() + + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) + + assert slack_alerting.log_queue[0]["url"] == "https://chat.example.com/hooks/abc" + + +@pytest.mark.asyncio +async def test_send_alert_prefers_slack_webhook_url_over_fallback(monkeypatch): + monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T0/B0/X0") + monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc") + slack_alerting: Final = _slack_alerting_with_env_resolution() + + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) + + assert slack_alerting.log_queue[0]["url"] == "https://hooks.slack.com/services/T0/B0/X0" + + +@pytest.mark.asyncio +async def test_send_alert_raises_when_no_webhook_url_configured(monkeypatch): + monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False) + monkeypatch.delenv("ALERTING_WEBHOOK_URL", raising=False) + slack_alerting: Final = _slack_alerting_with_env_resolution() + + with pytest.raises(ValueError, match="SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL"): + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py index edce5c5f3a2..d614823c0ef 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py @@ -79,6 +79,23 @@ class TestDigestMode(unittest.IsolatedAsyncioTestCase): self.assertEqual(len(self.slack_alerting.digest_buckets), 2) + async def test_digest_falls_back_to_alerting_webhook_url_env(self): + """With SLACK_WEBHOOK_URL unset, the digest entry resolves ALERTING_WEBHOOK_URL instead.""" + env = {k: v for k, v in os.environ.items() if k != "SLACK_WEBHOOK_URL"} + env["ALERTING_WEBHOOK_URL"] = "https://chat.example.com/hooks/abc" + with unittest.mock.patch.dict(os.environ, env, clear=True): + await self.slack_alerting.send_alert( + message="`Requests are hanging`", + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + + bucket = list(self.slack_alerting.digest_buckets.values())[0] + self.assertEqual(bucket["webhook_url"], "https://chat.example.com/hooks/abc") + async def test_non_digest_alert_goes_to_queue(self): """Alert types without digest enabled should go straight to the log queue.""" message = "Budget exceeded" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 115e385eda4..4aa28b5abfd 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" import json +from dataclasses import replace import pytest @@ -215,6 +216,27 @@ def test_genai_mapper_all_request_params(): assert attrs["server.port"] == 443 +def test_genai_mapper_cache_token_attrs(): + cached = replace( + _full_llm_call(), + usage=LLMUsage( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cache_creation_input_tokens=7, + cache_read_input_tokens=3, + ), + ) + attrs = GenAIMapper().map(cached) + assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7 + assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3 + + # No cache usage keeps the span sparse: neither key present. + uncached = GenAIMapper().map(_full_llm_call()) + assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached + assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached + + def test_genai_mapper_stamps_input_output_messages(): data = LLMCallSpanData( operation=GenAIOperation.CHAT, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index baa72b5a7fe..ca628aa3405 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -525,6 +525,28 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): + payload = _sample_payload() + payload["metadata"] = { + **payload["metadata"], + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 3, + }, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.usage.cache_creation_input_tokens == 7 + assert data.usage.cache_read_input_tokens == 3 + + +def test_llm_call_adapter_cache_tokens_none_without_usage_object(): + data = LLMCallSpanData.from_standard_logging_payload(_sample_payload()) + assert data.usage.cache_creation_input_tokens is None + assert data.usage.cache_read_input_tokens is None + + def test_llm_call_adapter_failure_path(): payload = _sample_payload( status="failure", diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 572b505e94c..4694fa8fbed 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens(): {"type": "document", "source": source}, ] ) + + +def test_openai_file_block_prices_like_the_equivalent_anthropic_document(): + """An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise. + + Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject` + is in the union this counter accepts, so every local count of a Responses `input_file` raised + `Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens. + """ + prompt = {"type": "text", "text": "Summarize this file."} + inline_file = { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"}, + } + document = { + "type": "document", + "title": "report.pdf", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + } + + assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document]) + assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt]) + + +def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): + """A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens.""" + prompt = {"type": "text", "text": "Summarize this file."} + + by_id = {"type": "file", "file": {"file_id": "file-abc123"}} + assert _count_user_content([prompt, by_id]) == _count_user_content([prompt]) + + named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}} + assert _count_user_content([prompt, named]) == _count_user_content( + [prompt, {"type": "text", "text": "report.pdf"}] + ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 226bba6826a..63f895e1819 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -957,6 +957,28 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert fields["tools"][0]["type"] == "computer_20250124" +def test_config_blocks_do_not_leak_into_inference_config(): + """Regression: inferenceConfig was built before the config blocks were popped, so a dead + nested copy of each block (guardrailConfig, performanceConfig, serviceTier) rode inside + inferenceConfig alongside the real top-level one.""" + data = AmazonConverseConfig()._transform_request_helper( + model="anthropic.claude-haiku-4-5-20251001-v1:0", + system_content_blocks=[], + optional_params={ + "maxTokens": 100, + "guardrailConfig": {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"}, + "performanceConfig": {"latency": "optimized"}, + "serviceTier": {"type": "priority"}, + }, + messages=[{"role": "user", "content": "hi"}], + ) + + assert data["inferenceConfig"] == {"maxTokens": 100} + assert data["guardrailConfig"] == {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"} + assert data["performanceConfig"] == {"latency": "optimized"} + assert data["serviceTier"] == {"type": "priority"} + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost @@ -2853,17 +2875,11 @@ def test_guarded_text_guardrail_config_preserved(): headers={}, ) - # GuardrailConfig should be present at top level assert "guardrailConfig" in result assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" - # GuardrailConfig should also be in inferenceConfig assert "inferenceConfig" in result - assert "guardrailConfig" in result["inferenceConfig"] - assert ( - result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] - == "gr-abc123" - ) + assert "guardrailConfig" not in result["inferenceConfig"] def test_auto_convert_last_user_message_to_guarded_text(): diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py index e1cc6a92927..c2efc1acdb9 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool(): } +def test_messages_to_responses_input_preserves_images(): + """An image block must survive the round trip, or OpenAI counts only the text. + + A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it + turned a 268-token request into a 13-token one. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"}, + }, + ], + } + ] + + input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert instructions is None + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + ), + } + ] + + +def test_messages_to_responses_input_image_without_detail_defaults_to_auto(): + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_bare_string_image_url_is_preserved(): + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string(): + """Text-only content must keep collapsing to a string so existing counts do not shift.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "first\nsecond"}] + + +def test_messages_to_responses_input_drops_unmappable_blocks(): + """A block with no Responses API equivalent is skipped, never forwarded verbatim.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_text", "text": "hi"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string(): + """An assistant turn must never forward chat `text` blocks. + + The Responses API only accepts output_text and refusal inside an assistant turn, so + forwarding them 400s the whole request and silently drops the count back to the local + tokenizer, which is exactly what defeats the image fix above. + """ + messages = [ + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Paris."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "Paris."}, + ] + + +def test_messages_to_responses_input_assistant_image_block_is_dropped(): + """An image part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + +def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ), + }, + {"role": "assistant", "content": "A cat."}, + ] + + +def test_messages_to_responses_input_preserves_inline_files(): + """An inline file must survive the round trip, or the count silently drops the file. + + A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same + request counting 13, the text-only total. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "Summarize this file."}, + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + }, + ), + } + ] + + +def test_messages_to_responses_input_drops_a_file_with_no_inline_data(): + """OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "Summarize this file."}] + + +def test_messages_to_responses_input_assistant_file_block_is_dropped(): + """A file part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + def test_validate_request_valid(): """Test that valid requests pass validation.""" config = OpenAICountTokensConfig() diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py new file mode 100644 index 00000000000..d8d9936e9a1 --- /dev/null +++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py @@ -0,0 +1,238 @@ +import json +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig +from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError +from litellm.llms.openai.openai import OpenAIChatCompletion +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openai.workload_identity import ( + OpenAIWorkloadIdentityConfig, + _workload_identity_auth, + get_workload_identity_bearer_token, + resolve_openai_workload_identity_config, +) +from litellm.types.router import GenericLiteLLMParams + +TOKEN_EXCHANGE_URL: Final = "https://auth.openai.com/oauth/token" + + +@pytest.fixture +def wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> OpenAIWorkloadIdentityConfig: + token_file: Final = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token-from-file") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123") + monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456") + monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file)) + _workload_identity_auth.cache_clear() + litellm.in_memory_llm_clients_cache.flush_cache() + return OpenAIWorkloadIdentityConfig( + identity_provider_id="idp_test123", + service_account_id="user-test456", + token_file=str(token_file), + ) + + +def mock_token_exchange(access_token: str = "exchanged-bearer-token") -> respx.Route: + return respx.post(TOKEN_EXCHANGE_URL).mock( + return_value=httpx.Response(200, json={"access_token": access_token, "expires_in": 3600}) + ) + + +class TestResolveConfig: + def test_resolves_from_env(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_static_api_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key="sk-static", api_base=None) is None + + def test_env_openai_api_key_wins( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + @pytest.mark.parametrize("empty_key", ["", " "]) + def test_empty_api_key_arg_does_not_disable_wif( + self, wif_env: OpenAIWorkloadIdentityConfig, empty_key: str + ) -> None: + assert resolve_openai_workload_identity_config(api_key=empty_key, api_base=None) == wif_env + + @pytest.mark.parametrize("empty_key", ["", " "]) + def test_empty_env_openai_api_key_does_not_disable_wif( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, empty_key: str + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", empty_key) + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None + + def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env + + def test_plaintext_http_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="http://api.openai.com/v1") is None + + def test_foreign_env_base_url_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://my-vllm.internal/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + def test_openai_env_base_url_allows( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_foreign_litellm_api_base_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "api_base", "https://my-vllm.internal/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + @pytest.mark.parametrize( + "missing_var", + ["OPENAI_IDENTITY_PROVIDER_ID", "OPENAI_SERVICE_ACCOUNT_ID", "OPENAI_IDENTITY_TOKEN_FILE"], + ) + def test_partial_env_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, missing_var: str + ) -> None: + monkeypatch.delenv(missing_var) + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + +class TestTokenExchange: + @respx.mock + def test_exchanges_subject_token_for_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + route: Final = mock_token_exchange() + assert get_workload_identity_bearer_token(wif_env) == "exchanged-bearer-token" + request_body: Final = json.loads(route.calls.last.request.content) + assert request_body["grant_type"] == "urn:ietf:params:oauth:grant-type:token-exchange" + assert request_body["subject_token"] == "subject-token-from-file" + assert request_body["subject_token_type"] == "urn:ietf:params:oauth:token-type:jwt" + assert request_body["identity_provider_id"] == "idp_test123" + assert request_body["service_account_id"] == "user-test456" + + @respx.mock + def test_token_cached_across_mints(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + route: Final = mock_token_exchange() + first: Final = get_workload_identity_bearer_token(wif_env) + second: Final = get_workload_identity_bearer_token(wif_env) + assert first == second == "exchanged-bearer-token" + assert route.call_count == 1 + + def test_old_sdk_raises_upgrade_error( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + import openai as openai_module + + monkeypatch.delattr(openai_module, "auth", raising=False) + monkeypatch.setitem(sys.modules, "openai.auth", None) + with pytest.raises(OpenAIError, match=r"openai>=2\.32\.0"): + wif_env.to_sdk_workload_identity() + + +class TestClientConstruction: + def test_sync_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None) + assert isinstance(client, OpenAI) + assert client.api_key == "workload-identity-auth" + assert client._workload_identity_auth is not None + + def test_async_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=True, api_key=None, api_base=None) + assert isinstance(client, AsyncOpenAI) + assert client.api_key == "workload-identity-auth" + assert client._workload_identity_auth is not None + + def test_static_key_client_unaffected(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key="sk-static", api_base=None) + assert isinstance(client, OpenAI) + assert client.api_key == "sk-static" + assert client._workload_identity_auth is None + + def test_cache_key_separates_wif_identities(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + other_config: Final = OpenAIWorkloadIdentityConfig( + identity_provider_id="idp_other", + service_account_id="user-other", + token_file=wif_env.token_file, + ) + keys: Final = tuple( + BaseOpenAILLM.get_openai_client_cache_key( + client_initialization_params={"api_key": None, "is_async": False, "workload_identity_config": config}, + client_type="openai", + ) + for config in (wif_env, other_config, None) + ) + assert len(set(keys)) == 3 + + @respx.mock + def test_request_carries_exchanged_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + mock_token_exchange() + completion_route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-wif", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + client = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None) + assert isinstance(client, OpenAI) + client.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}]) + auth_header: Final = completion_route.calls.last.request.headers["Authorization"] + assert auth_header == "Bearer exchanged-bearer-token" + + +class TestResponsesValidateEnvironment: + @respx.mock + def test_mints_bearer_when_wif_configured(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + mock_token_exchange() + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer exchanged-bearer-token" + + def test_static_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams(api_key="sk-responses") + ) + assert headers["Authorization"] == "Bearer sk-responses" + + def test_foreign_api_base_skips_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, + model="gpt-4o-mini", + litellm_params=GenericLiteLLMParams(api_base="https://my-vllm.internal/v1"), + ) + assert headers["Authorization"] == "Bearer None" + + def test_litellm_proxy_subclass_never_mints_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = LiteLLMProxyResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer None" diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py index 29d22e844a5..a4d67606698 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py @@ -982,6 +982,116 @@ class TestVertexBase: assert result_url == f"{gateway_api_base}:embedContent" + def test_check_custom_proxy_vertex_api_base_with_version_path_grafts_default_path(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://aiplatform.googleapis.com/v1beta1", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://aiplatform.googleapis.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_trailing_slash_grafts_default_path(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1/", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_grafts_before_query(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1beta1?key=abc", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent?key=abc" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_streaming_appends_alt_sse(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1beta1?key=abc", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="streamGenerateContent", + stream=True, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent?key=abc&alt=sse" + ) + + def test_check_custom_proxy_vertex_api_base_with_non_version_path_keeps_endpoint_append(self): + vertex_base = VertexBase() + gateway_api_base = "https://gateway.example.com/vertex-proxy" + + _, result_url = vertex_base._check_custom_proxy( + api_base=gateway_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert result_url == f"{gateway_api_base}:generateContent" + + def test_check_custom_proxy_vertex_api_base_without_projects_in_default_url_keeps_endpoint_append(self): + vertex_base = VertexBase() + gemma_api_base = "https://example.com/custom/gemma-deployment" + + _, result_url = vertex_base._check_custom_proxy( + api_base=gemma_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header=None, + url=gemma_api_base, + model="gemma-3-27b-it", + ) + + assert result_url == f"{gemma_api_base}:predict" + def test_check_custom_proxy_vertex_bare_host_streaming_keeps_single_alt_sse(self): vertex_base = VertexBase() diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 74bf1c95777..4a3b3ef22c4 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -157,6 +157,7 @@ class TestUpCommand: assert captured["settings"]["theme"] == "dark" assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:5483" assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key" + assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true" assert "apiKeyHelper" not in captured["settings"] assert captured["settings_mode"] == 0o600 diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py index 40d3e7f2aee..87a33c79a79 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py @@ -19,6 +19,13 @@ def test_sets_base_url_and_auth_token(): merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000/", "token-abc") assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" + + +def test_preserves_existing_tool_search(): + settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} + merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" def test_drops_stray_api_key(): diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index 32dfb8d521d..0191dad3d94 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -77,9 +77,19 @@ class TestBuildAgentEnv: ) assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000" assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key" + assert env["ENABLE_TOOL_SEARCH"] == "true" assert "OPENAI_BASE_URL" not in env assert "OPENAI_API_KEY" not in env + def test_anthropic_profile_preserves_existing_tool_search(self): + env = build_agent_env( + {"ENABLE_TOOL_SEARCH": "false"}, + "http://localhost:4000", + "sk-key", + frozenset({"anthropic"}), + ) + assert env["ENABLE_TOOL_SEARCH"] == "false" + def test_anthropic_profile_drops_existing_api_key(self): env = build_agent_env( {"ANTHROPIC_API_KEY": "real-key"}, @@ -96,6 +106,7 @@ class TestBuildAgentEnv: assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1" assert env["OPENAI_API_KEY"] == "sk-key" assert "ANTHROPIC_BASE_URL" not in env + assert "ENABLE_TOOL_SEARCH" not in env def test_both_profiles_set_everything(self): env = build_agent_env( @@ -105,6 +116,7 @@ class TestBuildAgentEnv: assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1" assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key" assert env["OPENAI_API_KEY"] == "sk-key" + assert env["ENABLE_TOOL_SEARCH"] == "true" def test_preserves_unrelated_env_and_does_not_mutate_input(self): base = {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"} @@ -201,6 +213,7 @@ class TestRunAgent: env = calls["env"] assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000" assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key" + assert env["ENABLE_TOOL_SEARCH"] == "true" assert "ANTHROPIC_API_KEY" not in env assert "OPENAI_BASE_URL" not in env @@ -218,6 +231,7 @@ class TestRunAgent: assert calls["env"]["OPENAI_BASE_URL"] == "http://localhost:4000/v1" assert calls["env"]["OPENAI_API_KEY"] == "sk-key" assert "ANTHROPIC_BASE_URL" not in calls["env"] + assert "ENABLE_TOOL_SEARCH" not in calls["env"] def test_codex_injects_proxy_provider_args_before_user_args(self): calls = {} diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 85a4d90abf9..1d0a99b8e0a 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1373,6 +1373,7 @@ class TestLoginConfigClaude: assert result.exit_code == 0 written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://test.example.com" + assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" assert "Configured Claude Code" in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index 9010fb4c022..898f9ab1ed7 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -48,6 +48,7 @@ class TestWriteClaudeSettings: written = json.loads(settings_path.read_text()) assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" + assert written["env"]["ENABLE_TOOL_SEARCH"] == "true" assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token" def test_updates_an_existing_file_preserving_unrelated_settings(self, paths, lite_on_path): diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 9958286884b..053c90c36b4 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -55,8 +55,14 @@ class TestMergeClaudeSettings: } merged = merge_claude_settings(settings, "http://localhost:4000/", "new-helper") assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000" + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true" assert merged["apiKeyHelper"] == "new-helper" + def test_preserves_existing_tool_search(self): + settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}} + merged = merge_claude_settings(settings, "http://localhost:4000", "helper") + assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false" + def test_drops_stray_api_key(self): settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} merged = merge_claude_settings(settings, "http://localhost:4000", "helper") @@ -64,7 +70,10 @@ class TestMergeClaudeSettings: def test_works_from_empty_settings(self): merged = merge_claude_settings({}, "http://localhost:4000", "helper") - assert merged["env"] == {"ANTHROPIC_BASE_URL": "http://localhost:4000"} + assert merged["env"] == { + "ANTHROPIC_BASE_URL": "http://localhost:4000", + "ENABLE_TOOL_SEARCH": "true", + } assert merged["apiKeyHelper"] == "helper" def test_does_not_mutate_input(self): @@ -486,6 +495,7 @@ class TestUpCommand: assert captured["backup_existed"] is True assert captured["settings"]["theme"] == "dark" assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000" + assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true" assert captured["settings"]["apiKeyHelper"] == "/usr/local/bin/lite auth print-token" assert json.loads(settings_path.read_text()) == original assert not backup_path.exists() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py index b140082a3bf..f5d51a601d7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py @@ -432,7 +432,7 @@ class TestHiddenlayerGuardrail: @pytest.mark.asyncio async def test_apply_guardrail_request_with_image(self, monkeypatch: pytest.MonkeyPatch): - """Test apply_guardrail sends multimodal content (image) to HiddenLayer v1.""" + """Test apply_guardrail strips images from multimodal content before sending to HiddenLayer v1.""" monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( @@ -485,12 +485,13 @@ class TestHiddenlayerGuardrail: logging_obj=logging_obj, ) - # v1 API requires string content — multimodal list is stringified + # v1 API requires string content — image_url items are stripped and the + # remaining (text-only) content is stringified before being sent. mock_post.assert_called_once() call_kwargs = mock_post.call_args.kwargs sent_content = call_kwargs["json"]["input"]["messages"][0]["content"] assert isinstance(sent_content, str) - assert sent_content == str(multimodal_content) + assert sent_content == str([{"type": "text", "text": "how much is on this receipt?"}]) # Result should be returned without error assert result is not None diff --git a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py index 26beaa78a46..ab4e15ff423 100644 --- a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py @@ -1,16 +1,16 @@ -from fastapi.exceptions import HTTPException -from unittest.mock import patch, AsyncMock -from httpx import Response, Request +import asyncio import base64 +from unittest.mock import AsyncMock, patch import pytest - -from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import ( - PromptSecurityGuardrailMissingSecrets, - PromptSecurityGuardrail, -) +from fastapi.exceptions import HTTPException +from httpx import ReadTimeout, Request, Response import litellm +from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import ( + PromptSecurityGuardrail, + PromptSecurityGuardrailMissingSecrets, +) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 @@ -30,6 +30,7 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch): "guardrail": "prompt_security", "mode": "during_call", "default_on": True, + "file_sanitization_fail_open": False, }, } ], @@ -41,6 +42,10 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch): assert registered[0].guardrail_name == "prompt_security" assert registered[0].default_on is True assert registered[0].event_hook == "during_call" + assert registered[0].file_sanitization_fail_open is False + config_model = registered[0].get_config_model() + assert config_model is not None + assert config_model().file_sanitization_fail_open is True def test_prompt_security_guard_config_no_api_key(monkeypatch: pytest.MonkeyPatch): @@ -374,6 +379,86 @@ async def test_file_sanitization(monkeypatch: pytest.MonkeyPatch): assert result is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "timeout", + ( + litellm.Timeout( + message="Prompt Security upload timed out", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ), + ReadTimeout( + "Prompt Security poll timed out", + request=Request(method="GET", url="https://test.prompt.security/api/sanitizeFile"), + ), + ), + ids=("litellm", "httpx"), +) +@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed")) +async def test_file_sanitization_request_timeout_policy( + monkeypatch: pytest.MonkeyPatch, timeout: Exception, fail_open: bool +): + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") + + guardrail = PromptSecurityGuardrail( + guardrail_name="test-guard", + event_hook="pre_call", + default_on=True, + file_sanitization_fail_open=fail_open, + ) + + with patch.object(guardrail.async_handler, "post", AsyncMock(side_effect=timeout)): + if not fail_open: + with pytest.raises(HTTPException) as exc_info: + await guardrail.sanitize_file_content(b"file-content", "document.pdf") + assert exc_info.value.status_code == 408 + assert exc_info.value.detail == "File sanitization timeout" + return + + result = await guardrail.sanitize_file_content(b"file-content", "document.pdf") + + assert result == { + "action": "allow", + "content": None, + "metadata": {}, + "violations": (), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed")) +async def test_file_sanitization_overall_timeout_policy(monkeypatch: pytest.MonkeyPatch, fail_open: bool): + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") + + guardrail = PromptSecurityGuardrail( + guardrail_name="test-guard", + event_hook="pre_call", + default_on=True, + file_sanitization_timeout=0.01, + file_sanitization_fail_open=fail_open, + ) + + async def hanging_post(*_args: object, **_kwargs: object) -> None: + await asyncio.sleep(60) + raise AssertionError("sanitization request should have been cancelled") + + with patch.object(guardrail.async_handler, "post", side_effect=hanging_post): + if not fail_open: + with pytest.raises(HTTPException) as exc_info: + await guardrail.sanitize_file_content(b"file-content", "document.pdf") + assert exc_info.value.status_code == 408 + assert exc_info.value.detail == "File sanitization timeout" + return + + result = await guardrail.sanitize_file_content(b"file-content", "document.pdf") + + assert result["action"] == "allow" + assert result["content"] is None + + @pytest.mark.asyncio async def test_file_sanitization_block(monkeypatch: pytest.MonkeyPatch): """Test that file sanitization blocks malicious files""" @@ -544,7 +629,7 @@ async def test_role_filtering(monkeypatch: pytest.MonkeyPatch): return mock_response with patch.object(guardrail.async_handler, "post", side_effect=mock_post): - result = await guardrail.apply_guardrail( + await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, input_type="request", diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index 5c163c44cb3..1225cb80224 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -83,6 +83,50 @@ def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth): assert response.json()["alias"] == "Updated Test User" +def test_update_customer_unblock(mock_prisma_client, mock_user_api_key_auth): + mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=False) + + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user) + mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user) + + response = client.post( + "/customer/update", + json={"user_id": "test-user-1", "blocked": False}, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + assert response.json()["blocked"] is False + update_mock = mock_prisma_client.db.litellm_endusertable.update + update_mock.assert_called_once() + assert update_mock.call_args.kwargs["data"]["blocked"] is False + + +def test_update_customer_keeps_blocked_when_omitted(mock_prisma_client, mock_user_api_key_auth): + """ + Regression test: updating a blocked customer without supplying `blocked` + must NOT reset it to unblocked. `blocked=False` is the model default and + should only be applied when explicitly provided by the caller. + """ + mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user) + mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user) + + response = client.post( + "/customer/update", + json={"user_id": "test-user-1", "alias": "Updated Test User"}, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + update_mock = mock_prisma_client.db.litellm_endusertable.update + update_mock.assert_called_once() + assert "blocked" not in update_mock.call_args.kwargs["data"] + + def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth): """ Test that update_end_user raises a 404 ProxyException when user_id does not exist. diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 791d64c6428..d7010de6405 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py """ import unittest +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from httpx import Response import litellm from litellm.proxy.proxy_server import app @@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText( - type="output_text", text="Hello from Cursor!" - ) - ], + content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")], ) ], ) @@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.llm_router") @patch("litellm.proxy.proxy_server.user_api_key_auth") - async def test_responses_api_key_spend_header_includes_response_cost( - self, mock_auth, mock_router - ): + async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router): """ Test that x-litellm-key-spend header includes the current request's response_cost for /v1/responses endpoint. @@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText(type="output_text", text="Test response") - ], + content=[ResponseOutputText(type="output_text", text="Test response")], ) ], ) @@ -356,6 +350,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "model": "gpt-4o", "input": "hello"} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -363,6 +358,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -370,6 +366,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = { "type": "response.create", "model": "flat-model", @@ -381,6 +378,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "input": "hello"} assert _extract_model_from_first_ws_event(event) is None @@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation: ws.send_text.assert_awaited_once() ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message") error_payload = json.loads(ws.send_text.await_args.args[0]) - assert ( - error_payload["error"]["message"] - == "First message must be a response.create JSON object." - ) + assert error_payload["error"]["message"] == "First message must be a response.create JSON object." @pytest.mark.asyncio async def test_rejects_non_object_json_first_frame(self): @@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth: ws.url = "ws://testserver/v1/responses" ws.accept = AsyncMock() ws.receive_text = AsyncMock( - return_value=json.dumps( - {"type": "response.create", "model": "gpt-4o-mini", "input": []} - ) + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) ) ws.close = AsyncMock() processor = MagicMock() - processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "gpt-4o-mini"}, MagicMock()) - ) + processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock())) async def fake_llm_call(): return None @@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth: _enforce_responses_ws_first_frame_model_auth, ) - request = Request( - {"type": "http", "method": "POST", "path": "/v1/responses", "headers": []} - ) + request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) user_api_key_dict = MagicMock() llm_router = MagicMock() @@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None ws.send_text.assert_not_awaited() - ws.close.assert_awaited_once_with( - code=1008, reason="Timed out waiting for first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message") @pytest.mark.asyncio async def test_invalid_json_sends_error_and_closes(self): @@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None payload = json.loads(ws.send_text.await_args.args[0]) assert payload["error"]["message"] == "First message is not valid JSON." - ws.close.assert_awaited_once_with( - code=1008, reason="Invalid JSON in first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message") @pytest.mark.asyncio async def test_missing_model_sends_error_and_closes(self): @@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "response.create", "input": []}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider: assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True def test_different_provider_is_not_same(self): - assert ( - self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") - is False - ) + assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False def test_inject_credentials_keeps_provider_for_same_provider_model(self): handler = self._handler("gpt-4o", custom_llm_provider="openai") @@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider: assert "custom_llm_provider" not in call_kwargs def test_unresolvable_connection_model_falls_back_to_custom_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") assert handler._same_provider("gpt-4o-mini") is True call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="gpt-4o-mini") assert call_kwargs["custom_llm_provider"] == "openai" def test_unresolvable_connection_model_still_drops_cross_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash") assert "custom_llm_provider" not in call_kwargs @@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s type="message", role="assistant", status="completed", - content=[ - ResponseOutputText(type="output_text", text="agent reply", annotations=[]) - ], + content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])], ) ], ) @@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s app.dependency_overrides[user_api_key_auth] = _auth_override try: - with patch.object(ps, "llm_router", mock_router), patch( - "litellm.proxy.response_api_endpoints.endpoints._read_request_body", - side_effect=capturing_read_request_body, + with ( + patch.object(ps, "llm_router", mock_router), + patch( + "litellm.proxy.response_api_endpoints.endpoints._read_request_body", + side_effect=capturing_read_request_body, + ), ): client = TestClient(app) response = client.post( @@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock: mock_router.router_general_settings.pass_through_all_models = False mock_router.default_deployment = None mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]} - mock_router.pattern_router.get_pattern.side_effect = ( - lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None + mock_router.pattern_router.get_pattern.side_effect = lambda model: ( + [{"model_name": "anthropic/*"}] if model == base_model else None ) return mock_router @@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups: from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant router = Router( - model_list=[ - {"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}} - ], + model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}], routing_groups=[ {"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"} ], @@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage: assert usage["input_tokens"] == 0 assert usage["output_tokens"] == 0 assert usage["total_tokens"] == 0 + + +class TestResponsesInputTokens: + """Regression tests for POST /v1/responses/input_tokens. + + The docs promise OpenAI-format token counting on the proxy, but the route was + never registered, so the POST fell through to the GET/DELETE-only + /v1/responses/{response_id} route and returned 405.""" + + def _post_input_tokens( + self, + body: dict[str, Any], + path: str = "/v1/responses/input_tokens", + counter: AsyncMock | None = None, + ) -> tuple[Response, AsyncMock]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter + from litellm.types.utils import TokenCountResponse + + token_counter_mock = ( + counter + if counter is not None + else AsyncMock( + return_value=TokenCountResponse( + total_tokens=13, + request_model=body.get("model", ""), + model_used=body.get("model", ""), + tokenizer_type="openai_api", + ) + ) + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path) + app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock + try: + client = TestClient(app) + response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"}) + return response, token_counter_mock + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + app.dependency_overrides.pop(_proxy_token_counter, None) + + def test_string_input_returns_openai_input_tokens_shape(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) + + assert response.status_code == 200, response.text + assert response.json() == {"object": "response.input_tokens", "input_tokens": 13} + counter.assert_awaited_once() + assert counter.call_args.kwargs["call_endpoint"] is True + token_request = counter.call_args.kwargs["request"] + assert token_request.model == "gpt-4o" + assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}] + + def test_every_route_alias_is_registered(self): + for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"): + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path) + assert response.status_code == 200, f"{path}: {response.status_code} {response.text}" + + def test_input_items_instructions_and_tools_are_forwarded(self): + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ] + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": [{"role": "user", "content": "What is the weather in Paris?"}], + "instructions": "You are terse.", + "tools": tools, + } + ) + + assert response.status_code == 200, response.text + token_request = counter.call_args.kwargs["request"] + assert token_request.messages == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "What is the weather in Paris?"}, + ] + assert token_request.tools == tools + + def test_missing_model_returns_openai_400(self): + response, counter = self._post_input_tokens({"input": "Hello"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'model'.", + "type": "invalid_request_error", + "param": "model", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_missing_input_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'input'.", + "type": "invalid_request_error", + "param": "input", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + @pytest.mark.parametrize("empty_input", ["", []]) + def test_empty_input_returns_openai_400(self, empty_input): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + "type": "invalid_request_error", + "param": None, + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_invalid_tools_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"}) + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + counter.assert_not_awaited() + + def test_provider_error_maps_status_code(self): + from litellm.proxy._types import ProxyException + + failing_counter = AsyncMock( + side_effect=ProxyException( + message="rate limited", + type="token_counting_error", + param="model", + code="429", + ) + ) + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter) + + assert response.status_code == 429, response.text + assert response.json()["error"]["message"] == "rate limited" diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py new file mode 100644 index 00000000000..f65f68812a2 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -0,0 +1,48 @@ +from typing import Final + +import pytest + +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request +from litellm.proxy.utils import ProxyLogging + +TOKEN_COUNTING_ROUTES: Final = ( + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + "/utils/token_counter", +) + + +def _budgeted_token() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0) + + +async def _reserve(route: str) -> dict | None: + return await reserve_budget_for_request( + request_body={"model": "gpt-4o", "input": "hello"}, + route=route, + llm_router=None, + valid_token=_budgeted_token(), + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES) +async def test_token_counting_routes_are_exempt_from_budget_reservation(route): + assert await _reserve(route) is None + + +@pytest.mark.asyncio +async def test_non_exempt_llm_route_still_reserves_budget(): + reservation: Final = await _reserve("/v1/responses") + + assert reservation is not None + assert reservation["reserved_cost"] > 0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 10c3e5fecf8..a0dcbf802ef 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2865,7 +2865,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2961,7 +2961,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -3055,7 +3055,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5022dab32be..9e5917637a8 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2,6 +2,7 @@ import asyncio import datetime import json from datetime import timezone +from collections.abc import Mapping from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -3956,3 +3957,71 @@ def test_passthrough_caching_carries_no_injection_marker(): ) metadata = json.loads(payload["metadata"]) assert metadata["litellm_gateway_injected_cache"] is None + + +def _routed_call_kwargs(model_info: Mapping[str, object]) -> dict[str, object]: + return { + "model": "claude-haiku-4-5", + "custom_llm_provider": "azure_ai", + "litellm_call_id": "router-corr-123", + "litellm_params": { + "metadata": { + "user_api_key": "test-key", + "model_group": "internal-router/gpt-5.4", + "deployment": "azure_ai/claude-haiku-4-5", + "model_info": model_info, + } + }, + } + + +def test_router_metadata_stamped_for_internal_router_model_deployment(): + """A deployment flagged model_info.internal_router_model gets a router_metadata + block correlating the requested model group with the selected deployment.""" + payload = get_logging_payload( + kwargs=_routed_call_kwargs({"id": "mi-1", "internal_router_model": True}), + response_obj=litellm.ModelResponse(id="chatcmpl-router-meta", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] == { + "requested_model": "internal-router/gpt-5.4", + "selected_model": "azure_ai/claude-haiku-4-5", + "selected_provider": "azure_ai", + "router_correlation_id": "router-corr-123", + } + + +def test_router_metadata_absent_without_internal_router_model_flag(): + payload = get_logging_payload( + kwargs=_routed_call_kwargs({"id": "mi-1"}), + response_obj=litellm.ModelResponse(id="chatcmpl-unflagged", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] is None + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_caller_forged_router_metadata_is_discarded(bucket): + """The raw request bucket is client-writable and _get_spend_logs_metadata projects + every SpendLogsMetadata key from it, so the server-derived value must overwrite + unconditionally or a caller could plant router provenance the router never produced.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": { + bucket: { + "user_api_key": "test-key", + "router_metadata": {"requested_model": "forged", "router_correlation_id": "forged-id"}, + } + }, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-forged-router-meta", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] is None diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py b/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py index cede859cb38..77c0f71dbf9 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py @@ -115,6 +115,35 @@ async def test_budget_alerts_slack_when_slack_alerting(proxy_logging): assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"} +@pytest.mark.asyncio +async def test_budget_alerts_webhook_only_forwards_to_slack_alerting_instance(proxy_logging): + proxy_logging.alerting = ["webhook"] + captured: Dict[str, Any] = {} + + async def fake_alert(**kwargs): + captured.update(kwargs) + + proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=fake_alert) + proxy_logging.email_logging_instance = None + await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info()) + snapshot = { + "type": captured["type"], + "user_info_is_callinfo": isinstance(captured["user_info"], CallInfo), + "user_id": captured["user_info"].user_id, + } + assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"} + + +@pytest.mark.asyncio +async def test_budget_alerts_email_only_skips_slack_alerting_instance(proxy_logging): + proxy_logging.alerting = ["email"] + proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=AsyncMock()) + proxy_logging.email_logging_instance = MagicMock(budget_alerts=AsyncMock()) + await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info()) + proxy_logging.slack_alerting_instance.budget_alerts.assert_not_called() + proxy_logging.email_logging_instance.budget_alerts.assert_called_once() + + @pytest.mark.asyncio async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_global(proxy_logging): proxy_logging.alerting = None diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b96d2eb5322..bb59e576568 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig: assert "extra_field" not in result["file"] assert "another_field" not in result["file"] + def test_transform_input_file_item_to_file_item_keeps_filename(self): + """OpenAI rejects file_data with no filename beside it, so dropping it 400s the request""" + result = ( + LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item( + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + } + ) + ) + assert result == { + "type": "file", + "file": { + "file_data": "data:application/pdf;base64,JVBERi0=", + "filename": "report.pdf", + }, + } + def test_transform_input_file_item_to_file_item_with_file_url(self): """file_url should be mapped to file_id for downstream URL handling""" result = ( diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 57ee0606dca..92c0509b072 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22462 + "limit": 22403 }, "LIT002": { - "limit": 26800 + "limit": 26780 }, "LIT003": { "limit": 269 @@ -27,10 +27,10 @@ "limit": 0 }, "LIT010": { - "limit": 16529 + "limit": 16512 }, "LIT011": { - "limit": 5556 + "limit": 5537 }, "LIT012": { "limit": 4495 diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 05e66985e6f..72da01d0918 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -522,7 +522,8 @@ const Settings: React.FC = ({ accessToken, userRole, userID,

- Alerts are only supported for Slack Webhook URLs. Get your webhook urls from{" "} + Alerts are sent to any Slack-compatible incoming webhook URL (Slack, Rocket.Chat, Mattermost, etc.). Get + Slack webhook urls from{" "} here @@ -532,7 +533,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, - Slack Webhook URL + Webhook URL (Slack-compatible) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 137c67e837c..6f6c3165e4b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9700,6 +9700,37 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses/{response_id}": { parameters: { query?: never; @@ -12619,6 +12650,37 @@ export interface paths { patch?: never; trace?: never; }; + "/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/responses/{response_id}": { parameters: { query?: never; @@ -15505,7 +15567,7 @@ export interface paths { * Use this if if you want different teams to have different success/failure callbacks * * Parameters: - * - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add + * - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials * - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of: * - "success": Callback for successful LLM calls * - "failure": Callback for failed LLM calls @@ -15521,6 +15583,8 @@ export interface paths { * - langsmith_api_key: The API key for the Langsmith callback * - langsmith_project: The project for the Langsmith callback * - langsmith_base_url: The base URL for the Langsmith callback + * - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400 + * - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key * * Example curl: * ``` @@ -16210,6 +16274,11 @@ export interface paths { * * Meant to optimize querying spend data for analytics for a user. * + * Reads daily spend records that only ever accumulate and are never affected by budget + * resets. Their total can legitimately exceed the `spend` field returned by + * `/v2/user/info`, which is a running budget counter that every budget reset sets back + * to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). + * * Returns: * (by date) * - spend @@ -16241,6 +16310,11 @@ export interface paths { * Get User Daily Activity Aggregated * @description Aggregated analytics for a user's daily activity without pagination. * Returns the same response shape as the paginated endpoint with page metadata set to single-page. + * + * Reads daily spend records that only ever accumulate and are never affected by budget + * resets. Their total can legitimately exceed the `spend` field returned by + * `/v2/user/info`, which is a running budget counter that every budget reset sets back + * to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). */ get: operations["get_user_daily_activity_aggregated_user_daily_activity_aggregated_get"]; put?: never; @@ -19182,6 +19256,37 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/responses/{response_id}": { parameters: { query?: never; @@ -21004,6 +21109,14 @@ export interface paths { * This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem * where the old endpoint loaded all keys and teams into memory. * + * Note on `spend`: this is the user's running budget counter, which the budget reset job + * resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default, + * or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT + * lifetime or per-period historical spend. For historical spend over a date range, use + * `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend + * records that only ever accumulate and are never reset. The two values are expected to + * diverge once a budget reset has occurred within the queried period. + * * Access control: * - Proxy admins can query any user * - Team admins can query users within their teams @@ -25182,7 +25295,7 @@ export interface components { alert_types?: components["schemas"]["AlertType"][] | null; /** * Alerting - * @description List of alerting integrations. Today, just slack - `alerting: ['slack']` + * @description List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL */ alerting?: unknown[] | null; /** @@ -38418,6 +38531,8 @@ export interface components { input_cost_per_character?: number | null; /** Input Cost Per Token */ input_cost_per_token?: number | null; + /** Internal Router Model */ + internal_router_model?: boolean | null; /** Output Cost Per Character */ output_cost_per_character?: number | null; /** Output Cost Per Token */ @@ -51474,6 +51589,26 @@ export interface operations { }; }; }; + responses_input_tokens_openai_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_openai_v1_responses__response_id__get: { parameters: { query?: never; @@ -54438,6 +54573,26 @@ export interface operations { }; }; }; + responses_input_tokens_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_responses__response_id__get: { parameters: { query?: never; @@ -62860,6 +63015,26 @@ export interface operations { }; }; }; + responses_input_tokens_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_v1_responses__response_id__get: { parameters: { query?: never;