diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index 36c6c790b84..c3b8ce22c68 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -4,17 +4,16 @@ description: >- so only the first job on a given Cargo.lock compiles the bridge from scratch. litellm builds through maturin, which compiles litellm-rust/crates/python-bridge - in release mode before it can produce a wheel. `uv sync` therefore pays a full - build in every job that installs the workspace: measured at 2m40s per unit shard - on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught - it, because the uv cache holds wheels uv downloads rather than wheels it builds, - and a path dependency whose source moves every commit could never hit that cache - anyway. Cargo rebuilds only what changed when its target directory survives, so a - warm job pays for the bridge crate alone. + in the dev profile for editable installs. `uv sync` therefore pays a full build + in every job that installs the workspace. Nothing caught it, because the uv cache + holds wheels uv downloads rather than wheels it builds, and a path dependency + whose source moves every commit could never hit that cache anyway. Cargo rebuilds + only what changed when its target directory survives, so a warm job pays for the + bridge crate alone. - The key namespace is separate from test-rust.yml's. Both cache the same directory, - but that workflow fills it with debug and clippy artifacts, which a release build - cannot reuse, and a shared key would let whichever ran first deny the other a save. + The key namespace is separate from test-rust.yml's check and release caches. They + cache the same directory for different workloads, and a shared key would let + whichever ran first deny the others a save. runs: using: composite @@ -26,6 +25,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-release- + ${{ runner.os }}-maturin-dev- diff --git a/.github/scripts/close_duplicate_issues.py b/.github/scripts/close_duplicate_issues.py deleted file mode 100755 index ec522af4f88..00000000000 --- a/.github/scripts/close_duplicate_issues.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python3 -""" -Detect and close duplicate GitHub issues using title similarity. - -Modes: - --scan Compare all open issues against each other (batch) - --issue-number N Check a single issue against older open issues - -Requires the `gh` CLI to be authenticated. -""" - -import argparse -import difflib -import json -import re -import subprocess -import sys - - -def normalize_title(title: str) -> str: - """Strip common prefixes, lowercase, and collapse whitespace.""" - title = re.sub( - r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*", - "", - title, - flags=re.IGNORECASE, - ) - return " ".join(title.lower().split()) - - -def gh(*args: str) -> str: - """Run a gh CLI command and return stdout.""" - result = subprocess.run( - ["gh", *args], - capture_output=True, - text=True, - check=True, - ) - return result.stdout - - -def fetch_open_issues(repo: str | None) -> list[dict]: - """Fetch all open issues (excluding PRs) via gh api --paginate.""" - if repo: - endpoint = ( - f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - ) - else: - endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc" - cmd = ["api", "--paginate", endpoint] - - raw = gh(*cmd) - # gh --paginate concatenates JSON arrays, so we may get multiple arrays - issues = [] - for line in raw.strip().splitlines(): - line = line.strip() - if not line: - continue - parsed = json.loads(line) - if isinstance(parsed, list): - issues.extend(parsed) - else: - issues.append(parsed) - - # Filter out pull requests (they also appear in the issues endpoint) - return [i for i in issues if "pull_request" not in i] - - -def close_as_duplicate( - issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool -) -> None: - """Close an issue as duplicate of another, adding a comment and label.""" - repo_args = ["--repo", repo] if repo else [] - - if dry_run: - print( - f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}" - ) - return - - # Add comment - comment_body = ( - f"Closing as duplicate of #{duplicate_of}.\n\n" - "If you believe this is not a duplicate, please reopen and add context " - "explaining how this differs." - ) - gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args) - - # Add label - gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args) - - # Close with not_planned reason - gh( - "api", - f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}", - "-X", - "PATCH", - "-f", - "state=closed", - "-f", - "state_reason=not_planned", - ) - - print(f" Closed #{issue_number} as duplicate of #{duplicate_of}") - - -def find_duplicate( - issue: dict, candidates: list[dict], threshold: float -) -> dict | None: - """Return the first candidate whose normalized title is above threshold.""" - norm = normalize_title(issue["title"]) - for candidate in candidates: - if candidate["number"] == issue["number"]: - continue - cand_norm = normalize_title(candidate["title"]) - ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio() - if ratio >= threshold: - return candidate - return None - - -def scan_all( - issues: list[dict], threshold: float, repo: str | None, dry_run: bool -) -> int: - """Compare every issue against all older issues. Returns count of duplicates found.""" - # Sort oldest first - issues.sort(key=lambda i: i["number"]) - closed_count = 0 - - for idx, issue in enumerate(issues): - older = issues[:idx] - if not older: - continue - dup = find_duplicate(issue, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(issue["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{issue['number']}: \"{issue['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue["number"], dup["number"], repo, dry_run) - closed_count += 1 - - return closed_count - - -def check_single( - issue_number: int, - issues: list[dict], - threshold: float, - repo: str | None, - dry_run: bool, -) -> bool: - """Check a single issue against all older open issues. Returns True if duplicate found.""" - target = None - for i in issues: - if i["number"] == issue_number: - target = i - break - - if target is None: - print(f"Issue #{issue_number} not found among open issues.") - return False - - older = [i for i in issues if i["number"] < issue_number] - dup = find_duplicate(target, older, threshold) - if dup: - ratio = difflib.SequenceMatcher( - None, - normalize_title(target["title"]), - normalize_title(dup["title"]), - ).ratio() - print( - f"#{target['number']}: \"{target['title']}\"\n" - f" -> duplicate of #{dup['number']}: \"{dup['title']}\" " - f"({ratio:.0%} similar)" - ) - close_as_duplicate(issue_number, dup["number"], repo, dry_run) - return True - - print(f"#{issue_number}: no duplicate found above threshold {threshold}") - return False - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Detect and close duplicate GitHub issues" - ) - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--scan", action="store_true", help="Scan all open issues") - mode.add_argument("--issue-number", type=int, help="Check a single issue number") - parser.add_argument( - "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)" - ) - parser.add_argument( - "--close", - action="store_true", - help="Actually close duplicates (default is dry-run)", - ) - parser.add_argument( - "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted." - ) - args = parser.parse_args() - - dry_run = not args.close - - if dry_run: - print("=== DRY RUN MODE (pass --close to actually close issues) ===\n") - - print("Fetching open issues...") - issues = fetch_open_issues(args.repo) - print(f"Found {len(issues)} open issues.\n") - - if args.scan: - count = scan_all(issues, args.threshold, args.repo, dry_run) - print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}") - else: - found = check_single( - args.issue_number, issues, args.threshold, args.repo, dry_run - ) - sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.yml new file mode 100644 index 00000000000..d8256917805 --- /dev/null +++ b/.github/workflows/auto-close-duplicates.yml @@ -0,0 +1,69 @@ +name: Auto-close duplicate issues + +on: + schedule: + - cron: "0 9 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Log which issues would close without closing anything + type: boolean + default: true + grace_period_days: + description: Days a duplicate notice must go unanswered before the close + type: number + default: 3 + pull_request: + paths: + - .github/workflows/auto-close-duplicates.yml + - scripts/auto-close-duplicates.ts + - scripts/auto-close-duplicates.test.ts + +permissions: {} + +jobs: + test: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.4.0" + + - name: Test the sweep + run: bun test scripts/auto-close-duplicates.test.ts + + sweep: + if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + # Exact version, never latest: the next step holds an issues: write token + bun-version: "1.4.0" + + - name: Close unanswered duplicates, reopen ones the reporter answered + run: bun run scripts/auto-close-duplicates.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ inputs.dry_run == true }} + GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }} diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml index 78198b2c7bb..41ec43a1d9b 100644 --- a/.github/workflows/check_duplicate_issues.yml +++ b/.github/workflows/check_duplicate_issues.yml @@ -1,12 +1,19 @@ name: Check Duplicate Issues +# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later, +# and only when its title is identical to an older open issue and nobody replied. +# The HTML marker below is the handshake between the two, so keep it in the template. + on: issues: types: [opened, edited] +permissions: {} + jobs: check-duplicate: runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write contents: read @@ -19,35 +26,12 @@ jobs: threshold: 0.6 reaction: eyes comment: | - **⚠️ Potential duplicate detected** + + **Potential duplicate detected** - This issue appears similar to existing issue(s): + This looks similar to: {{#issues}} - - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar) + - #{{number}} - {{title}} {{/issues}} - Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. - - - name: Checkout close script - if: github.event.action == 'opened' - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - sparse-checkout: .github/scripts - persist-credentials: false - - - name: Set up Python - if: github.event.action == 'opened' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Auto-close if high-confidence duplicate - if: github.event.action == 'opened' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - python3 .github/scripts/close_duplicate_issues.py \ - --issue-number ${{ github.event.issue.number }} \ - --repo ${{ github.repository }} \ - --threshold 0.85 \ - --close + If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open. diff --git a/.github/workflows/test-redis-compat.yml b/.github/workflows/test-redis-compat.yml new file mode 100644 index 00000000000..f29755a74b1 --- /dev/null +++ b/.github/workflows/test-redis-compat.yml @@ -0,0 +1,77 @@ +name: "Unit Tests: Redis Client Version Compatibility" + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + paths: + - "litellm/_redis.py" + - "litellm/_redis_credential_provider.py" + - "tests/test_litellm/test_redis.py" + - "tests/test_litellm/caching/test_redis_connection_pool.py" + - ".github/workflows/test-redis-compat.yml" + - "pyproject.toml" + - "uv.lock" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + redis-compat: + name: "redis-py ${{ matrix.redis-version }}" + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the + # newer legs prove the inspect.signature introspection in litellm/_redis.py + # keeps extracting kwargs on the redis-py releases people actually run now. + # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra) + # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in + # for the 6.x line. + redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"] + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Pin redis-py to the matrix version + env: + REDIS_VERSION: ${{ matrix.redis-version }} + run: | + uv pip install "redis==${REDIS_VERSION:?}" + uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)" + + - name: Run redis unit tests + run: | + uv run --no-sync pytest \ + tests/test_litellm/test_redis.py \ + tests/test_litellm/caching/test_redis_connection_pool.py \ + --tb=short -vv \ + --reruns 2 \ + --reruns-delay 1 \ + --durations=20 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index ed9d8800202..c2dff805772 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -103,6 +103,7 @@ jobs: tests/test_litellm/completion_extras tests/test_litellm/compression tests/test_litellm/containers + tests/test_litellm/endpoints tests/test_litellm/experimental_mcp_client tests/test_litellm/models tests/test_litellm/repositories diff --git a/CLAUDE.md b/CLAUDE.md index 930825aeb89..d9e9e8f1586 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,8 @@ When adding new features, add meaningful tests. Don't add tests that don't check Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression) +Never test structure of code only function of it + `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md` diff --git a/Dockerfile b/Dockerfile index 700b0d6525e..b3ee85e9ed1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ gcc \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ rust \ openssl \ openssl-dev \ @@ -51,6 +51,7 @@ RUN apk add --no-cache \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching @@ -65,7 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -86,7 +87,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -101,7 +102,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 4ca40944606..aa01b9fba8b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -57,7 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -71,7 +71,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index d60c3e9c0af..551423c7707 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -3,7 +3,7 @@ "limit": 16171 }, "reportArgumentType": { - "limit": 2226 + "limit": 2224 }, "reportAssignmentType": { "limit": 319 @@ -57,7 +57,7 @@ "limit": 5611 }, "reportMissingTypeArgument": { - "limit": 15350 + "limit": 15348 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,19 +99,19 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44368 + "limit": 44364 }, "reportUnknownLambdaType": { "limit": 109 }, "reportUnknownMemberType": { - "limit": 38468 + "limit": 38465 }, "reportUnknownParameterType": { - "limit": 19665 + "limit": 19663 }, "reportUnknownVariableType": { - "limit": 30066 + "limit": 30064 }, "reportUnnecessaryCast": { "limit": 111 diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index f0d6d02fccf..c1348f68231 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ gcc \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ openssl \ openssl-dev \ nodejs \ @@ -49,6 +49,7 @@ RUN apk add --no-cache \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching @@ -63,7 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -84,7 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -98,7 +99,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile +RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile WORKDIR /app ENV PATH="/app/.venv/bin:${PATH}" \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4a5df6ecd69..2221435a83a 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. @@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN for i in 1 2 3; do \ apk add --no-cache \ - python3 \ - python3-dev \ + python-3.13 \ + python-3.13-dev \ gcc \ rust \ bash \ @@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" \ LITELLM_NON_ROOT=true \ XDG_CACHE_HOME=/app/.cache @@ -69,7 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 + --python python3.13 # Copy full source tree COPY . . @@ -96,7 +97,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3 \ + --python python3.13 \ --no-sources-package litellm-proxy-extras; \ else \ uv sync --frozen --no-default-groups --no-editable \ @@ -105,7 +106,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --extra extra_proxy \ --extra semantic-router \ --extra saml \ - --python python3; \ + --python python3.13; \ fi RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ @@ -124,7 +125,7 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ done # Copy only what runtime needs. The application is installed inside the venv; diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 4a2e32e186e..308d70a6b26 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --extra semantic-router \ --extra bedrock-realtime \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --extra semantic-router \ --extra bedrock-realtime \ - --python python3 + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 05baf98bbb5..92b73867e67 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -86,6 +86,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/comprehendmedical", "/cohere/", "/gemini/", + "/gigachat/", "/google/", "/vertex_ai/", "/vertex-ai/", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql new file mode 100644 index 00000000000..b7dbe931dd2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql @@ -0,0 +1,21 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id' + ) THEN + ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id"; + END IF; +END $$; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key'; + +DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction"; + +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction" + ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL; + +DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx"; + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx" + ON "LiteLLM_ShadowEvalJob"("target_type", "target_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 60223265211..01a607b68a9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b2dc0a52c8f..b8032dd0d28 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -512,6 +512,13 @@ class ProxyExtrasDBManager: try: import psycopg except ImportError: + logger.warning( + "psycopg is not installed; skipping the LiteLLM_SpendLogs " + "partition check. If this table is partitioned (see " + "db_scripts/partition_spend_logs.sql), schema reconciliation " + "will try to rewrite its primary key and fail. Install the " + "litellm[extra_proxy] extra, which now includes psycopg." + ) return False cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 481ea3f8f66..c17a0605fc7 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -30,3 +30,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "unwind" +debug = false +incremental = false +strip = "symbols" diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 0c4a753f762..d461a483ae0 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,8 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["extension-module"] +default = ["abi3"] +abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] [dependencies] diff --git a/litellm/__init__.py b/litellm/__init__.py index c83e72a78b4..1447e05fdf7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,6 +7,9 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.* # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances # This warning can accumulate during streaming and cause memory leaks warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*") +# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it +# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked +warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*") ### INIT VARIABLES ######################### import threading import os diff --git a/litellm/_logging.py b/litellm/_logging.py index fbb35b72be2..9435562f890 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -264,13 +264,17 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: class LevelRoutingStreamHandler(logging.StreamHandler): - """Writes records below WARNING to stdout and WARNING and above to stderr. + """Writes records below WARNING and invalid-key warnings to stdout, others to stderr. Collectors that derive severity from the stream report every stderr line as an error. + Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them. """ def emit(self, record: logging.LogRecord) -> None: - preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr + is_stdout_record: Final = record.levelno < logging.WARNING or ( + record.levelno == logging.WARNING and record.name == verbose_proxy_stdout_logger.name + ) + preferred: Final = sys.stdout if is_stdout_record else sys.stderr if preferred is None or getattr(preferred, "closed", False): self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record else: @@ -508,6 +512,9 @@ else: handler.setFormatter(formatter) verbose_proxy_logger = logging.getLogger("LiteLLM Proxy") +# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler +# writes its WARNING records to stdout. It has no handler or level of its own. +verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout") verbose_router_logger = logging.getLogger("LiteLLM Router") verbose_logger = logging.getLogger("LiteLLM") @@ -520,6 +527,7 @@ verbose_logger.addHandler(handler) # handlers (JSON mode, uvicorn log config, a host app's root handler). verbose_router_logger.addFilter(_stdout_truncation_filter) verbose_proxy_logger.addFilter(_stdout_truncation_filter) +verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter) verbose_logger.addFilter(_stdout_truncation_filter) @@ -683,6 +691,7 @@ def _turn_on_json(): - Adds a JSON formatter to all loggers """ handler: Final = LevelRoutingStreamHandler() + handler.setLevel(numeric_level) handler.setFormatter(JsonFormatter()) _initialize_loggers_with_handler(handler) # Set up exception handlers @@ -700,12 +709,14 @@ def _disable_debugging(): verbose_logger.disabled = True verbose_router_logger.disabled = True verbose_proxy_logger.disabled = True + verbose_proxy_stdout_logger.disabled = True def _enable_debugging(): verbose_logger.disabled = False verbose_router_logger.disabled = False verbose_proxy_logger.disabled = False + verbose_proxy_stdout_logger.disabled = False def print_verbose(print_statement): diff --git a/litellm/_redis.py b/litellm/_redis.py index 9381357931e..3e68d50cf16 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -13,6 +13,7 @@ import json # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation import os from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from urllib.parse import urlsplit, urlunsplit @@ -38,9 +39,25 @@ from ._logging import verbose_logger AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" -def _get_redis_kwargs(): - arg_spec: Final = inspect.getfullargspec(redis.Redis) +def _unwrapped_init_args(cls: type) -> frozenset[str]: + """Every parameter on a single class's own ``__init__``, decorator-unwrapped. + Unlike ``_init_arg_names`` below, this does not walk the MRO: ``redis.Redis`` + and ``redis.RedisCluster`` (sync and async) each declare every real + constructor parameter directly on their own ``__init__``, so MRO-walking is + unnecessary — and it actively breaks the several tests here that mock the + class with ``patch(..., autospec=True)``, since ``inspect.getmro`` needs a + real ``__mro__`` that an autospec'd stand-in for a class does not provide. + + Still unwraps first: redis-py >= 7.4 decorates these ``__init__``s with + ``@deprecated_args`` too, which the same class of bug as ``_init_arg_names`` + would otherwise silently empty this allowlist through (see its docstring). + """ + spec: Final = inspect.getfullargspec(inspect.unwrap(cls.__init__)) + return frozenset(spec.args + spec.kwonlyargs) + + +def _get_redis_kwargs(): # Only allow primitive arguments exclude_args: Final = { "self", @@ -60,7 +77,7 @@ def _get_redis_kwargs(): "azure_client_secret", } - available_args: Final = {x for x in arg_spec.args if x not in exclude_args} | include_args + available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args return available_args @@ -120,15 +137,23 @@ def _get_redis_url_kwargs(client: type | None = None) -> tuple[str, ...]: return tuple(x for x in _init_arg_names(connection_cls) if x not in exclude_args) + include_args -def _get_redis_cluster_kwargs(client=None): +def _get_redis_cluster_kwargs(client: type | None = None): + """Config kwargs the target cluster client's constructor actually accepts. + + Defaults to the sync ``redis.RedisCluster``, but the async cluster client + (``redis.asyncio.cluster.RedisCluster``) declares connection settings such as + ``decode_responses`` on its own constructor, where the sync class takes them + through ``**kwargs`` and so never names them in its signature. Introspecting + only the sync class regardless of which client is actually built silently + drops those for every async cluster caller. + """ if client is None: - client = redis.Redis.from_url - arg_spec: Final = inspect.getfullargspec(redis.RedisCluster) + client = redis.RedisCluster # Only allow primitive arguments exclude_args: Final = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"} - available_args = {x for x in arg_spec.args if x not in exclude_args} + available_args = {x for x in _unwrapped_init_args(client) if x not in exclude_args} available_args |= { "password", "username", @@ -161,6 +186,79 @@ def _get_redis_env_kwarg_mapping(): return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment} +def _str_to_bool(value: str) -> bool: + return value.lower() in ("true", "1", "yes") + + +def _coerce_redis_kwargs_types( + redis_kwargs: Mapping[str, object], + client: type | tuple[type, ...] = redis.Redis, +) -> dict[str, object]: # mutable-ok: a caller mutates the returned kwargs before constructing its client + """Coerces string values to the numeric/boolean type ``client``'s constructor + declares for that parameter. ``client`` may be a tuple of client classes; a + parameter's type is taken from the first signature that declares it, which + lets cluster callers coerce cluster-only kwargs such as + ``cluster_error_retry_attempts`` alongside the shared connection kwargs. + + Environment variables are always strings, and Helm ``--set`` stringifies values + too, so a config value like ``health_check_interval`` or ``socket_timeout`` + can arrive as ``"30"``/``"5.5"`` rather than a real number. redis-py's own + connection-health-check arithmetic (``loop.time() + self.health_check_interval``) + then raises ``TypeError`` on every Redis operation instead of connecting. + + ``max_connections``, ``socket_timeout``, and ``socket_connect_timeout`` use an + explicit target type rather than the parameter's own signature default: redis-py + 8.x changed the timeout defaults from ``None`` to int ``5``, so inferring the + type from the default would make a fractional ``"5.5"`` fail ``int()`` and get + silently dropped on 8.x while working on older versions. ``socket_keepalive`` + is explicit too: its signature default is ``None``, which carries no type to + infer from, and leaving it a string makes ``"false"`` truthy. + """ + signatures: Final = tuple(inspect.signature(c) for c in (client if isinstance(client, tuple) else (client,))) + explicit_param_types: Final = MappingProxyType( + { + "max_connections": int, + "socket_timeout": float, + "socket_connect_timeout": float, + "socket_keepalive": bool, + } + ) + result: Final = dict(redis_kwargs) # mutable-ok: per-key try/except coercion below needs to drop individual keys + for key, value in redis_kwargs.items(): + if not isinstance(value, str): + continue + param = next((sig.parameters[key] for sig in signatures if key in sig.parameters), None) + if param is None: + continue + explicit_type = explicit_param_types.get(key) + if explicit_type is bool: + result[key] = _str_to_bool(value) + continue + if explicit_type is not None: + try: + result[key] = explicit_type(value) + except (ValueError, TypeError): + del result[key] + continue + default: object = param.default # pyright: ignore[reportAny] # inspect.Parameter.default is stubbed as Any + if default is inspect.Parameter.empty: + continue + # bool must be checked before int, since bool subclasses int + if isinstance(default, bool): + result[key] = _str_to_bool(value) + elif isinstance(default, int): + try: + result[key] = int(value) + except (ValueError, TypeError): + del result[key] + elif isinstance(default, float): + try: + result[key] = float(value) + except (ValueError, TypeError): + del result[key] + return result + + def _redis_kwargs_from_environment(): mapping: Final = _get_redis_env_kwarg_mapping() @@ -505,7 +603,12 @@ def _get_redis_client_logic(**env_overrides): raise ValueError("Either 'host' or 'url' must be specified for redis.") # litellm.print_verbose(f"redis_kwargs: {redis_kwargs}") - return redis_kwargs + coercion_client: Final = ( + (redis.Redis, redis.RedisCluster, async_redis.RedisCluster) + if redis_kwargs.get("startup_nodes") + else redis.Redis + ) + return _coerce_redis_kwargs_types(redis_kwargs, client=coercion_client) def init_redis_cluster(redis_kwargs) -> redis.RedisCluster: @@ -657,7 +760,9 @@ def get_redis_client(**env_overrides): if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: return _init_redis_sentinel(redis_kwargs) - return redis.Redis(**redis_kwargs) + return redis.Redis( # pyright: ignore[reportCallIssue] # object-valued kwargs match no overload statically + **redis_kwargs, # pyright: ignore[reportArgumentType] # allow-listed and coerced against this signature + ) def get_redis_async_client( @@ -669,7 +774,7 @@ def get_redis_async_client( if "startup_nodes" in redis_kwargs: from redis.cluster import ClusterNode - args = _get_redis_cluster_kwargs() + args = _get_redis_cluster_kwargs(async_redis.RedisCluster) cluster_kwargs: Final = {} for arg in redis_kwargs: if arg in args: diff --git a/litellm/constants.py b/litellm/constants.py index 9872783bfab..c482ab0e39a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -484,6 +484,22 @@ FIREWORKS_AI_80_B: Final = int(os.getenv("FIREWORKS_AI_80_B", 80)) #### Logging callback constants #### REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM" MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = int(os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)) +# Backpressure + lifetime bounds for the /v1/messages streaming relay (see +# BaseAnthropicMessagesStreamingIterator.async_sse_wrapper). The relay queue is +# bounded so a slow client throttles the upstream pump instead of letting it +# buffer the whole response in memory; the detached-drain cap bounds how many +# post-disconnect drains may run concurrently so client behavior can't create +# unbounded worker state. +ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = int( + os.getenv("ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "1024") +) +# Setting this to 0 disables detached draining entirely: every post-disconnect +# pump bills whatever partial output it has already collected and aborts the +# upstream stream immediately, instead of continuing to drain for the real +# terminal usage. +ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int( + os.getenv("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", "100") +) LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0 LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000)) LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)) @@ -806,6 +822,7 @@ openai_compatible_endpoints: Final[list] = [ "https://api.meta.ai/v1", "https://api.cognition.ai/v1", "https://api.scx.ai/v1", + "https://gigachat.devices.sberbank.ru/api/v1", ] @@ -1410,6 +1427,12 @@ DEFAULT_SOFT_BUDGET: Final = float( ) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash" +# Prefix of the 401 raised when a submitted virtual key is not shaped like one. +INVALID_VIRTUAL_KEY_ERROR_MESSAGE: Final = "LiteLLM Virtual Key expected" +# Attribute stamped on that 401 at its raise site so log routing recognises it by +# provenance. Message text is caller-influenceable on other 401s, so it must not +# be used to classify. +INVALID_VIRTUAL_KEY_ERROR_MARKER: Final = "_litellm_invalid_virtual_key_error" # Python garbage collection threshold configuration # Format: "gen0,gen1,gen2" e.g., "1000,50,50" diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index fb66edbf272..9b757ce86be 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -1,10 +1,14 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +from typing_extensions import NotRequired, ReadOnly, TypedDict + from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS if TYPE_CHECKING: from litellm import Logging as LiteLLMLoggingObj - from litellm.types.llms.openai import HttpxBinaryResponseContent + from litellm.types.llms.openai import ChatCompletionUserMessage, HttpxBinaryResponseContent from litellm.types.utils import ModelResponse @@ -16,7 +20,42 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None: return response_cost if isinstance(response_cost, float) else None +GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16" + + +class ChatAudioParam(TypedDict): + voice: ReadOnly[str] + format: ReadOnly[NotRequired[str]] + + class SpeechToCompletionBridgeTransformationHandler: + def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + { + param: value + for param, value in optional_params.items() + if param in OPENAI_CHAT_COMPLETION_PARAMS and param != "response_format" + } + ) + + def _chat_audio_format(self, model: str, optional_params: Mapping[str, object]) -> str | None: + if self._is_gemini_tts_model(model): + return GEMINI_TTS_CHAT_AUDIO_FORMAT + response_format: Final = optional_params.get("response_format") + return response_format if isinstance(response_format, str) else None + + def _chat_audio_param( + self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object] + ) -> ChatAudioParam | None: + if not isinstance(voice, str): + return None + audio_format: Final = self._chat_audio_format(model, optional_params) + if audio_format is None: + voice_only: Final[ChatAudioParam] = {"voice": voice} + return voice_only + audio: Final[ChatAudioParam] = {"voice": voice, "format": audio_format} + return audio + def transform_request( self, model: str, @@ -28,36 +67,19 @@ class SpeechToCompletionBridgeTransformationHandler: litellm_logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, ) -> dict: - passed_optional_params: Final = {} - for op in optional_params: - if op in OPENAI_CHAT_COMPLETION_PARAMS: - passed_optional_params[op] = optional_params[op] - - if voice is not None: - if isinstance(voice, str): - passed_optional_params["audio"] = {"voice": voice} - if "response_format" in optional_params: - passed_optional_params["audio"]["format"] = optional_params["response_format"] - - return_kwargs = { + user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input} + return_kwargs: Final = { "model": model, - "messages": [ - { - "role": "user", - "content": input, - } - ], + "messages": [user_message], "modalities": ["audio"], - **passed_optional_params, + **self._chat_completion_params(optional_params), + "audio": self._chat_audio_param(model, voice, optional_params), **litellm_params, "headers": headers, "litellm_logging_obj": litellm_logging_obj, "custom_llm_provider": custom_llm_provider, } - - # filter out None values - return_kwargs = {k: v for k, v in return_kwargs.items() if v is not None} - return return_kwargs + return {k: v for k, v in return_kwargs.items() if v is not None} def _convert_pcm16_to_wav(self, pcm_data: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes: """ diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index b73a1443330..8ea19deef4c 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -108,7 +108,6 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper): def __init__(self, completion_stream: object): self.sent_first_chunk = False - # State tracking for accumulating partial tool calls self.accumulated_tool_calls = dict[int, _ToolCallAccumulator]() self._returned_response = False super().__init__(completion_stream) diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index b09498f9292..3ac92b04c27 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -62,6 +62,8 @@ class GenAIMapper: GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds, GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens, GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens, + GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens, + GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens, Error.TYPE: lambda d: d.error.error_type if d.error else None, Server.ADDRESS: lambda d: d.server.address if d.server else None, Server.PORT: lambda d: d.server.port if d.server else None, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index f70c777e1a7..d35405538f6 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -95,6 +95,22 @@ class LLMUsage: input_tokens: int | None = None output_tokens: int | None = None total_tokens: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + + @classmethod + def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage: + # Cache token counts only exist on the raw provider usage object under metadata + metadata: Final[Mapping[str, object]] = payload.get("metadata") or {} + raw_usage: Final = metadata.get("usage_object") + usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {} + return cls( + input_tokens=as_int(payload.get("prompt_tokens")), + output_tokens=as_int(payload.get("completion_tokens")), + total_tokens=as_int(payload.get("total_tokens")), + cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")), + cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")), + ) @dataclass(frozen=True) @@ -363,11 +379,7 @@ class LLMCallSpanData: response_model=context.response_model, response_id=as_str(response.get("id")), request_params=LLMRequestParams.from_model_parameters(params), - usage=LLMUsage( - input_tokens=as_int(payload.get("prompt_tokens")), - output_tokens=as_int(payload.get("completion_tokens")), - total_tokens=as_int(payload.get("total_tokens")), - ), + usage=LLMUsage.from_standard_logging_payload(payload), finish_reasons=finish_reasons, error=_parse_error(payload), response_cost=as_float(payload.get("response_cost")), diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 4ad0cb5d1b4..f7a6280f95b 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -110,6 +110,8 @@ class GenAI: # usage USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens" + USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens" + USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens" # content (opt-in, gated by capture mode) INPUT_MESSAGES: Final = "gen_ai.input.messages" OUTPUT_MESSAGES: Final = "gen_ai.output.messages" diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index cf8aa38d86e..18bda0a9d55 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -592,7 +592,12 @@ _JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs" class ShadowEvalLogger(CustomLogger): - """Fires blind pairwise shadow evaluations for keys with an active shadow-eval job.""" + """Fires blind pairwise shadow evaluations for targets with an active shadow-eval job. + + A job targets a virtual key, a team, or a user; a request qualifies for a job when + any of its resolved identities (key hash, team id, user id) matches the job's + target, so team and user jobs cover JWT-authenticated traffic, which carries no + key hash at all.""" def __init__( self, @@ -617,10 +622,10 @@ class ShadowEvalLogger(CustomLogger): # generation; the refill absorbs written rows and resets. self._job_starts: dict[str, int] = {} # mutable-ok: per-generation counter - async def _active_jobs(self) -> Mapping[str, tuple[ActiveShadowEvalJob, ...]]: - """Active jobs by api_key_id, cache-first. A key holds at most one job per - direction, so the value is a collection. A DB fault returns empty without - caching, so sampling pauses for that request and the next one retries.""" + async def _active_jobs(self) -> Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]: + """Active jobs by (target_type, target_id), cache-first. A target holds at most + one job per direction, so the value is a collection. A DB fault returns empty + without caching, so sampling pauses for that request and the next one retries.""" cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY) if cached is not None: return cached # pyright: ignore[reportReturnType] # cache stores exactly this mapping shape @@ -652,10 +657,10 @@ class ShadowEvalLogger(CustomLogger): ) for row in grouped or [] } - by_key: Final = tuple( + by_target: Final = tuple( sorted( ( - (str(record.api_key_id), job) + ((str(record.target_type), str(record.target_id)), job) for record in records or [] if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None ), @@ -663,7 +668,7 @@ class ShadowEvalLogger(CustomLogger): ) ) jobs: Final = MappingProxyType( - {key: tuple(job for _, job in group) for key, group in groupby(by_key, key=itemgetter(0))} + {target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))} ) await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs) self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill @@ -720,8 +725,18 @@ class ShadowEvalLogger(CustomLogger): if should_redact_message_logging(dict(kwargs)): # mutable-ok: predicate takes a plain dict return metadata: Final = payload.get("metadata") or _EMPTY_METADATA - api_key_hash: Final = metadata.get("user_api_key_hash") - if not api_key_hash: + # Each identity the request resolved to is a candidate target; JWT-auth + # requests carry no key hash but do carry a team and user. + targets: Final = tuple( + (target_type, str(value)) + for target_type, value in ( + ("key", metadata.get("user_api_key_hash")), + ("team", metadata.get("user_api_key_team_id")), + ("user", metadata.get("user_api_key_user_id")), + ) + if value + ) + if not targets: return request_id: Final = payload.get("id") or "" if not request_id: @@ -731,8 +746,11 @@ class ShadowEvalLogger(CustomLogger): return # only surfaces this table can normalize are comparable; unknown types fail closed if ops.wire_params and _request_mutating_guardrail_ran(request_metadata): return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content + active_jobs: Final = await self._active_jobs() eligible: Final = self._sampled_jobs( - (await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id + tuple(job for target in targets for job in active_jobs.get(target, ())), + request_metadata, + request_id, ) if not eligible: return @@ -1056,7 +1074,7 @@ class ShadowEvalLogger(CustomLogger): ) -_EMPTY_JOBS: Final[Mapping[str, tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({}) +_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({}) def _default_prisma_provider() -> "PrismaClient | None": diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index b12c715c9f5..389e6f7f501 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -50,6 +50,9 @@ OPTIONAL_KWARGS_KEYS: Final = ( "vertex_ai_project", "vertex_ai_location", "vertex_ai_credentials", + "gigachat_scope", + "gigachat_auth_url", + "gigachat_access_token", "tpm", "rpm", "itpm", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 74a1d3e5008..9b53b79bbe6 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -369,6 +369,9 @@ def get_llm_provider( elif endpoint == "https://api.meta.ai/v1": custom_llm_provider = "meta" dynamic_api_key = get_secret_str("META_API_KEY") + elif endpoint == "https://gigachat.devices.sberbank.ru/api/v1": + custom_llm_provider = "gigachat" + dynamic_api_key = get_secret_str("GIGACHAT_API_KEY") elif (json_provider := JSONProviderRegistry.get_by_base_url(endpoint)) is not None: custom_llm_provider = json_provider.slug dynamic_api_key = api_key if api_key is not None else get_secret_str(json_provider.api_key_env) @@ -867,6 +870,9 @@ def _get_openai_compatible_provider_info( # Manus is OpenAI compatible for responses API api_base = api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im" dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY") + elif custom_llm_provider == "gigachat": + api_base = api_base or get_secret_str("GIGACHAT_API_BASE") or "https://gigachat.devices.sberbank.ru/api/v1" + dynamic_api_key = api_key or get_secret_str("GIGACHAT_API_KEY") if api_base is not None and not isinstance(api_base, str): raise Exception(f"api base needs to be a string. api_base={api_base}") diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 97c4d038734..350e5403e4c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2141,6 +2141,9 @@ class Logging(LiteLLMLoggingBaseClass): logging_result: Final = self.normalize_logging_result(result=result) + if isinstance(result, Response) and isinstance(logging_result, (ModelResponse, EmbeddingResponse)): + result = logging_result + if standard_logging_object is None and result is not None and self.stream is not True: if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance( logging_result, (dict, list) @@ -6152,7 +6155,10 @@ def get_standard_logging_object_payload( def emit_standard_logging_payload(payload: StandardLoggingPayload): if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"): - print(json.dumps(payload, indent=4), flush=True) # noqa: T201 + try: + print(json.dumps(payload, indent=4, default=str), flush=True) # noqa: T201 + except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging + verbose_logger.exception("Error serializing standard logging payload for debug output: %s", e) def get_standard_logging_metadata( diff --git a/litellm/litellm_core_utils/model_response_utils.py b/litellm/litellm_core_utils/model_response_utils.py index 7bf667164ae..dc4f375daa7 100644 --- a/litellm/litellm_core_utils/model_response_utils.py +++ b/litellm/litellm_core_utils/model_response_utils.py @@ -144,7 +144,6 @@ def _is_choice_non_empty(choice: StreamingChoices) -> bool: # Check model_extra for dynamically added fields on the choice choice_extra_fields: Final[Mapping[str, object]] = choice.model_extra or {} for extra_field_name, extra_field_value in choice_extra_fields.items(): - # Skip certain structural fields that are just default/None placeholders if extra_field_name == "index" and extra_field_value == 0: continue if extra_field_name in {"finish_reason", "logprobs"} and extra_field_value is None: @@ -192,7 +191,6 @@ def _is_delta_non_empty(delta: Delta) -> bool: # Check model_extra for dynamically added fields (this is where Pydantic stores them) delta_extra_fields: Final[Mapping[str, object]] = delta.model_extra or {} for extra_field_value in delta_extra_fields.values(): - # Even structural fields are meaningful if they have actual content if _has_meaningful_content(extra_field_value): return True diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 1c8f10d3307..bda365102af 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -205,6 +205,41 @@ def is_non_content_values_set(message: AllMessageValues) -> bool: return any(message.get(key, None) is not None for key in message if key not in ignore_keys) +_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"}) +_IMAGE_SCAN_MAX_DEPTH: Final = 4 + + +def _content_parts_contain_image(parts: Sequence[object]) -> bool: + """Depth-bounded frontier walk over nested content lists, iterative because the repo bans + recursion; an Anthropic tool_result nests its image parts exactly one level down.""" + frontier = parts # rebind-ok: depth-bounded frontier walk + for _ in range(_IMAGE_SCAN_MAX_DEPTH): + if any(isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_PART_TYPES for part in frontier): + return True + frontier = tuple( # rebind-ok: depth-bounded frontier walk + nested + for part in frontier + if isinstance(part, Mapping) + for content in (part.get("content"),) + if isinstance(content, list) + for nested in content + ) + if not frontier: + return False + return False + + +def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool: + """Whether any message carries an image content part, across the dialects that reach + pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``, + and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks.""" + return any( + isinstance(content, list) and _content_parts_contain_image(content) + for message in messages + for content in (message.get("content"),) + ) + + def _audio_or_image_in_message_content(message: AllMessageValues) -> bool: """ Checks if message content contains an image or audio diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 256bee7b348..c350bc5569e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -693,6 +693,26 @@ def _count_document_tokens( ) +def _count_file_tokens( + file_value: object, + count_function: TokenCounterFunction, + use_default_image_token_count: bool, +) -> int: + """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" + if not isinstance(file_value, Mapping): + return 0 + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 + if not isinstance(file_data, str) or not file_data: + return name_tokens + return name_tokens + calculate_img_tokens( + data=file_data, + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + + def _count_anthropic_content( content: Mapping[str, Any], count_function: TokenCounterFunction, @@ -778,6 +798,12 @@ def _count_content_list( use_default_image_token_count, default_token_count, ) + elif c["type"] == "file": + num_tokens += _count_file_tokens( + c.get("file"), + count_function, + use_default_image_token_count, + ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( c, @@ -807,7 +833,7 @@ def _count_content_list( raise ValueError( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " - f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)." + f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index eb6dbd3e5b3..9395cc3d33e 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -859,12 +859,28 @@ class AnthropicMessagesHandler(BaseTranslation): @staticmethod def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]: + """Normalize an Anthropic image block into strings a guardrail can read. + + base64 becomes a data URI so the format travels with the payload, which is what + the OpenAI path already puts in this field. A file source yields nothing: those + bytes live behind the Files API and this extractor has no client to fetch them. + """ source: Final = block.get("source") if not isinstance(source, Mapping): return () - # Could be base64 or url + + source_type: Final = source.get("type") + if source_type == "url": + url: Final = source.get("url") + return (url,) if isinstance(url, str) and url else () + data: Final = source.get("data") - return (data,) if data else () + if not isinstance(data, str) or not data: + return () + media_type: Final = source.get("media_type") + if isinstance(media_type, str) and media_type: + return (f"data:{media_type};base64,{data}",) + return (data,) async def _apply_guardrail_responses_to_input( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index a282d5f4d4f..45c7825344b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -8,6 +8,10 @@ import httpx from pydantic import TypeAdapter from typing_extensions import TypedDict +from litellm.constants import ( + ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, + ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE, +) from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -21,6 +25,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() +_UPSTREAM_PUMP_TASKS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: stdlib strong-ref set for pump tasks +_DETACHED_STREAM_DRAINS: Final[set[asyncio.Task[None]]] = set() # mutable-ok: bounded strong-ref set, detached drains + INCOMPLETE_STREAM_ERROR_MESSAGE: Final = ( "Provider stream ended before emitting a message_stop event; " "the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated." @@ -133,6 +140,34 @@ def _is_terminal_stream_chunk(chunk: object) -> bool: return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk) +def _try_claim_detached_drain_slot() -> bool: + """Claim a detached-drain slot for the current task, bounding concurrency. + + Returns True if a slot was claimed (the caller may keep draining upstream + for billing) or False if the cap is already reached (the caller should stop + and bill what it has). Only touched from the event loop, so the check + + insert need no lock. + """ + if len(_DETACHED_STREAM_DRAINS) >= ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: + return False + current_task: Final = asyncio.current_task() + if current_task is not None: + _DETACHED_STREAM_DRAINS.add(current_task) + current_task.add_done_callback(_DETACHED_STREAM_DRAINS.discard) + return True + + +def _exception_left_unconsumed(queue: "asyncio.Queue[bytes | None | BaseException]", exc: BaseException) -> bool: + """After client detach the relay never reads the queue again, so drain it here. + + The forwarded exception still sitting in the queue means the relay tore + down before re-raising it, so the proxy's failure handling never ran and + the caller must salvage spend itself. + """ + remaining: Final = tuple(queue.get_nowait() for _ in range(queue.qsize())) + return any(item is exc for item in remaining) + + def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes: return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() @@ -414,17 +449,167 @@ class BaseAnthropicMessagesStreamingIterator: async def async_sse_wrapper( self, - completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | dict], + completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]], ) -> AsyncIterator[bytes]: """ Generic async SSE wrapper that converts streaming chunks to SSE format and handles logging. + The upstream read runs in a detached background task (``_pump_upstream``) + so that a client disconnect tears down only this client-facing generator, + never the upstream drain + billing. The provider (e.g. Bedrock) keeps + generating and billing the full response regardless of the client, so + draining it to completion is what lets spend tracking see the real + terminal ``message_delta`` / ``message_stop`` usage instead of a + truncated placeholder count. + + Chunks reach the client through a bounded queue. While the client is + connected the pump blocks on a full queue (racing the disconnect + signal), so a slow reader throttles the upstream read exactly as the old + direct ``yield`` did instead of letting the whole response buffer in + memory. Once the client goes away the pump stops enqueueing and only + keeps a single ``collected_chunks`` copy for billing, and the number of + such post-disconnect drains running at once is capped so client behavior + can't create unbounded worker state; over the cap the pump bills what it + has rather than draining further. Detached-drain lifetime is otherwise + bounded by the upstream stream/read timeout. + + An upstream failure (Bedrock read / decode / chunk-conversion error) + that happens while the client is still connected is forwarded through + the queue and re-raised here, so the original provider exception (and + its status) reaches the proxy's failure handling unchanged rather than + being masked by a generic incomplete-stream event. + This method provides the common logic for both Anthropic and Bedrock implementations. """ - collected_chunks: Final = [] - saw_terminal_event = False + queue: Final[asyncio.Queue[bytes | None | BaseException]] = asyncio.Queue( + maxsize=ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE + ) + client_detached: Final = asyncio.Event() + pump_task: Final = asyncio.create_task(self._pump_upstream_to_queue(completion_stream, queue, client_detached)) + _UPSTREAM_PUMP_TASKS.add(pump_task) + pump_task.add_done_callback(_UPSTREAM_PUMP_TASKS.discard) + + reached_end = False # rebind-ok: flipped once the relay consumes the end-of-stream sentinel + try: + while True: + item = await queue.get() + if item is None: + reached_end = True + break + if isinstance(item, BaseException): + raise item + yield item + finally: + client_detached.set() + if not reached_end: + self._dispatch_pending_deferred_logging() + + def _dispatch_pending_deferred_logging(self) -> None: + """Fire deferred billing that a torn-down response would otherwise drop. + + When the pump finishes draining while the client is still connected it + stores the logging coroutine for ProxyLogging._fire_deferred_stream_logging, + which the proxy only fires on a normally completed response: a client + disconnect (GeneratorExit / CancelledError) re-raises past it. Without + this dispatch that window loses the spend row entirely. + """ + deferred_cb: Final = getattr(self.litellm_logging_obj, "_on_deferred_stream_complete", None) + deferred_args: Final = getattr(self.litellm_logging_obj, "_deferred_stream_complete_args", None) + if deferred_cb is None or deferred_args is None: + return + self.litellm_logging_obj._on_deferred_stream_complete = None + self.litellm_logging_obj._deferred_stream_complete_args = None + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=deferred_cb(*deferred_args)) + + async def _bill_collected_chunks( + self, + collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _handle_streaming_logging + *, + stream_teardown: bool, + ) -> None: + from litellm._logging import verbose_proxy_logger + + try: + await self._handle_streaming_logging(collected_chunks, stream_teardown=stream_teardown) + except Exception as exc: # noqa: BLE001 # billing is best-effort; never crash the pump + verbose_proxy_logger.warning( + "async_sse_wrapper billing failed after %d chunks: %s(%s)", + len(collected_chunks), + type(exc).__name__, + exc, + ) + + @staticmethod + async def _abort_upstream( + completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]], + ) -> None: + """Close the upstream provider stream so it stops generating and billing.""" + from litellm._logging import verbose_proxy_logger + + try: + await aclose_if_supported(completion_stream) + except Exception as exc: # noqa: BLE001 # abort is best-effort; log and continue + verbose_proxy_logger.warning( + "async_sse_wrapper failed to abort upstream stream: %s(%s)", + type(exc).__name__, + exc, + ) + + @staticmethod + async def _enqueue_for_client( + queue: "asyncio.Queue[bytes | None | BaseException]", + client_detached: "asyncio.Event", + item: bytes | None | BaseException, + ) -> bool: + """Deliver one item to the client, applying backpressure. + + Returns True if the item was queued, False if the client disconnected + before there was room (the item is then dropped, since a gone client + can't receive it). Never blocks once the client has detached. + """ + if client_detached.is_set(): + return False + try: + queue.put_nowait(item) + except asyncio.QueueFull: + pass + else: + return True + put_task: Final = asyncio.ensure_future(queue.put(item)) + detached_task: Final = asyncio.ensure_future(client_detached.wait()) + try: + await asyncio.wait(frozenset((put_task, detached_task)), return_when=asyncio.FIRST_COMPLETED) + finally: + if not detached_task.done(): + detached_task.cancel() + if put_task.done() and not put_task.cancelled(): + return True + put_task.cancel() + return False + + async def _pump_upstream_to_queue( + self, + completion_stream: AsyncIterator[bytes | GenericStreamingChunk | ModelResponseStream | Mapping[str, object]], + queue: "asyncio.Queue[bytes | None | BaseException]", + client_detached: "asyncio.Event", + ) -> None: + """Drain the whole upstream into ``queue`` (backpressured) and bill once. + + Runs detached so a client disconnect can't interrupt the upstream read; + see ``async_sse_wrapper`` for the full rationale. On a completed drain + the success billing (or deferred park) happens before the end-of-stream + sentinel is enqueued: the relay can only tear down after consuming the + sentinel, so its teardown can never outrun the park and get mistaken + for a client disconnect, and a sentinel the client never consumes falls + back to dispatching the parked billing here. + """ + from litellm._logging import verbose_proxy_logger + + collected_chunks: Final[list[bytes]] = [] # mutable-ok: SSE billing buffer appended to across the drain + saw_terminal_event = False # rebind-ok: accumulates across the upstream loop + draining_detached = False # rebind-ok: set once this pump claims a detached-drain slot try: async for chunk in completion_stream: if self.completion_start_time is None: @@ -432,17 +617,62 @@ class BaseAnthropicMessagesStreamingIterator: saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) encoded_chunk = self._convert_chunk_to_sse_format(chunk) collected_chunks.append(encoded_chunk) - yield encoded_chunk - except (GeneratorExit, asyncio.CancelledError): - # A client disconnect tears the generator down at the yield, so the - # post-loop logging below never runs and the tokens already streamed - # (and billed by the provider) would never reach spend tracking. See LIT-5839. - if collected_chunks: - await self._handle_streaming_logging(collected_chunks, stream_teardown=True) - raise + if not client_detached.is_set(): + await self._enqueue_for_client(queue, client_detached, encoded_chunk) + continue + if not draining_detached: + if not _try_claim_detached_drain_slot(): + verbose_proxy_logger.warning( + "async_sse_wrapper: detached-drain cap (%d) reached; billing %d partial " + "chunks and aborting the upstream stream to stop provider billing", + ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS, + len(collected_chunks), + ) + await self._bill_collected_chunks(collected_chunks, stream_teardown=True) + await self._abort_upstream(completion_stream) + return + draining_detached = True + except Exception as exc: # noqa: BLE001 # upstream errors are handled/forwarded by _handle_pump_upstream_error + await self._handle_pump_upstream_error(queue, client_detached, collected_chunks, exc) + return - if not saw_terminal_event: - yield _incomplete_stream_error_sse_event() + if client_detached.is_set(): + await self._bill_collected_chunks(collected_chunks, stream_teardown=True) + return + if not saw_terminal_event and not await self._enqueue_for_client( + queue, client_detached, _incomplete_stream_error_sse_event() + ): + await self._bill_collected_chunks(collected_chunks, stream_teardown=True) + return + await self._bill_collected_chunks(collected_chunks, stream_teardown=False) + if not await self._enqueue_for_client(queue, client_detached, None): + self._dispatch_pending_deferred_logging() - # Handle logging after all chunks are processed - await self._handle_streaming_logging(collected_chunks) + async def _handle_pump_upstream_error( + self, + queue: "asyncio.Queue[bytes | None | BaseException]", + client_detached: "asyncio.Event", + collected_chunks: list[bytes], # mutable-ok: SSE buffer forwarded to list-typed _bill_collected_chunks + exc: BaseException, + ) -> None: + """Forward a provider error to a still-connected client, else salvage partial spend. + + Handing the original exception to the client-facing generator lets it + re-raise so the proxy's failure handling keeps the provider status and + owns logging (no success-bill). If the client already went away, or + disconnects before ever consuming the queued exception, no failure hook + runs, so bill the partial instead of dropping the request. + """ + from litellm._logging import verbose_proxy_logger + + if not client_detached.is_set() and await self._enqueue_for_client(queue, client_detached, exc): + await client_detached.wait() + if not _exception_left_unconsumed(queue, exc): + return + verbose_proxy_logger.warning( + "async_sse_wrapper upstream pump failed after client disconnect (%d chunks): %s(%s)", + len(collected_chunks), + type(exc).__name__, + exc, + ) + await self._bill_collected_chunks(collected_chunks, stream_teardown=True) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index db9c8a5cedd..395d99a4caa 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1631,6 +1631,11 @@ class AmazonConverseConfig(BaseConfig): bedrock_tool_config["toolChoice"] = tool_choice_values self._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params) + config_block_entries: Final = tuple( + (config_name, config_class, inference_params.pop(config_name, None)) + for config_name, config_class in self.get_config_blocks().items() + ) + data: Final[CommonRequestObject] = { "inferenceConfig": self._transform_inference_params(inference_params=inference_params), } @@ -1641,9 +1646,7 @@ class AmazonConverseConfig(BaseConfig): if system_content_blocks: data["system"] = system_content_blocks - # Handle all config blocks - for config_name, config_class in self.get_config_blocks().items(): - config_value = inference_params.pop(config_name, None) + for config_name, config_class, config_value in config_block_entries: if config_value is not None: data[config_name] = config_class(**config_value) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 3bda8dd8359..42fe8941443 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -7,13 +7,18 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic. import asyncio import contextlib import json +from collections.abc import AsyncIterator, Mapping from typing import Final, Protocol from pydantic import JsonValue, TypeAdapter +import litellm from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes +from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput from ..base_aws_llm import BaseAWSLLM @@ -32,6 +37,17 @@ def _json_str(value: JsonValue) -> str | None: return value if isinstance(value, str) else None +def _should_log_event(openai_message: Mapping[str, object]) -> bool: + logged_types: Final = ( + litellm.logged_real_time_event_types + if litellm.logged_real_time_event_types is not None + else DefaultLoggedRealTimeEventTypes + ) + if logged_types == "*": + return True + return openai_message.get("type") in logged_types + + class RealtimeClientWebSocket(Protocol): """The client-facing websocket surface the realtime bridge talks to.""" @@ -205,16 +221,22 @@ class BedrockRealtime(BaseAWSLLM): ) ) - bedrock_to_client_task: Final = asyncio.create_task( - self._forward_bedrock_to_client( - bedrock_stream, - websocket, - transformation_config, - model, - logging_obj, - session_state, + async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]: + return tuple( + [ + event + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + ) + ] ) - ) + + bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events()) # Wait for both tasks to complete await asyncio.gather( @@ -223,6 +245,27 @@ class BedrockRealtime(BaseAWSLLM): return_exceptions=True, ) + forwarded_logged_events: Final = ( + bedrock_to_client_task.result() + if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None + else () + ) + logged_events: Final = ( + *forwarded_logged_events, + *( + leftover_event + for leftover_event in transformation_config.leftover_usage_done_events() + if _should_log_event(leftover_event) + ), + ) + if logged_events: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + logging_obj.dispatch_success_handlers( + list(logged_events), # mutable-ok: realtime spend logging requires a list result + prefer_async_handlers=True, + ) + ) + except Exception as e: verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e) try: @@ -304,8 +347,8 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, - ): - """Forward messages from Bedrock stream to client WebSocket.""" + ) -> AsyncIterator[OpenAIRealtimeEvents]: + """Forward messages from Bedrock to the client, yielding the ones to record for spend logging.""" try: while True: # Receive from Bedrock @@ -353,11 +396,14 @@ class BedrockRealtime(BaseAWSLLM): ) # Send transformed messages to client - openai_messages = transformed_response.get("response", []) + response_value = transformed_response["response"] + openai_messages = response_value if isinstance(response_value, list) else (response_value,) for openai_message in openai_messages: message_json = json.dumps(openai_message) await client_ws.send_text(message_json) verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200]) + if _should_log_event(openai_message): + yield openai_message except Exception as e: verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True) diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 951bf636b2f..28c2e446d10 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -7,7 +7,7 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format. import base64 import json import uuid as uuid_lib -from typing import Any, Final +from typing import Any, Final, cast from pydantic import BaseModel @@ -20,29 +20,54 @@ from litellm.types.llms.openai import ( OpenAIRealtimeContentPartDone, OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, + OpenAIRealtimeInputAudioBufferSpeechEvent, + OpenAIRealtimeInputAudioTranscriptionCompleted, + OpenAIRealtimeInputAudioTranscriptionDelta, OpenAIRealtimeOutputItemDone, OpenAIRealtimeResponseAudioDone, OpenAIRealtimeResponseContentPartAdded, OpenAIRealtimeResponseDelta, OpenAIRealtimeResponseDoneObject, OpenAIRealtimeResponseTextDone, + OpenAIRealtimeResponseUsage, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamResponseOutputItemAdded, OpenAIRealtimeStreamSession, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeUsageTokenDetails, ) from litellm.types.realtime import ( ALL_DELTA_TYPES, RealtimeResponseTransformInput, RealtimeResponseTypedDict, ) -from litellm.utils import get_empty_usage class BedrockContentEnd(BaseModel): stopReason: str | None = None +class BedrockUsageTokenDetails(BaseModel): + speechTokens: int = 0 + textTokens: int = 0 + + +class BedrockUsageDetailsTotal(BaseModel): + input: BedrockUsageTokenDetails = BedrockUsageTokenDetails() + output: BedrockUsageTokenDetails = BedrockUsageTokenDetails() + + +class BedrockUsageDetails(BaseModel): + total: BedrockUsageDetailsTotal = BedrockUsageDetailsTotal() + + +class BedrockUsageEvent(BaseModel): + totalInputTokens: int = 0 + totalOutputTokens: int = 0 + totalTokens: int = 0 + details: BedrockUsageDetails = BedrockUsageDetails() + + TRIGGER_AUDIO_SAMPLE_RATE_HERTZ: Final = 16000 TRIGGER_AUDIO_BYTES_PER_SECOND: Final = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2 TRIGGER_LEADING_SILENCE: Final = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2) @@ -87,6 +112,15 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Text configuration self.text_media_type = "text/plain" + # Response-stream state (Bedrock events carry no role on textOutput, + # so the USER/ASSISTANT split from contentStart is tracked here) + self._user_transcript_active = False + self._user_transcript_generation_stage: str | None = None + self._user_item_id: str | None = None + self._user_transcript_buffer = "" + self._cumulative_usage = BedrockUsageEvent() + self._reported_usage = BedrockUsageEvent() + def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict: """Validate environment - no special validation needed for Bedrock.""" return headers @@ -691,6 +725,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): role: Final = content_start.get("role") if role != "ASSISTANT": + if role == "USER" and content_start.get("type") == "TEXT": + self._user_transcript_active = True + self._user_transcript_generation_stage = self._parse_generation_stage( + content_start.get("additionalModelFields") + ) return ( [], current_response_id, @@ -700,6 +739,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) verbose_logger.debug("Handling ASSISTANT contentStart") + is_new_response: Final = current_response_id is None # Initialize IDs if needed if not current_response_id: @@ -715,7 +755,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): returned_messages: Final[list[OpenAIRealtimeEvents]] = [] - # Send response.created + # Send response.created only once per response (a response can contain + # multiple content blocks, e.g. TEXT then AUDIO) response_created: Final = OpenAIRealtimeStreamResponseBaseObject( type="response.created", event_id=f"event_{uuid.uuid4()}", @@ -727,7 +768,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): "conversation_id": current_conversation_id, }, ) - returned_messages.append(response_created) + if is_new_response: + returned_messages.append(response_created) # Send response.output_item.added output_item_added: Final = OpenAIRealtimeStreamResponseOutputItemAdded( @@ -767,6 +809,108 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): current_delta_type, ) + @staticmethod + def _parse_generation_stage(additional_model_fields: object) -> str | None: + if not isinstance(additional_model_fields, str): + return None + try: + parsed: Final = json.loads(additional_model_fields) + except json.JSONDecodeError: + return None + stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None + return stage if isinstance(stage, str) else None + + def _current_user_item_id(self, new_utterance: bool = False) -> str: + """Item id shared by all events of one user utterance (speech boundaries and transcript).""" + if new_utterance or self._user_item_id is None: + self._user_item_id = f"item_{uuid.uuid4()}" + return self._user_item_id + + def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]: + """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events.""" + verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End") + speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = { + "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped", + "event_id": f"event_{uuid.uuid4()}", + "item_id": self._current_user_item_id(new_utterance=is_speech_start), + } + return (speech_event,) + + def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None: + """Record Bedrock's session-cumulative usage totals for the next response.done.""" + verbose_logger.debug("Handling usageEvent") + self._cumulative_usage = usage_event + + def _take_usage_delta(self) -> OpenAIRealtimeResponseUsage: + """Usage for the response now completing: cumulative totals minus what prior response.done events reported.""" + prior: Final = self._reported_usage + latest: Final = self._cumulative_usage + self._reported_usage = latest + input_details: Final[OpenAIRealtimeUsageTokenDetails] = { + "audio_tokens": latest.details.total.input.speechTokens - prior.details.total.input.speechTokens, + "text_tokens": latest.details.total.input.textTokens - prior.details.total.input.textTokens, + "cached_tokens": 0, + } + output_details: Final[OpenAIRealtimeUsageTokenDetails] = { + "audio_tokens": latest.details.total.output.speechTokens - prior.details.total.output.speechTokens, + "text_tokens": latest.details.total.output.textTokens - prior.details.total.output.textTokens, + } + usage_delta: Final[OpenAIRealtimeResponseUsage] = { + "input_tokens": latest.totalInputTokens - prior.totalInputTokens, + "output_tokens": latest.totalOutputTokens - prior.totalOutputTokens, + "total_tokens": latest.totalTokens - prior.totalTokens, + "input_token_details": input_details, + "output_token_details": output_details, + } + return usage_delta + + def leftover_usage_done_events(self) -> tuple[OpenAIRealtimeEvents, ...]: + """Logged-only response.done for usage Bedrock reports after the final turn's contentEnd.""" + if self._cumulative_usage == self._reported_usage: + return () + usage: Final = self._take_usage_delta() + leftover_done: Final = OpenAIRealtimeDoneEvent( + type="response.done", + event_id=f"event_{uuid.uuid4()}", + response=OpenAIRealtimeResponseDoneObject( + object="realtime.response", + id=f"resp_{uuid.uuid4()}", + status="completed", + conversation_id=f"conv_{uuid.uuid4()}", + usage=dict(usage), # mutable-ok: OpenAIRealtimeResponseDoneObject types usage as plain dict + ), + ) + return (leftover_done,) + + def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]: + """Transform a USER-role Bedrock textOutput (ASR transcript) to an OpenAI transcription delta.""" + verbose_logger.debug("Handling USER textOutput (ASR transcript)") + delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = { + "type": "conversation.item.input_audio_transcription.delta", + "event_id": f"event_{uuid.uuid4()}", + "item_id": self._current_user_item_id(), + "content_index": 0, + "delta": transcript, + } + if self._user_transcript_generation_stage != "SPECULATIVE": + self._user_transcript_buffer += transcript + return (delta_event,) + + def user_transcript_completed_events(self) -> tuple[OpenAIRealtimeEvents, ...]: + """One completed event with the full transcript once the FINAL user content block ends.""" + transcript: Final = self._user_transcript_buffer + if not transcript: + return () + self._user_transcript_buffer = "" + completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = { + "type": "conversation.item.input_audio_transcription.completed", + "event_id": f"event_{uuid.uuid4()}", + "item_id": self._current_user_item_id(), + "content_index": 0, + "transcript": transcript, + } + return (completed_event,) + def transform_text_output_event( self, event: dict, @@ -985,7 +1129,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not current_response_id or not current_conversation_id: return [], None, None, None - usage_obj: Final = get_empty_usage() + usage: Final = self._take_usage_delta() response_done: Final = OpenAIRealtimeDoneEvent( type="response.done", event_id=f"event_{uuid.uuid4()}", @@ -995,11 +1139,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): status="completed", output=[], conversation_id=current_conversation_id, - usage={ - "prompt_tokens": usage_obj.prompt_tokens, - "completion_tokens": usage_obj.completion_tokens, - "total_tokens": usage_obj.total_tokens, - }, + usage=dict(usage), ), ) @@ -1042,8 +1182,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Create a function call arguments done event # This is a custom event format that matches what clients expect - from typing import cast - function_call_event: Final[dict[str, Any]] = { "type": "response.function_call_arguments.done", "event_id": f"event_{uuid.uuid4()}", @@ -1194,18 +1332,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): returned_messages.extend(events) elif "textOutput" in event: - events, current_delta_chunks = self.transform_text_output_event( - event, - current_output_item_id, - current_response_id, - current_delta_chunks, - ) - returned_messages.extend(events) + if self._user_transcript_active: + returned_messages.extend(self.transform_user_transcript_event(event["textOutput"].get("content", ""))) + else: + events, current_delta_chunks = self.transform_text_output_event( + event, + current_output_item_id, + current_response_id, + current_delta_chunks, + ) + returned_messages.extend(events) elif "audioOutput" in event: events = self.transform_audio_output_event(event, current_output_item_id, current_response_id) returned_messages.extend(events) + elif "contentEnd" in event and self._user_transcript_active: + self._user_transcript_active = False + self._user_transcript_generation_stage = None + returned_messages.extend(self.user_transcript_completed_events()) + elif "contentEnd" in event: events, current_delta_chunks = self.transform_content_end_event( event, @@ -1224,6 +1370,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) = self._response_done_events(current_response_id, current_conversation_id) returned_messages.extend(done_events) + elif "userSpeechStart" in event or "userSpeechEnd" in event: + returned_messages.extend(self.transform_user_speech_event("userSpeechStart" in event)) + + elif "usageEvent" in event: + self.transform_usage_event(BedrockUsageEvent.model_validate(event["usageEvent"])) + elif "toolUse" in event: events, tool_call_id, tool_name = self.transform_tool_use_event( event, current_output_item_id, current_response_id diff --git a/litellm/llms/gigachat/__init__.py b/litellm/llms/gigachat/__init__.py index 3ddbd7864d9..e7c2206ffaa 100644 --- a/litellm/llms/gigachat/__init__.py +++ b/litellm/llms/gigachat/__init__.py @@ -15,9 +15,11 @@ API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/overview from .chat.transformation import GigaChatConfig, GigaChatError from .embedding.transformation import GigaChatEmbeddingConfig +from .passthrough.transformation import GigaChatPassthroughConfig -__all__ = [ +__all__ = ( "GigaChatConfig", "GigaChatEmbeddingConfig", "GigaChatError", -] + "GigaChatPassthroughConfig", +) diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index 9ef6fe7a93c..d6b217d5746 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -7,6 +7,7 @@ Based on official GigaChat SDK authentication flow. import time import uuid +from collections.abc import Mapping from typing import Final import httpx @@ -16,7 +17,7 @@ from litellm.caching.caching import InMemoryCache from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, - _get_httpx_client, + _get_httpx_client, # pyright: ignore[reportPrivateUsage] # house cached-client factory has no public alias get_async_httpx_client, ) from litellm.secret_managers.main import get_secret_str @@ -63,6 +64,7 @@ def get_access_token( credentials: str | None = None, scope: str | None = None, auth_url: str | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> str: """ Get valid access token, using cache if available. @@ -78,71 +80,88 @@ def get_access_token( Raises: GigaChatAuthError: If authentication fails """ - credentials = credentials or _get_credentials() - if not credentials: + if not litellm_params: + litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + + access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + if access_token: + return access_token + + effective_credentials: Final = credentials or _get_credentials() + if not effective_credentials: raise GigaChatAuthError( status_code=401, message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - scope = scope or _get_scope() - auth_url = auth_url or _get_auth_url() + effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() # Check cache - cache_key: Final = f"gigachat_token:{credentials[:16]}" + cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: - token, expires_at = cached + _token, _expires_at = cached # Check if token is still valid (with buffer) - if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS: verbose_logger.debug("Using cached GigaChat access token") - return token + return _token # Request new token - token, expires_at = _request_token_sync(credentials, scope, auth_url) + new_token, new_expires_at = _request_token_sync(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str - # Cache token - ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) - if ttl_seconds > 0: - _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + if new_expires_at: + # Cache token + ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) - return token + return new_token async def get_access_token_async( credentials: str | None = None, scope: str | None = None, auth_url: str | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> str: """Async version of get_access_token.""" - credentials = credentials or _get_credentials() - if not credentials: + if not litellm_params: + litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + + access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + if access_token: + return access_token + + effective_credentials: Final = credentials or _get_credentials() + if not effective_credentials: raise GigaChatAuthError( status_code=401, message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - scope = scope or _get_scope() - auth_url = auth_url or _get_auth_url() + effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() # Check cache - cache_key: Final = f"gigachat_token:{credentials[:16]}" + cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: - token, expires_at = cached - if time.time() * 1000 < expires_at - TOKEN_EXPIRY_BUFFER_MS: + _token, _expires_at = cached + if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS: verbose_logger.debug("Using cached GigaChat access token") - return token + return _token # Request new token - token, expires_at = await _request_token_async(credentials, scope, auth_url) + new_token, new_expires_at = await _request_token_async(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str - # Cache token - ttl_seconds: Final = max(0, (expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) - if ttl_seconds > 0: - _token_cache.set_cache(cache_key, (token, expires_at), ttl=ttl_seconds) + if new_expires_at: + # Cache token + ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) + if ttl_seconds > 0: + _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) - return token + return new_token def _request_token_sync( @@ -154,7 +173,7 @@ def _request_token_sync( Request new access token from GigaChat OAuth endpoint (sync). Returns: - Tuple of (access_token, expires_at_ms) + tuple of (access_token, expires_at_ms) """ headers: Final = { "Authorization": f"Basic {credentials}", @@ -169,7 +188,7 @@ def _request_token_sync( client: Final = _get_http_client() response: Final = client.post(auth_url, headers=headers, data=data, timeout=30) response.raise_for_status() - return _parse_token_response(response) + return _parse_token_response(response) # pyright: ignore[reportArgumentType] # httpx Response may be None at type level except httpx.HTTPStatusError as e: raise GigaChatAuthError( status_code=e.response.status_code, @@ -204,7 +223,7 @@ async def _request_token_async( ) response: Final = await client.post(auth_url, headers=headers, data=data, timeout=30) response.raise_for_status() - return _parse_token_response(response) + return _parse_token_response(response) # pyright: ignore[reportArgumentType] # httpx Response may be None at type level except httpx.HTTPStatusError as e: raise GigaChatAuthError( status_code=e.response.status_code, @@ -223,7 +242,7 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]: # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at' access_token: Final = data.get("tok") or data.get("access_token") - expires_at = data.get("exp") or data.get("expires_at") + expires_at_raw: Final = data.get("exp") or data.get("expires_at") if not access_token: raise GigaChatAuthError( @@ -232,8 +251,11 @@ def _parse_token_response(response: httpx.Response) -> tuple[str, int]: ) # expires_at is in milliseconds - if isinstance(expires_at, str): - expires_at = int(expires_at) + expires_at: int # rebind-ok: conditionally assigned from str or int + if isinstance(expires_at_raw, str): + expires_at = int(expires_at_raw) # rebind-ok: conditionally assigned from str or int + else: + expires_at = expires_at_raw # pyright: ignore[reportAssignmentType] # raw value is int or str; converted above; rebind-ok: conditionally assigned from str or int verbose_logger.debug("GigaChat access token obtained successfully") return access_token, expires_at diff --git a/litellm/llms/gigachat/chat/__init__.py b/litellm/llms/gigachat/chat/__init__.py index eb9492b90b3..0f9be19fedd 100644 --- a/litellm/llms/gigachat/chat/__init__.py +++ b/litellm/llms/gigachat/chat/__init__.py @@ -5,8 +5,8 @@ GigaChat Chat Module from .streaming import GigaChatModelResponseIterator from .transformation import GigaChatConfig, GigaChatError -__all__ = [ +__all__ = ( "GigaChatConfig", "GigaChatError", "GigaChatModelResponseIterator", -] +) diff --git a/litellm/llms/gigachat/chat/streaming.py b/litellm/llms/gigachat/chat/streaming.py index 219209773ea..2875b30232e 100644 --- a/litellm/llms/gigachat/chat/streaming.py +++ b/litellm/llms/gigachat/chat/streaming.py @@ -4,13 +4,15 @@ GigaChat Streaming Response Handler import json import uuid +from collections.abc import Mapping, Sequence from typing import Any, Final +from litellm.llms.gigachat.utils import convert_usage from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) -from litellm.types.utils import GenericStreamingChunk +from litellm.types.utils import ChatCompletionUsageBlock, GenericStreamingChunk class GigaChatModelResponseIterator: @@ -26,14 +28,9 @@ class GigaChatModelResponseIterator: self.response_iterator = self.streaming_response self.json_mode = json_mode - def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: + def chunk_parser(self, chunk: Mapping[str, object]) -> GenericStreamingChunk: """Parse a single streaming chunk from GigaChat.""" - text = "" - tool_use: ChatCompletionToolCallChunk | None = None - is_finished = False - finish_reason: str | None = None - - choices: Final = chunk.get("choices", []) + choices: Sequence = chunk.get("choices") or () # mutable-ok: tuple literal as default if not choices: return GenericStreamingChunk( text="", @@ -45,40 +42,63 @@ class GigaChatModelResponseIterator: ) choice: Final = choices[0] - delta: Final = choice.get("delta", {}) - finish_reason = choice.get("finish_reason") + delta: Mapping[str, object] = choice.get("delta") or {} # mutable-ok: empty dict default for get + chunk_finish_reason: Final = choice.get("finish_reason") # Extract text content - text = delta.get("content", "") or "" + text: Final = delta.get("content", "") or "" + + usage_block: ChatCompletionUsageBlock | None = None # rebind-ok: conditionally assigned after stop detection + tool_use: ChatCompletionToolCallChunk | None = None # rebind-ok: conditionally assigned on function_call + finish_reason: str | None = chunk_finish_reason # Handle function_call in stream - if finish_reason == "function_call" and delta.get("function_call"): - func_call: Final = delta["function_call"] - args = func_call.get("arguments", {}) - - if isinstance(args, dict): - args = json.dumps(args, ensure_ascii=False) + raw_function_call: Final = delta.get("function_call") + if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call: + func_call: Final[Mapping[str, object]] = raw_function_call + args_raw: Final[object] = func_call.get("arguments") or {} + args_str: str # rebind-ok: conditionally assigned from dict or str + if isinstance(args_raw, dict): + args_str = json.dumps(args_raw, ensure_ascii=False) # rebind-ok: build from dict + else: + args_str = str(args_raw) + name_raw: Final = func_call.get("name") tool_use = ChatCompletionToolCallChunk( id=f"call_{uuid.uuid4().hex[:24]}", type="function", function=ChatCompletionToolCallFunctionChunk( - name=func_call.get("name", ""), - arguments=args, + name=name_raw if isinstance(name_raw, str) else "", + arguments=args_str, ), index=0, ) finish_reason = "tool_calls" - if finish_reason is not None: - is_finished = True + usage_data: Final = chunk.get("usage") or {} # mutable-ok: empty dict default + if usage_data and isinstance(usage_data, dict): + validated_usage: Final = {k: int(v) for k, v in usage_data.items()} + usage = convert_usage(validated_usage) + _prompt_details: dict | None = ( + usage.prompt_tokens_details.model_dump() if usage.prompt_tokens_details else None + ) # rebind-ok: conditional + _completion_details: dict | None = ( + usage.completion_tokens_details.model_dump() if usage.completion_tokens_details else None + ) # rebind-ok: conditional + usage_block = ChatCompletionUsageBlock( # pyright: ignore[reportCallIssue] # TypedDict kwarg constructor + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + prompt_tokens_details=_prompt_details, + completion_tokens_details=_completion_details, + ) return GenericStreamingChunk( - text=text, + text=str(text), tool_use=tool_use, - is_finished=is_finished, + is_finished=chunk_finish_reason is not None, finish_reason=finish_reason or "", - usage=None, + usage=usage_block, index=choice.get("index", 0), ) diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index b859a843251..8f23c5175ec 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -4,19 +4,22 @@ GigaChat Chat Transformation Transforms OpenAI-format requests to GigaChat format and back. """ +from __future__ import annotations + import json import time import uuid -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx from litellm._logging import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.gigachat.utils import convert_usage, get_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import Choices, Message, ModelResponse, Usage +from litellm.types.utils import Choices, Message, ModelResponse from ..authenticator import get_access_token from ..file_handler import upload_file_sync @@ -30,9 +33,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -# GigaChat API endpoint -GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" - def is_valid_json(value: str) -> bool: """Checks whether the value passed is a valid serialized JSON string""" @@ -90,30 +90,30 @@ class GigaChatConfig(BaseConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], stream: bool | None = None, ) -> str: """Get complete API URL for chat completions.""" - base: Final = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL + base: Final = get_api_base(api_base) return f"{base}/chat/completions" def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: mutates in place per GigaChat OAuth setup model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict: # mutable-ok: base class contract returns dict for httpx """ Set up headers with OAuth token. """ # Get access token credentials: Final = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") - access_token: Final = get_access_token(credentials=credentials) + access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params) # Store credentials for image uploads self._current_credentials = credentials @@ -125,9 +125,9 @@ class GigaChatConfig(BaseConfig): return headers - def get_supported_openai_params(self, model: str) -> list[str]: + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: base class contract returns list """Return list of supported OpenAI parameters.""" - return [ + return [ # mutable-ok: base class contract returns list "stream", "temperature", "top_p", @@ -143,11 +143,11 @@ class GigaChatConfig(BaseConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: Mapping[str, object], + optional_params: dict, # mutable-ok: mutated in place per GigaChat mapping model: str, drop_params: bool, - ) -> dict: + ) -> dict: # mutable-ok: base class contract returns dict """Map OpenAI parameters to GigaChat parameters.""" for param, value in non_default_params.items(): if param == "stream": @@ -167,42 +167,50 @@ class GigaChatConfig(BaseConfig): pass elif param == "tools": # Convert tools to functions format - optional_params["functions"] = self._convert_tools_to_functions(value) + if isinstance(value, Sequence): + optional_params["functions"] = self._convert_tools_to_functions(value) elif param == "tool_choice": # Map OpenAI tool_choice to GigaChat function_call - mapped_choice = self._map_tool_choice(value) - if mapped_choice is not None: - optional_params["function_call"] = mapped_choice + if isinstance(value, (str, Mapping)): + mapped_choice = self._map_tool_choice(value) + if mapped_choice is not None: + optional_params["function_call"] = mapped_choice elif param == "functions": optional_params["functions"] = value elif param == "function_call": optional_params["function_call"] = value elif param == "response_format": # Handle structured output via function calling - if value.get("type") == "json_schema": + if isinstance(value, Mapping) and value.get("type") == "json_schema": json_schema = value.get("json_schema", {}) schema_name = json_schema.get("name", "structured_output") schema = json_schema.get("schema", {}) - function_def = { + function_def = { # mutable-ok: request payload for httpx "name": schema_name, "description": f"Output structured response: {schema_name}", "parameters": schema, } - if "functions" not in optional_params: - optional_params["functions"] = [] - optional_params["functions"].append(function_def) - optional_params["function_call"] = {"name": schema_name} + existing_functions = optional_params.get("functions") + optional_params["functions"] = [ + *( + existing_functions + if isinstance(existing_functions, Sequence) and not isinstance(existing_functions, str) + else () + ), + function_def, + ] + optional_params["function_call"] = {"name": schema_name} # mutable-ok: request payload optional_params["_structured_output"] = True return optional_params - def _convert_tools_to_functions(self, tools: list[dict]) -> list[dict]: + def _convert_tools_to_functions(self, tools: Sequence) -> Sequence[dict]: """Convert OpenAI tools format to GigaChat functions format.""" - functions: Final = [] + functions: Final[list[dict]] = [] # mutable-ok: accumulator for building functions list for tool in tools: - if tool.get("type") == "function": + if isinstance(tool, dict) and tool.get("type") == "function": func = tool.get("function", {}) functions.append( { @@ -213,7 +221,7 @@ class GigaChatConfig(BaseConfig): ) return functions - def _map_tool_choice(self, tool_choice: str | dict) -> str | dict | None: + def _map_tool_choice(self, tool_choice: str | Mapping[str, object]) -> str | Mapping[str, object] | None: """ Map OpenAI tool_choice to GigaChat function_call format. @@ -246,8 +254,9 @@ class GigaChatConfig(BaseConfig): # OpenAI format: {"type": "function", "function": {"name": "func_name"}} # GigaChat format: {"name": "func_name"} if tool_choice.get("type") == "function": - func_name: Final = tool_choice.get("function", {}).get("name") - if func_name: + function_spec: Final = tool_choice.get("function") + func_name: Final = function_spec.get("name") if isinstance(function_spec, Mapping) else None + if isinstance(func_name, str) and func_name: return {"name": func_name} # Default to None (don't set function_call) @@ -273,20 +282,51 @@ class GigaChatConfig(BaseConfig): verbose_logger.error("Failed to upload image: %s", e) return None + def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[str]]: + """ + Extract text and image attachments from a multimodal message content list. + + Args: + content: List of content parts (OpenAI multimodal format) + + Returns: + Tuple of (combined text, list of attachment file ids) + """ + texts: Final[list[str]] = [] # mutable-ok: accumulator + attachments: Final[list[str]] = [] # mutable-ok: accumulator + for part in content: + if isinstance(part, dict): + if part.get("type") == "text": + texts.append(part.get("text", "")) + elif part.get("type") == "image_url": + # Extract image URL and upload to GigaChat + image_url: object = part.get("image_url", {}) + upload_url: str + if isinstance(image_url, str): + upload_url = image_url + else: + upload_url = str(image_url.get("url", "")) if isinstance(image_url, dict) else "" + if upload_url: + file_id = self._upload_image(upload_url) + if file_id: + attachments.append(file_id) + text: Final = "\n".join(texts) if texts else "" + return text, attachments + def transform_request( self, model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + headers: Mapping[str, object], + ) -> dict: # mutable-ok: request payload sent to httpx """Transform OpenAI request to GigaChat format.""" # Transform messages giga_messages: Final = self._transform_messages(messages) # Build request - request_data: Final = { + request_data: Final[dict[str, object]] = { "model": model.replace("gigachat/", ""), "messages": giga_messages, } @@ -311,9 +351,9 @@ class GigaChatConfig(BaseConfig): return request_data - def _transform_messages(self, messages: list[AllMessageValues]) -> list[dict]: + def _transform_messages(self, messages: Sequence[AllMessageValues]) -> Sequence[dict]: """Transform OpenAI messages to GigaChat format.""" - transformed: Final = [] + transformed: Final[list[dict]] = [] # mutable-ok: accumulator for building transformed messages for i, msg in enumerate(messages): message = dict(msg) @@ -341,24 +381,7 @@ class GigaChatConfig(BaseConfig): # Handle list content (multimodal) - extract text and images content = message.get("content") if isinstance(content, list): - texts = [] - attachments = [] - for part in content: - if isinstance(part, dict): - if part.get("type") == "text": - texts.append(part.get("text", "")) - elif part.get("type") == "image_url": - # Extract image URL and upload to GigaChat - image_url = part.get("image_url", {}) - if isinstance(image_url, str): - url = image_url - else: - url = image_url.get("url", "") - if url: - file_id = self._upload_image(url) - if file_id: - attachments.append(file_id) - message["content"] = "\n".join(texts) if texts else "" + message["content"], attachments = self._transform_list_content(content) if attachments: message["attachments"] = attachments @@ -393,7 +416,7 @@ class GigaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: tiktoken.Encoding | None, api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -408,7 +431,7 @@ class GigaChatConfig(BaseConfig): is_structured_output: Final = optional_params.get("_structured_output", False) - choices: Final = [] + choices: Final[list[Choices]] = [] # mutable-ok: accumulator for building response choices for choice in response_json.get("choices", []): message_data = choice.get("message", {}) finish_reason = choice.get("finish_reason", "stop") @@ -462,11 +485,7 @@ class GigaChatConfig(BaseConfig): # Build usage usage_data: Final = response_json.get("usage", {}) - usage: Final = Usage( - prompt_tokens=usage_data.get("prompt_tokens", 0), - completion_tokens=usage_data.get("completion_tokens", 0), - total_tokens=usage_data.get("total_tokens", 0), - ) + usage: Final = convert_usage(usage_data) model_response.id = response_json.get("id", f"chatcmpl-{uuid.uuid4().hex[:12]}") model_response.created = response_json.get("created", int(time.time())) diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index bb495cea423..2ec8324e33c 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -5,6 +5,8 @@ Transforms OpenAI /v1/embeddings format to GigaChat format. API Documentation: https://developers.sber.ru/docs/ru/gigachat/api/reference/rest/post-embeddings """ +from __future__ import annotations + import types from typing import Final @@ -14,14 +16,12 @@ from litellm import LlmProviders from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.llms.gigachat.utils import get_api_base from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse from ..authenticator import get_access_token -# GigaChat API endpoint -GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" - class GigaChatEmbeddingError(BaseLLMException): """GigaChat Embedding API error.""" @@ -78,9 +78,9 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): Returns provider info for GigaChat. Returns: - Tuple of (custom_llm_provider, api_base, dynamic_api_key) + tuple of (custom_llm_provider, api_base, dynamic_api_key) """ - api_base = api_base or GIGACHAT_BASE_URL + api_base = get_api_base(api_base) return LlmProviders.GIGACHAT.value, api_base, api_key def get_complete_url( @@ -93,7 +93,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): stream: bool | None = None, ) -> str: """Get the complete URL for embeddings endpoint.""" - base: Final = api_base or GIGACHAT_BASE_URL + base: Final = get_api_base(api_base) return f"{base}/embeddings" def transform_embedding_request( @@ -114,14 +114,12 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): """ # Normalize input to list if isinstance(input, str): - input_list: list = [input] - elif isinstance(input, list): - input_list = input + input_list: list = [input] # rebind-ok: locally scoped conversion else: - input_list = [input] + input_list = input # Remove gigachat/ prefix from model if present - model = model.removeprefix("gigachat/") + model = model.removeprefix("gigachat/") # rebind-ok: parameter reassignment for normalization return { "model": model, @@ -191,7 +189,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): Set up headers with OAuth token for GigaChat. """ # Get access token via OAuth - access_token: Final = get_access_token(api_key) + access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params) default_headers: Final = { "Content-Type": "application/json", diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index 4cbde551fa2..163e944f124 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -9,6 +9,7 @@ import base64 import hashlib import re import uuid +from collections.abc import Mapping from typing import Final from litellm._logging import verbose_logger @@ -16,13 +17,11 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) +from litellm.llms.gigachat.utils import get_api_base from litellm.types.utils import LlmProviders from .authenticator import get_access_token, get_access_token_async -# GigaChat API endpoint -GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" - # Simple in-memory cache for file IDs _file_cache: Final[dict[str, str]] = {} @@ -82,6 +81,7 @@ def upload_file_sync( image_url: str, credentials: str | None = None, api_base: str | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> str | None: """ Upload file to GigaChat and return file_id (sync). @@ -114,10 +114,10 @@ def upload_file_sync( filename: Final = f"{uuid.uuid4()}.{ext}" # Get access token - access_token: Final = get_access_token(credentials) + access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params) # Upload to GigaChat - base_url: Final = api_base or GIGACHAT_BASE_URL + base_url: Final = get_api_base(api_base) upload_url: Final = f"{base_url}/files" client: Final = _get_httpx_client(params={"ssl_verify": False}) @@ -147,6 +147,7 @@ async def upload_file_async( image_url: str, credentials: str | None = None, api_base: str | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> str | None: """ Upload file to GigaChat and return file_id (async). @@ -179,10 +180,10 @@ async def upload_file_async( filename: Final = f"{uuid.uuid4()}.{ext}" # Get access token - access_token: Final = await get_access_token_async(credentials) + access_token: Final = await get_access_token_async(credentials=credentials, litellm_params=litellm_params) # Upload to GigaChat - base_url: Final = api_base or GIGACHAT_BASE_URL + base_url: Final = get_api_base(api_base) upload_url: Final = f"{base_url}/files" client: Final = get_async_httpx_client( diff --git a/litellm/llms/gigachat/passthrough/__init__.py b/litellm/llms/gigachat/passthrough/__init__.py new file mode 100644 index 00000000000..a66a078dbeb --- /dev/null +++ b/litellm/llms/gigachat/passthrough/__init__.py @@ -0,0 +1,7 @@ +""" +GigaChat passthrough Module +""" + +from .transformation import GigaChatPassthroughConfig + +__all__ = ("GigaChatPassthroughConfig",) diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py new file mode 100644 index 00000000000..a0edc6f5682 --- /dev/null +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +import httpx + +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.llms.gigachat.authenticator import get_access_token +from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator +from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import EmbeddingResponse + +if TYPE_CHECKING: + from httpx import URL, Response + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import CostResponseTypes + + +class GigaChatPassthroughConfig(BasePassthroughConfig): + def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: + return request_data.get("stream", False) + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + endpoint: str, + request_query_params: Mapping[str, object] | None, + litellm_params: Mapping[str, object], + ) -> tuple[URL, str]: + """Get complete API URL for chat completions.""" + base_target_url: Final = self.get_api_base(api_base) + + if base_target_url is None: + raise Exception("GigaChat api base not found") + + complete_url: Final = f"{base_target_url}/{endpoint.lstrip('/')}" + + return ( + httpx.URL(complete_url), + base_target_url, + ) + + def validate_environment( + self, + headers: dict, # mutable-ok: mutates in place to set OAuth headers + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: # mutable-ok: base class contract returns dict for httpx + """ + Set up headers with OAuth token. + """ + # Get access token + access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params) + + headers["Authorization"] = f"Bearer {access_token}" # rebind-ok: mutating for OAuth setup + headers["Content-Type"] = "application/json" # rebind-ok: mutating for OAuth setup + headers["Accept"] = "application/json" # rebind-ok: mutating for OAuth setup + + return headers + + def logging_non_streaming_response( + self, + model: str, + custom_llm_provider: str, + httpx_response: Response, + request_data: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + endpoint: str, + ) -> CostResponseTypes | None: + from litellm import encoding + from litellm.types.utils import LlmProviders, ModelResponse + from litellm.utils import ProviderConfigManager + + # cost tracking only for completions and embeddings + if "completions" in endpoint: + provider_chat_config: Final = ProviderConfigManager.get_provider_chat_config( + provider=LlmProviders(custom_llm_provider), + model=model, + ) + + if provider_chat_config is None: + raise ValueError(f"No provider config found for model: {model}") + + raw_messages: Final = request_data.get("messages") + litellm_model_response: Final = provider_chat_config.transform_response( + model=model, + messages=list(raw_messages) + if isinstance(raw_messages, list) + else [], # mutable-ok: transform_response wants a list + raw_response=httpx_response, + model_response=ModelResponse(), + logging_obj=logging_obj, + optional_params={}, # mutable-ok: empty dict kwarg for transform_response + litellm_params={}, # mutable-ok: empty dict kwarg for transform_response + api_key="", + request_data=dict(request_data), # mutable-ok: transform_response wants a dict + encoding=encoding, + ) + + return litellm_model_response + + if "embeddings" in endpoint: + provider_embedding_config: Final = ProviderConfigManager.get_provider_embedding_config( + provider=LlmProviders(custom_llm_provider), + model=model, + ) + + if provider_embedding_config is None: + raise ValueError(f"No provider config found for model: {model}") + + litellm_embedding_response: Final[EmbeddingResponse] = ( + provider_embedding_config.transform_embedding_response( + model=model, + raw_response=httpx_response, + model_response=EmbeddingResponse(), + logging_obj=logging_obj, + optional_params={}, # mutable-ok: empty dict kwarg for transform_embedding_response + api_key="", + request_data=dict(request_data), # mutable-ok: transform_embedding_response wants a dict + litellm_params={}, # mutable-ok: empty dict kwarg for transform_embedding_response + ) + ) + + return litellm_embedding_response + + return None + + def handle_logging_collected_chunks( + self, + all_chunks: Sequence[str], + litellm_logging_obj: LiteLLMLoggingObj, + model: str, + custom_llm_provider: str, + endpoint: str, + ) -> CostResponseTypes | None: + """ + 1. Convert all_chunks to a ModelResponseStream + 2. combine model_response_stream to model_response + 3. Return the model_response + """ + + from litellm.litellm_core_utils.streaming_handler import ( + convert_generic_chunk_to_model_response_stream, + generic_chunk_has_all_required_fields, + ) + from litellm.main import stream_chunk_builder + from litellm.types.utils import ModelResponseStream + + all_translated_chunks: Final[list[object]] = [] # mutable-ok: accumulator + + for chunk in all_chunks: + chunk = chunk.strip() + if not chunk or chunk == "[DONE]": + continue + chunk = chunk.removeprefix("data: ") + try: + message = json.loads(chunk) + except json.JSONDecodeError: + continue + + gigachat_iterator = GigaChatModelResponseIterator( + streaming_response=None, + sync_stream=False, + ) + translated_chunk = gigachat_iterator.chunk_parser(chunk=message) + + if isinstance(translated_chunk, dict) and generic_chunk_has_all_required_fields( # pyright: ignore[reportUnnecessaryIsInstance] # runtime guard for patched chunk_parser + dict(translated_chunk) + ): + chunk_obj = convert_generic_chunk_to_model_response_stream( + translated_chunk # pyright: ignore[reportArgumentType] # validated TypedDict + ) + elif isinstance(translated_chunk, ModelResponseStream): + chunk_obj = translated_chunk + else: + continue + + all_translated_chunks.append(chunk_obj) + + if len(all_translated_chunks) > 0: + return stream_chunk_builder( + chunks=all_translated_chunks, + logging_obj=litellm_logging_obj, + ) + return None + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL + + @staticmethod + def get_api_key( + api_key: str | None = None, + ) -> str | None: + return api_key or get_secret_str("GIGACHAT_API_KEY") + + @staticmethod + def get_base_model(model: str) -> str | None: + return model + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return list(super().get_models(api_key, api_base)) diff --git a/litellm/llms/gigachat/utils.py b/litellm/llms/gigachat/utils.py new file mode 100644 index 00000000000..cbb35cd1b57 --- /dev/null +++ b/litellm/llms/gigachat/utils.py @@ -0,0 +1,26 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + +# GigaChat API endpoint +GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" + + +def convert_usage(usage_data: Mapping[str, int]) -> Usage: + precached_prompt_tokens: Final = usage_data.get("precached_prompt_tokens", 0) + prompt_tokens_details: Final = ( + PromptTokensDetailsWrapper(cached_tokens=precached_prompt_tokens) if precached_prompt_tokens > 0 else None + ) + + return Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0) + precached_prompt_tokens, + completion_tokens=usage_data.get("completion_tokens", 0), + prompt_tokens_details=prompt_tokens_details, + total_tokens=usage_data.get("total_tokens", 0) + precached_prompt_tokens, + ) + + +def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py index 384e7ec4cf8..6e9bb83b0a0 100644 --- a/litellm/llms/oci/chat/cohere.py +++ b/litellm/llms/oci/chat/cohere.py @@ -9,7 +9,7 @@ response parsing, and streaming chunk parsing for models served with import datetime import json from collections.abc import Iterable, Mapping, Sequence -from typing import Any, Final +from typing import Final import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -76,7 +76,7 @@ def _content_text(content: str | Iterable[Mapping[str, object]] | None) -> str: return str(content) -def _extract_text_content(content: Any) -> str: +def _extract_text_content(content: str | Iterable[Mapping[str, object]] | None) -> str: """Return the plain-text representation of a message content value.""" return _content_text(content) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 1b1ab80e85d..4d774f6f165 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -268,6 +268,7 @@ class BaseOpenAILLM: "max_retries", "organization", "api_base", + "workload_identity_config", ) openai_client_fields: Final = ( BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type=client_type) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 6e66c998acf..16fa0017b23 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -51,6 +51,7 @@ from .common_utils import ( drop_params_from_unprocessable_entity_error, is_output_token_limit_error, ) +from .workload_identity import resolve_openai_workload_identity_config openaiOSeriesConfig: Final = OpenAIOSeriesConfig() openAIGPT5Config: Final = OpenAIGPT5Config() @@ -349,6 +350,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client: OpenAI | AsyncOpenAI | None = None, shared_session: Optional["ClientSession"] = None, ) -> OpenAI | AsyncOpenAI | None: + workload_identity_config: Final = resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base) client_initialization_params: Final[dict] = locals() if client is None: if not isinstance(max_retries, int): @@ -364,28 +366,49 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): if cached_client: if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI): return cached_client - http_client: Final[httpx.Client | httpx.AsyncClient | None] = ( - OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) - if is_async - else OpenAIChatCompletion._get_sync_http_client() - ) if is_async: - _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI( - api_key=api_key, - base_url=api_base, - http_client=http_client, - timeout=timeout, - max_retries=max_retries, - organization=organization, + async_http_client: Final = OpenAIChatCompletion._get_async_http_client(shared_session=shared_session) + http_client: httpx.Client | httpx.AsyncClient | None = async_http_client + _new_client: OpenAI | AsyncOpenAI = ( + AsyncOpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + http_client=async_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + if workload_identity_config is not None + else AsyncOpenAI( + api_key=api_key, + base_url=api_base, + http_client=async_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) ) else: - _new_client = OpenAI( - api_key=api_key, - base_url=api_base, - http_client=http_client, - timeout=timeout, - max_retries=max_retries, - organization=organization, + sync_http_client: Final = OpenAIChatCompletion._get_sync_http_client() + http_client = sync_http_client + _new_client = ( + OpenAI( + workload_identity=workload_identity_config.to_sdk_workload_identity(), + base_url=api_base, + http_client=sync_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) + if workload_identity_config is not None + else OpenAI( + api_key=api_key, + base_url=api_base, + http_client=sync_http_client, + timeout=timeout, + max_retries=max_retries, + organization=organization, + ) ) ## SAVE CACHE KEY diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 6b2f4535df1..88f04c59e01 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic. This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint. """ -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal + +from typing_extensions import ReadOnly, TypedDict + + +class ResponsesInputTextPart(TypedDict): + type: ReadOnly[Literal["input_text"]] + text: ReadOnly[str] + + +class ResponsesInputImagePart(TypedDict): + type: ReadOnly[Literal["input_image"]] + image_url: ReadOnly[str] + detail: ReadOnly[str] + + +class ResponsesInputFilePart(TypedDict): + type: ReadOnly[Literal["input_file"]] + filename: ReadOnly[str] + file_data: ReadOnly[str] + + +ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart + +ResponsesContentRole = Literal["user", "assistant"] + + +def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None: + url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url + if not isinstance(url, str) or not url: + return None + detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None + part: Final[ResponsesInputImagePart] = { + "type": "input_image", + "image_url": url, + "detail": detail if isinstance(detail, str) and detail else "auto", + } + return part + + +def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None: + """Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it.""" + if not isinstance(file_value, Mapping): + return None + filename: Final = file_value.get("filename") + file_data: Final = file_value.get("file_data") + if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data: + return None + part: Final[ResponsesInputFilePart] = { + "type": "input_file", + "filename": filename, + "file_data": file_data, + } + return part + + +def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None: + if isinstance(block, str): + bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block} + return bare + if not isinstance(block, Mapping): + return None + match block.get("type"): + case "text": + text_value: Final = block.get("text") + text: Final[ResponsesInputTextPart] = { + "type": "input_text", + "text": text_value if isinstance(text_value, str) else "", + } + return text + case "image_url" if role == "user": + return _chat_image_block_to_responses_part(block.get("image_url")) + case "file" if role == "user": + return _chat_file_block_to_responses_part(block.get("file")) + case _: + return None + + +def chat_content_blocks_to_responses_content( + content: Sequence[object], + role: ResponsesContentRole, +) -> str | tuple[ResponsesInputPart, ...]: + """Text-only content collapses to a joined string, which every role accepts and counts identically. + + Only a user turn may carry an image or file part: the Responses API rejects any part but + output_text and refusal inside an assistant turn. + """ + parts: Final = tuple( + part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None + ) + if any(part["type"] != "input_text" for part in parts): + return parts + return "\n".join(part["text"] for part in parts if part["type"] == "input_text") class OpenAICountTokensConfig: @@ -120,18 +213,13 @@ class OpenAICountTokensConfig: instructions_parts.append("\n".join(text_parts)) elif role == "user": if isinstance(content, list): - # Extract text from content blocks for Responses API - text_parts = [] - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text_parts.append(block.get("text", "")) - elif isinstance(block, str): - text_parts.append(block) - content = "\n".join(text_parts) + content = chat_content_blocks_to_responses_content(content, "user") input_items.append({"role": "user", "content": content}) elif role == "assistant": # Map tool_calls to Responses API function_call items tool_calls = msg.get("tool_calls") + if isinstance(content, list): + content = chat_content_blocks_to_responses_content(content, "assistant") if content: input_items.append({"role": "assistant", "content": content}) if tool_calls: diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index eac844a790d..eadc087383a 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -21,6 +21,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from ..common_utils import OpenAIError +from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -392,6 +393,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm_params = litellm_params or GenericLiteLLMParams() api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") headers.setdefault("Content-Type", "application/json") + workload_identity_config: Final = ( + resolve_openai_workload_identity_config(api_key=api_key, api_base=litellm_params.api_base) + if self.custom_llm_provider is LlmProviders.OPENAI + else None + ) + if workload_identity_config is not None: + headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}" + return headers headers["Authorization"] = f"Bearer {api_key}" return headers diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py new file mode 100644 index 00000000000..ecec161ed46 --- /dev/null +++ b/litellm/llms/openai/workload_identity.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import litellm +from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str + +from .common_utils import OpenAIError + +if TYPE_CHECKING: + from collections.abc import Callable + + from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth + +OPENAI_WIF_CLIENT_ID: Final = "litellm" +_OPENAI_API_HOST: Final = "api.openai.com" +_SDK_UPGRADE_MESSAGE: Final = ( + "OpenAI workload identity federation requires openai>=2.32.0. " + "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / " + "OPENAI_SERVICE_ACCOUNT_ID / OPENAI_IDENTITY_TOKEN_FILE." +) + + +@dataclass(frozen=True, slots=True) +class OpenAIWorkloadIdentityConfig: + identity_provider_id: str + service_account_id: str + token_file: str + + def to_sdk_workload_identity(self) -> WorkloadIdentity: + k8s_token_provider: Final = _load_sdk_k8s_token_provider() + workload_identity: Final[WorkloadIdentity] = { + "client_id": OPENAI_WIF_CLIENT_ID, + "identity_provider_id": self.identity_provider_id, + "service_account_id": self.service_account_id, + "provider": k8s_token_provider(self.token_file), + } + return workload_identity + + +def resolve_openai_workload_identity_config( + api_key: str | None, + api_base: str | None, +) -> OpenAIWorkloadIdentityConfig | None: + static_api_key: Final = normalize_nonempty_secret_str(api_key) or normalize_nonempty_secret_str( + get_secret_str("OPENAI_API_KEY") + ) + if static_api_key is not None: + return None + effective_api_base: Final = ( + api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") + ) + if not _targets_openai_api(effective_api_base): + return None + identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID") + service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID") + token_file: Final = get_secret_str("OPENAI_IDENTITY_TOKEN_FILE") + if not identity_provider_id or not service_account_id or not token_file: + return None + return OpenAIWorkloadIdentityConfig( + identity_provider_id=identity_provider_id, + service_account_id=service_account_id, + token_file=token_file, + ) + + +def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) -> str: + return _workload_identity_auth(config).get_token() + + +def _targets_openai_api(api_base: str | None) -> bool: + if api_base is None: + return True + parsed: Final = urlparse(api_base) + return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST + + +@lru_cache(maxsize=16) +def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth: + sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth() + return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity()) + + +def _load_sdk_workload_identity_auth() -> type[WorkloadIdentityAuth]: + try: + from openai.auth import WorkloadIdentityAuth as sdk_workload_identity_auth + except ImportError as e: + raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e + return sdk_workload_identity_auth + + +def _load_sdk_k8s_token_provider() -> Callable[[str], SubjectTokenProvider]: + try: + from openai.auth import k8s_service_account_token_provider + except ImportError as e: + raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e + return k8s_service_account_token_provider diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index c7289eb47f5..c7696a1cb29 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -160,14 +160,12 @@ class RunwayMLVideoConfig(BaseVideoConfig): **self._prompt_image_param(video_create_optional_params), **self._ratio_param(video_create_optional_params), **self._duration_param(video_create_optional_params), - # Pass through other parameters that aren't OpenAI-specific **{key: value for key, value in video_create_optional_params.items() if key not in supported_openai_params}, } @staticmethod def _prompt_image_param(video_create_optional_params: VideoCreateOptionalRequestParams) -> Mapping[str, object]: # Handle input_reference parameter - map to promptImage - # RunwayML supports URLs and data URIs directly if "input_reference" in video_create_optional_params: return {"promptImage": video_create_optional_params["input_reference"]} return {} diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 5889a8eba06..725a7f39917 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -182,7 +182,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): else None ) - # Generation config with proper structure for image editing generation_config: Final[dict[str, object]] = { key: value for key, value in (("response_modalities", ["IMAGE"]), ("image_config", image_config)) if value } diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index eedd488ecdb..5c250fc1a7e 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -203,7 +203,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): if value is not None } - # Build the request body for Vertex AI RAG API query_body: Final[Mapping[str, object]] = { key: value for key, value in (("text", query), ("rag_retrieval_config", rag_retrieval_config or None)) @@ -294,7 +293,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): # Add metadata if provided metadata: Final = vector_store_create_optional_params.get("metadata") - # Build the request body for Vertex AI RAG Corpus creation request_body: Final[dict[str, object]] = { key: value for key, value in ( diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 75098515deb..aca257dc095 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -27,6 +27,15 @@ from .common_utils import ( get_vertex_base_url, ) + +def _graft_default_vertex_path(api_base: str, default_url: str) -> str: + parsed_api_base: Final = urlparse(api_base) + default_segments: Final = urlparse(default_url).path.lstrip("/").split("/") + graft_segments: Final = default_segments[1:] if default_segments[0] in ("v1", "v1beta1") else default_segments + grafted_path: Final = parsed_api_base.path.rstrip("/") + "/" + "/".join(graft_segments) + return parsed_api_base._replace(path=grafted_path).geturl() + + GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform" ) @@ -621,8 +630,9 @@ class VertexBase: Handles custom api_base for: 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} - 2. Vertex AI with standard proxies - constructs {api_base}:{endpoint}; - if api_base has no path (bare host), grafts the default vertex URL path onto it + 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the + api_base when its path is empty or only an API version (/v1, /v1beta1); + otherwise constructs {api_base}:{endpoint} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -669,10 +679,14 @@ class VertexBase: ) elif urlparse(api_base).path in ("", "/"): url = api_base.rstrip("/") + urlparse(url).path + elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: + url = _graft_default_vertex_path(api_base=api_base, default_url=url) else: url = f"{api_base}:{endpoint}" if stream is True: - url = url + "?alt=sse" + parsed_stream_url: Final = urlparse(url) + stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" + url = parsed_stream_url._replace(query=stream_query).geturl() return auth_header, url def _get_token_and_url( diff --git a/litellm/main.py b/litellm/main.py index c341db08155..0c8bff16f81 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5507,6 +5507,9 @@ def completion( tpm=kwargs.get("tpm"), rpm=kwargs.get("rpm"), use_xai_oauth=kwargs.get("use_xai_oauth", False), + gigachat_scope=kwargs.get("gigachat_scope"), + gigachat_auth_url=kwargs.get("gigachat_auth_url"), + gigachat_access_token=kwargs.get("gigachat_access_token"), **{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs}, ) cast(LiteLLMLoggingObj, logging).update_environment_variables( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 05c1cfd3179..718e6c489fd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -553,6 +553,26 @@ "supports_response_schema": true, "supports_vision": true }, + "amazon.nova-sonic-v1:0": { + "input_cost_per_audio_token": 3.4e-06, + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.36e-05, + "output_cost_per_token": 2.4e-07, + "supports_audio_input": true, + "supports_audio_output": true + }, + "amazon.nova-2-sonic-v1:0": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2.75e-06, + "supports_audio_input": true, + "supports_audio_output": true + }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, @@ -3726,7 +3746,7 @@ "output_cost_per_token": 0, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-services/", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure/eu/gpt-4o-2024-08-06": { @@ -5328,7 +5348,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/gpt-realtime-whisper", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -9107,7 +9127,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/Cohere-embed-v3-multilingual": { @@ -9118,7 +9138,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/FLUX-1.1-pro": { @@ -9134,7 +9154,7 @@ "litellm_provider": "azure_ai", "mode": "image_generation", "output_cost_per_image": 0.04, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/images/generations" ] @@ -9467,7 +9487,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9481,7 +9501,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.04e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9494,7 +9514,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -9543,7 +9563,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -9554,7 +9574,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { @@ -9566,7 +9586,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.1e-07, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -9756,7 +9776,7 @@ "supported_endpoints": [ "/v1/ocr" ], - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", @@ -9967,7 +9987,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/embeddings" ], @@ -10151,7 +10171,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://azure.microsoft.com/en-us/products/ai-services/ai-foundry/models/jais-30b-chat" + "source": "https://ai.azure.com/catalog/models/jais-30b-chat" }, "azure_ai/jamba-instruct": { "input_cost_per_token": 5e-07, @@ -10208,7 +10228,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10231,7 +10251,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10243,7 +10263,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10280,7 +10300,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", "supports_function_calling": true }, "azure_ai/mistral-small": { @@ -12315,7 +12335,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "claude-haiku-4-5-20251001": { "deprecation_date": "2026-10-15", @@ -17767,7 +17788,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "deprecation_date": "2027-01-08" }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -19564,6 +19586,61 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "friendliai/zai-org/GLM-5.3-Flash": { + "litellm_provider": "friendliai", + "supports_reasoning": true, + "supports_function_calling": true, + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "mode": "chat", + "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks", + "source": "https://api.friendli.ai/serverless/v1/models", + "supports_vision": true, + "supports_image_input": true, + "supports_video_input": true + }, + "friendliai/zai-org/GLM-5.3": { + "litellm_provider": "friendliai", + "supports_reasoning": true, + "supports_function_calling": true, + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.26e-06, + "output_cost_per_token": 3.96e-06, + "cache_read_input_token_cost": 2.34e-07, + "supports_prompt_caching": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "mode": "chat", + "comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery", + "source": "https://api.friendli.ai/serverless/v1/models", + "supports_vision": false, + "supports_image_input": false + }, "ft:babbage-002": { "deprecation_date": "2026-10-23", "input_cost_per_token": 1.6e-06, @@ -22843,7 +22920,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22897,7 +22974,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22962,7 +23039,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23021,7 +23098,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23226,7 +23303,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23310,7 +23387,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23373,7 +23450,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23430,7 +23507,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -24342,7 +24419,7 @@ "supports_response_schema": true, "supports_vision": true }, - "gigachat/GigaChat-2-Lite": { + "gigachat/GigaChat-2": { "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 128000, @@ -24404,6 +24481,15 @@ "output_cost_per_token": 0.0, "output_vector_size": 2560 }, + "gigachat/GigaEmbeddings-3B-2025-09": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2048 + }, "gmi/anthropic/claude-opus-4.5": { "input_cost_per_token": 5e-06, "litellm_provider": "gmi", @@ -25327,7 +25413,7 @@ "supports_vision": true }, "gpt-4o-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -25650,7 +25736,7 @@ "supports_vision": true }, "gpt-4o-mini-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -25688,7 +25774,7 @@ "gpt-4o-mini-realtime-preview": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -25787,7 +25873,8 @@ "output_cost_per_token": 5e-06, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-4o-mini-tts": { "input_cost_per_token": 2.5e-06, @@ -25809,7 +25896,7 @@ }, "gpt-4o-realtime-preview": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25828,7 +25915,7 @@ }, "gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25847,7 +25934,7 @@ }, "gpt-4o-realtime-preview-2025-06-03": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25926,7 +26013,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, @@ -27077,7 +27165,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/gpt-5.6-cyber", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-cyber", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27116,7 +27204,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27156,7 +27244,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -27168,7 +27256,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "source": "https://platform.openai.com/docs/models/chat-latest", + "source": "https://developers.openai.com/api/docs/models/chat-latest", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -30463,7 +30551,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30504,7 +30592,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30545,7 +30633,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30578,7 +30666,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30594,7 +30682,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30610,7 +30698,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -30627,7 +30715,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -32572,7 +32660,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 164000, @@ -32584,7 +32672,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { "max_tokens": 128000, @@ -32595,7 +32683,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { "max_tokens": 128000, @@ -32606,7 +32694,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 128000, @@ -32617,7 +32705,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { "max_tokens": 128000, @@ -32629,7 +32717,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -32640,7 +32728,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { "max_tokens": 128000, @@ -32650,7 +32738,7 @@ "output_cost_per_token": 6e-08, "litellm_provider": "nebius", "mode": "chat", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 128000, @@ -32661,7 +32749,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { "max_tokens": 128000, @@ -32672,7 +32760,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { "max_tokens": 128000, @@ -32683,7 +32771,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { "max_tokens": 128000, @@ -32694,7 +32782,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 128000, @@ -32705,7 +32793,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { "max_tokens": 128000, @@ -32716,7 +32804,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { "max_tokens": 131072, @@ -32727,7 +32815,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { "max_tokens": 262144, @@ -32738,7 +32826,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { "max_tokens": 32768, @@ -32749,7 +32837,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -32760,7 +32848,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { "max_tokens": 32768, @@ -32771,7 +32859,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { "max_tokens": 32768, @@ -32782,7 +32870,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { "max_tokens": 32768, @@ -32794,7 +32882,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { "max_tokens": 128000, @@ -32805,7 +32893,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { "max_tokens": 128000, @@ -32816,7 +32904,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { "max_tokens": 32768, @@ -32827,7 +32915,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { "max_tokens": 131072, @@ -32839,7 +32927,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -32851,7 +32939,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { "max_tokens": 131072, @@ -32862,7 +32950,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, @@ -32871,7 +32959,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { "max_tokens": 8192, @@ -32880,7 +32968,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { "max_tokens": 32768, @@ -32889,7 +32977,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, @@ -33630,7 +33718,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33643,7 +33731,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33656,7 +33744,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33713,7 +33801,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true, @@ -33727,7 +33815,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": false, "supports_response_schema": false, "supports_native_streaming": true @@ -33738,7 +33826,7 @@ "max_input_tokens": 512, "mode": "embedding", "output_vector_size": 1024, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -34635,7 +34723,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", + "source": "https://openrouter.ai/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { @@ -34870,7 +34958,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -34911,7 +34999,7 @@ "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -35996,7 +36084,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/deepseek-r1-distill-llama-70b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36010,7 +36098,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llama-3-1-8b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36023,7 +36111,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-1-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": false, "supports_tool_choice": false @@ -36036,7 +36124,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-3-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36049,7 +36137,7 @@ "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-7b-instruct-v0-3", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36062,7 +36150,7 @@ "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-nemo-instruct-2407", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36075,7 +36163,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-small-3-2-24b-instruct-2506", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -36089,7 +36177,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mixtral-8x7b-instruct-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36102,7 +36190,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-coder-32b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36115,7 +36203,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-vl-72b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36129,7 +36217,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen3-32b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36143,7 +36231,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-120b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36157,7 +36245,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-20b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36171,7 +36259,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llava-next-mistral-7b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36185,7 +36273,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mamba-codestral-7b-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -38321,7 +38409,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { - "deprecation_date": "2026-01-14", + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38946,7 +39034,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, - "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", + "source": "https://www.together.ai/models/qwen3-5-397b-a17b", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -43183,7 +43271,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43199,7 +43287,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43216,7 +43304,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43232,7 +43320,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43352,7 +43440,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43366,7 +43454,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43381,7 +43469,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43396,7 +43484,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -44100,7 +44188,8 @@ "output_cost_per_second": 0.0001, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "xai/grok-3": { "cache_read_input_token_cost": 2e-07, @@ -45088,7 +45177,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2", "supported_modalities": [ "text" ], @@ -45100,7 +45189,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -45112,7 +45201,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -49322,7 +49411,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-realtime-whisper", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -50574,7 +50663,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50612,7 +50701,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50650,7 +50739,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50688,7 +50777,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -51437,7 +51526,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { "max_tokens": 131072, @@ -51450,7 +51539,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { "max_tokens": 131072, @@ -51463,7 +51552,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { "max_tokens": 131072, @@ -51476,7 +51565,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { "max_tokens": 163840, @@ -51489,7 +51578,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { "max_tokens": 1000192, @@ -51502,7 +51591,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, @@ -51606,7 +51695,7 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-transcribe", "supported_endpoints": [ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" @@ -51624,7 +51713,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-live-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-live-transcribe", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -51645,7 +51734,7 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", - "source": "https://platform.openai.com/docs/models/gpt-realtime-translate", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-translate", "supported_modalities": [ "audio" ], @@ -51673,7 +51762,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -51712,7 +51801,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_assistant_prefill": false, diff --git a/litellm/models/model.py b/litellm/models/model.py index 209f26d4837..a0c840341ab 100644 --- a/litellm/models/model.py +++ b/litellm/models/model.py @@ -29,6 +29,8 @@ class LiteLLM_ProxyModelTable(LiteLLMPydanticObjectBase): @model_validator(mode="before") @classmethod def check_potential_json_str(cls, values): + if not isinstance(values, dict): + return values if isinstance(values.get("litellm_params"), str): try: values["litellm_params"] = json.loads(values["litellm_params"]) diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 4b30afb2f98..9095cee15a9 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -2,17 +2,22 @@ This module is used to pass through requests to the LLM APIs. """ +from __future__ import annotations + import asyncio import contextvars -from collections.abc import AsyncGenerator, Coroutine, Generator +from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Generator, Iterator from functools import partial -from typing import TYPE_CHECKING, Any, Final, Optional, cast +from types import TracebackType +from typing import Any, Final, cast import httpx -from httpx._types import CookieTypes, QueryParamTypes, RequestFiles +from httpx._types import CookieTypes, QueryParamTypes, RequestContent, RequestFiles from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.passthrough.utils import CommonUtils @@ -21,9 +26,222 @@ from litellm.utils import client base_llm_http_handler = BaseLLMHTTPHandler() from .utils import BasePassthroughUtils -if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + +async def _as_async_generator(iterable: AsyncIterator[bytes]) -> AsyncGenerator[bytes, Any]: + async for chunk in iterable: + yield chunk + + +def _as_generator(iterable: Iterator[bytes]) -> Generator[bytes, Any, Any]: + yield from iterable + + +class AsyncPassthroughStreamingResponse(AsyncGenerator[Any, Any]): + def __init__( + self, + response: Coroutine[Any, Any, httpx.Response], + litellm_logging_obj: LiteLLMLoggingObj, + provider_config: BasePassthroughConfig, + ) -> None: + self._initialized = False + self._status_code: int = 0 + self._headers = httpx.Headers() + self._response_coro = response + self._response: httpx.Response + self._iterator: AsyncGenerator[bytes, Any] + self._litellm_logging_obj = litellm_logging_obj + self._provider_config = provider_config + self._raw_bytes: list[bytes] = [] # mutable-ok: instance buffer for streaming chunks + self._flush_scheduled = False + self._background_tasks: set[asyncio.Task] = set() # mutable-ok: instance set for background task tracking + self._hidden_params: dict[str, object] = {} # mutable-ok: router attaches response headers here in place + + @property + def status_code(self) -> int: + if not self._initialized: + raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing status_code") + return self._status_code + + @status_code.setter + def status_code(self, value: int) -> None: + self._status_code = value + + @property + def headers(self) -> httpx.Headers: + if not self._initialized: + raise RuntimeError("AsyncPassthroughStreamingResponse must be awaited before accessing headers") + return self._headers + + @headers.setter + def headers(self, value: httpx.Headers) -> None: + self._headers = value + + def __await__(self) -> Iterator[Any]: + async def _init(): + if not self._initialized: + self._response = await self._response_coro + self.headers = self._response.headers + self.status_code = self._response.status_code + self._initialized = True + try: + self._response.raise_for_status() + self._iterator = _as_async_generator(self._response.aiter_bytes()) + except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic + try: + await self._response.aread() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + try: + await self._response.aclose() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + raise + return self + + return _init().__await__() + + def _start_flush(self) -> None: + if self._flush_scheduled or not self._raw_bytes: + return + self._flush_scheduled = True + + try: + task: Final = asyncio.create_task( + self._litellm_logging_obj.async_flush_passthrough_collected_chunks( + raw_bytes=self._raw_bytes, + provider_config=self._provider_config, + ) + ) + + # Compliant: Save a strong reference to prevent GC + self._background_tasks.add(task) + + # Remove the task from the set when it finishes to avoid memory leaks + task.add_done_callback(self._background_tasks.discard) + except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s", + len(self._raw_bytes), + e, + ) + + def __aiter__(self) -> AsyncPassthroughStreamingResponse: + return self + + def aiter_bytes(self) -> AsyncPassthroughStreamingResponse: + return self + + async def __anext__(self) -> bytes: + if not self._initialized: + await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__ + try: + chunk: Final = await anext(self._iterator) + self._raw_bytes.append(chunk) + except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic + self._start_flush() + try: + await self._response.aclose() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + raise + else: + return chunk + + async def asend(self, value: bytes) -> bytes: + if not self._initialized: + await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__ + return await self._iterator.asend(value) + + async def athrow( + self, + typ: BaseException | type[BaseException], + val: BaseException | object = None, + tb: TracebackType | None = None, + ) -> bytes: + if not self._initialized: + await self # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__ + return await self._iterator.athrow(typ, val, tb) # pyright: ignore[reportCallIssue, reportArgumentType] # matches one of the athrow overloads + + async def aclose(self) -> None: + self._start_flush() + try: + if self._initialized: + await self._iterator.aclose() + await self._response.aclose() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + + +class PassthroughStreamingResponse(Generator[Any, Any, Any]): + def __init__( + self, + response: httpx.Response, + litellm_logging_obj: LiteLLMLoggingObj, + provider_config: BasePassthroughConfig, + ) -> None: + self._response = response + self.headers = response.headers + self.status_code = response.status_code + self._litellm_logging_obj = litellm_logging_obj + self._provider_config = provider_config + self._iterator: Generator[bytes, Any, Any] = _as_generator(response.iter_bytes()) + self._raw_bytes: list[bytes] = [] # mutable-ok: instance buffer for streaming chunks + self._flush_scheduled = False + + def _start_flush(self) -> None: + if self._flush_scheduled or not self._raw_bytes: + return + self._flush_scheduled = True + + from litellm.utils import executor + + try: + executor.submit( + self._litellm_logging_obj.flush_passthrough_collected_chunks, + raw_bytes=self._raw_bytes, + provider_config=self._provider_config, + ) + except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging + verbose_logger.exception( + "Failed to schedule passthrough spend-tracking flush; %d buffered chunks dropped: %s", + len(self._raw_bytes), + e, + ) + + def __iter__(self) -> PassthroughStreamingResponse: + return self + + def __next__(self) -> bytes: + try: + chunk: Final = next(self._iterator) + self._raw_bytes.append(chunk) + except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic + self._start_flush() + try: + self._response.close() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + raise + else: + return chunk + + def send(self, value: bytes) -> bytes: + return self._iterator.send(value) + + def throw( + self, + typ: BaseException | type[BaseException], + val: BaseException | object = None, + tb: TracebackType | None = None, + ) -> bytes: + return self._iterator.throw(typ, val, tb) # pyright: ignore[reportCallIssue, reportArgumentType] # matches one of the throw overloads + + def close(self) -> None: + self._start_flush() + try: + self._response.close() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass @client @@ -37,10 +255,10 @@ async def allm_passthrough_route( api_key: str | None = None, request_query_params: dict | None = None, request_headers: dict | None = None, - content: Any | None = None, + content: RequestContent | None = None, data: dict | None = None, files: RequestFiles | None = None, - json: Any | None = None, + json: object | None = None, params: QueryParamTypes | None = None, cookies: CookieTypes | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, @@ -64,7 +282,7 @@ async def allm_passthrough_route( from litellm.utils import ProviderConfigManager provider_config = cast( - Optional["BasePassthroughConfig"], kwargs.get("provider_config") + BasePassthroughConfig | None, kwargs.get("provider_config") ) or ProviderConfigManager.get_provider_passthrough_config( provider=LlmProviders(custom_llm_provider), model=model, @@ -132,12 +350,12 @@ async def allm_passthrough_route( if resolved_custom_llm_provider: try: provider_config = cast( - Optional["BasePassthroughConfig"], kwargs.get("provider_config") + BasePassthroughConfig | None, kwargs.get("provider_config") ) or ProviderConfigManager.get_provider_passthrough_config( provider=LlmProviders(resolved_custom_llm_provider), model=model, ) - except Exception: + except Exception: # noqa: BLE001 S110 # If we can't get provider config, pass None pass @@ -162,10 +380,10 @@ def llm_passthrough_route( api_key: str | None = None, request_query_params: dict | None = None, request_headers: dict | None = None, - content: Any | None = None, + content: RequestContent | None = None, data: dict | None = None, files: RequestFiles | None = None, - json: Any | None = None, + json: object | None = None, params: QueryParamTypes | None = None, cookies: CookieTypes | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, @@ -190,7 +408,9 @@ def llm_passthrough_route( _is_async: Final = bool(kwargs.get("allm_passthrough_route", False)) - litellm_logging_obj: Final = cast("LiteLLMLoggingObj", kwargs.get("litellm_logging_obj")) + litellm_logging_obj: Final = cast( + LiteLLMLoggingObj, kwargs.get("litellm_logging_obj") + ) # cast-ok: logging obj is constructed upstream; tests inject mocks model, custom_llm_provider, api_key, api_base = get_llm_provider( model=model, @@ -235,7 +455,7 @@ def llm_passthrough_route( ) provider_config: Final = cast( - Optional["BasePassthroughConfig"], kwargs.get("provider_config") + BasePassthroughConfig | None, kwargs.get("provider_config") ) or ProviderConfigManager.get_provider_passthrough_config( provider=LlmProviders(custom_llm_provider), model=model, @@ -276,10 +496,13 @@ def llm_passthrough_route( forward_headers=False, ) + _request_data: dict | None = ( + data if isinstance(data, dict) else (json if isinstance(json, dict) else None) + ) # rebind-ok: conditional headers, signed_json_body = provider_config.sign_request( headers=headers, litellm_params=litellm_params_dict, - request_data=data if data else json, + request_data=_request_data, api_base=str(updated_url), model=model, ) @@ -301,9 +524,12 @@ def llm_passthrough_route( ) ## IS STREAMING REQUEST + _streaming_request_data: dict = ( + data if isinstance(data, dict) else (json if isinstance(json, dict) else {}) + ) # rebind-ok: conditional is_streaming_request: Final = provider_config.is_streaming_request( endpoint=endpoint, - request_data=data or json or {}, + request_data=_streaming_request_data, ) # Update logging object with streaming status @@ -334,18 +560,26 @@ def llm_passthrough_route( else: # Sync path - client.client.send returns Response directly response: httpx.Response = client.client.send(request=request, stream=is_streaming_request) - response.raise_for_status() + try: + response.raise_for_status() + except Exception: # noqa: BLE001 # Safe catch-all for cleanup logic + try: + response.read() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + try: + response.close() + except Exception: # noqa: BLE001 S110 # Safe catch-all for cleanup logic + pass + raise - if ( - hasattr(response, "iter_bytes") and is_streaming_request - ): # yield the chunk, so we can store it in the logging object - return _sync_streaming(response, litellm_logging_obj, provider_config) + if hasattr(response, "iter_bytes") and is_streaming_request: + return PassthroughStreamingResponse(response, litellm_logging_obj, provider_config) else: - # For non-streaming responses, yield the entire response return response except Exception as e: - if provider_config is None: - raise e + # provider_config is guaranteed non-None here due to the earlier guard + assert provider_config is not None raise base_llm_http_handler._handle_error( e=e, provider_config=provider_config, @@ -356,8 +590,8 @@ async def _async_passthrough_request( client: HTTPHandler | AsyncHTTPHandler, request: httpx.Request, is_streaming_request: bool, - litellm_logging_obj: "LiteLLMLoggingObj", - provider_config: "BasePassthroughConfig", + litellm_logging_obj: LiteLLMLoggingObj, + provider_config: BasePassthroughConfig, ) -> httpx.Response | AsyncGenerator[Any, Any]: """ Handle async passthrough requests. @@ -369,8 +603,7 @@ async def _async_passthrough_request( # Check if it's a coroutine and await it if asyncio.iscoroutine(response_result): if is_streaming_request: - # Pass the coroutine to _async_streaming which will await it - return _async_streaming( + return await AsyncPassthroughStreamingResponse( # pyright: ignore[reportGeneralTypeIssues] # structural type check misses __await__ response=response_result, litellm_logging_obj=litellm_logging_obj, provider_config=provider_config, @@ -383,84 +616,3 @@ async def _async_passthrough_request( else: # Fallback for sync-like behavior (shouldn't happen in async path) raise Exception("Expected coroutine from async client") - - -def _sync_streaming( - response: httpx.Response, - litellm_logging_obj: "LiteLLMLoggingObj", - provider_config: "BasePassthroughConfig", -): - from litellm.utils import executor - - raw_bytes: Final[list[bytes]] = [] - flush_scheduled = False - try: - for chunk in response.iter_bytes(): - raw_bytes.append(chunk) - yield chunk - finally: - if not flush_scheduled and raw_bytes: - flush_scheduled = True - try: - executor.submit( - litellm_logging_obj.flush_passthrough_collected_chunks, - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - except Exception as e: - verbose_logger.exception( - "Failed to schedule passthrough spend-tracking flush " - "in _sync_streaming; %d buffered chunks dropped: %s", - len(raw_bytes), - e, - ) - - -async def _async_streaming( - response: Coroutine[Any, Any, httpx.Response], - litellm_logging_obj: "LiteLLMLoggingObj", - provider_config: "BasePassthroughConfig", -): - iter_response: Final = await response - - try: - iter_response.raise_for_status() - except Exception: - try: - await iter_response.aclose() - except Exception: - pass - raise - - raw_bytes: Final[list[bytes]] = [] - flush_scheduled = False - try: - async for chunk in iter_response.aiter_bytes(): - raw_bytes.append(chunk) - yield chunk - except Exception: - try: - await iter_response.aclose() - except Exception: - pass - raise - finally: - # GeneratorExit (raised on client disconnect) is not caught by - # `except Exception`; the finally block ensures partial usage - # still gets flushed for spend tracking. See LIT-2642. - if not flush_scheduled and raw_bytes: - flush_scheduled = True - try: - asyncio.create_task( - litellm_logging_obj.async_flush_passthrough_collected_chunks( - raw_bytes=raw_bytes, - provider_config=provider_config, - ) - ) - except Exception as e: - verbose_logger.exception( - "Failed to schedule passthrough spend-tracking flush " - "in _async_streaming; %d buffered chunks dropped: %s", - len(raw_bytes), - e, - ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0f2d97b8b1c..c549f48126e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum): "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", "/openai/v1/responses/{response_id}/cancel", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", # vector stores "/vector_stores", "/v1/vector_stores", @@ -471,6 +474,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/milvus", + "/gigachat", "/watsonx", ] @@ -3549,6 +3553,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ) +class SpendLogsRouterMetadata(TypedDict): + """ + Router provenance stamped on spend logs for deployments flagged with + model_info.internal_router_model, correlating the requested model group + with the provider deployment that served the call + """ + + requested_model: ReadOnly[str | None] + selected_model: ReadOnly[str | None] + selected_provider: ReadOnly[str | None] + router_correlation_id: ReadOnly[str | None] + + class SpendLogsMetadata(TypedDict): """ Specific metadata k,v pairs logged to spendlogs for easier cost tracking @@ -3591,6 +3608,7 @@ class SpendLogsMetadata(TypedDict): compression_savings: CompressionSavingsMetadata | None autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed litellm_gateway_injected_cache: ReadOnly[str | None] + router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index a42187b3a44..64878a480a7 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -2,13 +2,14 @@ Handles Authentication Errors """ +import logging from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from fastapi import HTTPException, Request, status import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger from litellm.constants import EMPTY_MAPPING from litellm.integrations.otel.runtime import seed_request_identity from litellm.litellm_core_utils.core_helpers import is_expected_client_error @@ -18,7 +19,11 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_utils import _get_request_ip_address +from litellm.proxy.auth.auth_utils import ( + _get_request_ip_address, + is_invalid_virtual_key_error, + mark_invalid_virtual_key_error, +) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -36,6 +41,41 @@ else: Span = Any +def _as_proxy_exception(e: Exception) -> ProxyException: + """Convert an authentication failure into the ProxyException the client receives.""" + if isinstance(e, litellm.BudgetExceededError): + return ProxyException( + message=e.message, + type=ProxyErrorTypes.budget_exceeded, + param=None, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + ) + if isinstance(e, HTTPException): + return ProxyException( + message=getattr(e, "detail", f"Authentication Error({e})"), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), + ) + if isinstance(e, ProxyException): + return e + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + return ProxyException( + message=( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + return ProxyException( + message="Authentication Error, " + str(e), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=status.HTTP_401_UNAUTHORIZED, + ) + + def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]: """Auth gate rejections are raised before `add_litellm_data_to_request` records the caller IP, so their failure logs would otherwise carry no IP nor key/user identity.""" @@ -110,16 +150,21 @@ class UserAPIKeyAuthExceptionHandler: request=request, use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True, ) - log_fn: Final = ( - verbose_proxy_logger.error - if is_expected_client_error(e) and not litellm.log_client_error_tracebacks - else verbose_proxy_logger.exception - ) - log_fn( + + # Log authentication failures before identity seeding and callbacks, so the log + # survives a raising callback pipeline. Classify and route malformed virtual-key + # rejections to WARNING on stdout (suppressible via LITELLM_LOG=ERROR). + log_extra: Final = {"requester_ip": requester_ip} + is_invalid_virtual_key: Final = is_invalid_virtual_key_error(e) + is_quiet_log: Final = is_invalid_virtual_key and not litellm.log_client_error_tracebacks + logger: Final = verbose_proxy_stdout_logger if is_quiet_log else verbose_proxy_logger + logger.log( + logging.WARNING if is_quiet_log else logging.ERROR, "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", e, requester_ip, - extra={"requester_ip": requester_ip}, + exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None, + extra=log_extra, ) # Log this exception to OTEL, Datadog etc. Reuse the identity resolved @@ -167,35 +212,13 @@ class UserAPIKeyAuthExceptionHandler: if transformed_exception is not None: e = transformed_exception - if isinstance(e, litellm.BudgetExceededError): - raise ProxyException( - message=e.message, - type=ProxyErrorTypes.budget_exceeded, - param=None, - code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + final_exception: Final = mark_invalid_virtual_key_error(_as_proxy_exception(e), is_invalid_virtual_key) + # If a quiet-logged malformed-key transform yields non-401, escalate to ERROR + if is_quiet_log and str(final_exception.code) != str(status.HTTP_401_UNAUTHORIZED): + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", + final_exception, + requester_ip, + extra=log_extra, ) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), - ) - elif isinstance(e, ProxyException): - raise e - if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - raise ProxyException( - message=( - "Service Unavailable, the authentication database is " - "temporarily unreachable. Please retry shortly." - ), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_401_UNAUTHORIZED, - ) + raise final_exception diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9b1a6ba5aa7..83ee10b8108 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, EMPTY_MAPPING, + INVALID_VIRTUAL_KEY_ERROR_MARKER, MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) @@ -34,6 +35,43 @@ from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS from litellm.types.utils import CustomPricingLiteLLMParams +def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: + """True when an authentication error rejects a malformed virtual key. + + Classifies only by the marker stamped where that 401 is raised. Message + content is never inspected: other 401s interpolate caller-supplied values + (vector store ids, organization ids) into their messages, so a phrase + match would let a request body demote an authorization failure to the + quiet log path. + """ + if not isinstance(exception, (HTTPException, ProxyException)): + return False + + code: Final[object] = getattr(exception, "code", None) + status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None) + if str(status_code) != str(status.HTTP_401_UNAUTHORIZED): + return False + + return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True + + +def mark_invalid_virtual_key_error(exception: ProxyException, is_invalid_virtual_key: bool) -> ProxyException: + """Return an independently marked malformed-key exception after callback transformations.""" + if not is_invalid_virtual_key or str(exception.code) != str(status.HTTP_401_UNAUTHORIZED): + return exception + marked_exception: Final = ProxyException( + message=exception.message, + type=exception.type, + param=exception.param, + code=exception.code, + headers=exception.headers.copy(), + openai_code=None if exception.openai_code is None else str(exception.openai_code), + provider_specific_fields=exception.provider_specific_fields, + ) + setattr(marked_exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return marked_exception + + def _get_request_ip_address(request: Request, use_x_forwarded_for: bool | None = False) -> str | None: client_ip = None if use_x_forwarded_for is True and "x-forwarded-for" in request.headers: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e92d090a2fb..5fb6dad0cd7 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -19,12 +19,15 @@ import fastapi import orjson from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader +from starlette.exceptions import WebSocketException import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( GLOBAL_PROXY_SPEND_CACHE_KEY, + INVALID_VIRTUAL_KEY_ERROR_MARKER, + INVALID_VIRTUAL_KEY_ERROR_MESSAGE, LITELLM_PROXY_BUDGET_NAME, LITELLM_PROXY_MASTER_KEY_ALIAS, ) @@ -65,6 +68,7 @@ from litellm.proxy.auth.auth_utils import ( get_model_from_request, get_request_route, get_request_route_template, + is_invalid_virtual_key_error, iter_request_fallback_targets, normalize_request_route, pre_db_read_auth_checks, @@ -539,6 +543,8 @@ async def user_api_key_auth_websocket(websocket: WebSocket): try: return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}") except Exception as e: + if is_invalid_virtual_key_error(e): + raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) verbose_proxy_logger.exception(e) await websocket.close(code=status.WS_1008_POLICY_VIOLATION) raise HTTPException(status_code=403, detail=str(e)) @@ -1867,13 +1873,17 @@ async def _user_api_key_auth_builder( _masked_key: Final = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****" if not api_key.startswith("sk-"): _hint = _JWT_AUTH_DISABLED_HINT if not enable_jwt_auth and JWTHandler.is_jwt(token=api_key) else "" - raise HTTPException( + _malformed_key_error = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail=( - f"LiteLLM Virtual Key expected. Received={_masked_key}, " + f"{INVALID_VIRTUAL_KEY_ERROR_MESSAGE}. Received={_masked_key}, " f"expected to start with 'sk-'.{_hint}" ), ) # prevent token hashes from being used + # Stamp provenance here so log routing classifies this 401 by + # where it was raised, never by its message text. + setattr(_malformed_key_error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + raise _malformed_key_error else: verbose_logger.warning( "litellm.proxy.proxy_server.user_api_key_auth(): Warning - Key is not a string. Got type={}".format( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ed66bb65ced..1d860b02875 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1495,6 +1495,35 @@ class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict): self.data = data + @staticmethod + def _merge_passthrough_streaming_headers( + response_headers: httpx.Headers | dict | None, + custom_headers: dict, + ) -> dict: + """ + Merge upstream passthrough headers with proxy/custom headers. + + Proxy/custom headers win on key collisions. + """ + excluded_headers: Final = { # mutable-ok: set of header names to exclude from forwarding + "transfer-encoding", + "content-encoding", + "set-cookie", + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "upgrade", + } + + merged_headers: Final = { # mutable-ok: dict comprehension for merged headers forwarded to httpx + key: value for key, value in dict(response_headers or {}).items() if key.lower() not in excluded_headers + } + merged_headers.update(custom_headers) + return merged_headers + @staticmethod def get_custom_headers( *, @@ -2389,6 +2418,16 @@ class ProxyBaseLLMRequestProcessing: ) if route_type == "allm_passthrough_route": + upstream_response_headers: Final = getattr(response, "headers", None) + streaming_headers: Final = ( + ProxyBaseLLMRequestProcessing._merge_passthrough_streaming_headers( + response_headers=upstream_response_headers, + custom_headers=custom_headers, + ) + if upstream_response_headers is not None + else custom_headers + ) + # Check if response is an async generator if self._is_streaming_response(response): if asyncio.iscoroutine(response): @@ -2418,11 +2457,11 @@ class ProxyBaseLLMRequestProcessing: # For passthrough routes, stream directly without error parsing # since we're dealing with raw binary data (e.g., AWS event streams) - return StreamingResponse( - content=generator, - status_code=status.HTTP_200_OK, + return _UpstreamClosingStreamingResponse( + content=generator, # pyright: ignore[reportArgumentType] # generator-configured StreamingResponse + status_code=getattr(response, "status_code", status.HTTP_200_OK), media_type=self._passthrough_event_stream_media_type(), - headers=custom_headers, + headers=streaming_headers, ) else: _early = await self._handle_non_streaming_allm_passthrough_route( @@ -2437,7 +2476,7 @@ class ProxyBaseLLMRequestProcessing: return StreamingResponse( content=response.aiter_bytes(), status_code=response.status_code, - headers=custom_headers, + headers=streaming_headers, ) elif route_type == "anthropic_messages": # Check if response is actually a streaming response (async generator) diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index f9188f95cfd..8cf2f737063 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -7,20 +7,15 @@ instead of aggregating LiteLLM_SpendLogs every time a window counter goes cold (issue #35766). Raw SQL rather than the Prisma upsert helper because the conditional roll cannot be expressed through the query builder. -Seeding a row that does not exist yet reads LiteLLM_SpendLogs once, excluding -the requests whose increments are in the same batch so neither source counts -them twice. One gap survives that exclusion: without the Redis transaction -buffer every pod flushes its own increments, so a row seeded by one pod can -include spend logs whose increments are still queued on another pod, and those -increments are added again when that pod flushes. That is bounded by a single -flush interval, happens at most once per window row, and only ever over-counts: -the seed never omits spend, because every increment not yet in the row still -reaches it on its own pod's next flush. A row therefore lags real spend by at -most one flush interval of queued increments, the same lag the SpendLogs -aggregate it replaces (and every other spend column) already has. +Seeding a row that does not exist yet reads LiteLLM_SpendLogs once and takes +off what the increments being flushed will add, so neither source counts the +same request twice. A row therefore lags real spend by at most one flush +interval of increments queued elsewhere: the same lag the SpendLogs aggregate +it replaces (and every other spend column) already has. """ from collections.abc import Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Final, Protocol @@ -67,33 +62,46 @@ _ROLL_WINDOW_SPEND_SQL: Final = ( ) _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = ( - 'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" ' - "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') " - "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))" + "SELECT COALESCE(SUM(spend), 0.0) AS total, " + "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + 'FROM "LiteLLM_SpendLogs" ' + "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" ) _SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = ( - 'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" ' - "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC') " - "AND NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))" + "SELECT COALESCE(SUM(spend), 0.0) AS total, " + "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " + 'FROM "LiteLLM_SpendLogs" ' + "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" ) _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = ( - 'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" ' + "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " + 'FROM "LiteLLM_SpendLogs" ' "WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" ) _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = ( - 'SELECT COALESCE(SUM(spend), 0.0) AS total FROM "LiteLLM_SpendLogs" ' + "SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch " + 'FROM "LiteLLM_SpendLogs" ' "WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" ) _UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60) +@dataclass(frozen=True, slots=True) +class WindowSeedTotals: + """The two sums a seed needs: everything persisted for the window, and the + part of it that predates the batch being flushed.""" + + total: float + before_batch: float + + class WindowSpendLogsAggregate(Protocol): - """Sums LiteLLM_SpendLogs for one entity since window_start, ignoring the - requests whose ids are handed in. + """Sums LiteLLM_SpendLogs for one entity since window_start, split at the + batch's earliest request. Injected so the flush can be exercised without a database and so the expensive aggregate stays swappable. @@ -105,21 +113,19 @@ class WindowSpendLogsAggregate(Protocol): entity_type: str, entity_id: str, window_start: datetime, - exclude_request_ids: Sequence[str], - exclude_started_at: datetime | None, - ) -> float | None: ... + batch_started_at: datetime | None, + ) -> WindowSeedTotals | None: ... -async def spend_logs_total_excluding( +async def spend_logs_seed_totals( prisma_client: "PrismaClient", entity_type: str, entity_id: str, window_start: datetime, - exclude_request_ids: Sequence[str], - exclude_started_at: datetime | None, -) -> float | None: - """LiteLLM_SpendLogs spend for one entity since window_start, minus the - requests already accounted for by the increments being flushed. + batch_started_at: datetime | None, +) -> WindowSeedTotals | None: + """LiteLLM_SpendLogs spend for one entity since window_start, both in full + and up to the start of the batch being flushed, in one scan. The spend log writer drains its own queue on a ~2s poll whenever anything is queued, while window increments flush on the much slower batch tick, so @@ -127,13 +133,12 @@ async def spend_logs_total_excluding( already in the table. Counting them in the seed and again in the increment is what made a fresh row land at twice the true spend. - The exclusion is bounded to rows that started at or after the batch's - earliest request. request_id can be chosen by the client - (x-litellm-call-id), so an unbounded exclusion would let a replayed old id - erase a historical row from the seed while its increment still lands. - Without a known start the batch's ids are not excluded at all: that can - only over-count once, which enforcement tolerates, whereas under-counting - is a budget bypass. + Both halves are needed because neither is safe alone: the full sum + double-counts this batch, and the sum before the batch drops spend another + pod has already persisted but not yet incremented. _seed_base picks between + them. Without a known batch start the two are the same sum, so the seed + counts everything: that can only over-count once, which enforcement + tolerates, whereas under-counting is a budget bypass. """ if entity_type == Litellm_EntityType.KEY.value: bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_KEY_SQL, _SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL @@ -143,21 +148,23 @@ async def spend_logs_total_excluding( return None rows: Final = ( await prisma_client.db.query_raw(unbounded_sql, entity_id, window_start) - if exclude_started_at is None or not exclude_request_ids + if batch_started_at is None else await prisma_client.db.query_raw( bounded_sql, entity_id, window_start, - tuple(exclude_request_ids), - _exclusion_lower_bound(exclude_started_at), + _exclusion_upper_bound(batch_started_at), ) ) if not rows: - return 0.0 - return float(rows[0].get("total") or 0.0) + return WindowSeedTotals(total=0.0, before_batch=0.0) + return WindowSeedTotals( + total=float(rows[0].get("total") or 0.0), + before_batch=float(rows[0].get("before_batch") or 0.0), + ) -def _exclusion_lower_bound(started_at: datetime) -> datetime: +def _exclusion_upper_bound(started_at: datetime) -> datetime: """LiteLLM_SpendLogs.startTime is TIMESTAMP(3); floor to the second so a millisecond rounding of the batch's own earliest row cannot slip under it.""" return to_naive_utc(started_at).replace(microsecond=0) @@ -194,20 +201,33 @@ async def _seed_base_for_missing_row( This is the LiteLLM_SpendLogs aggregate the window counter reseed runs on every cold counter today, but here it runs once per window lifetime and off - the request path, and it excludes this batch's own requests so they are - counted by their increments alone. + the request path, and it discounts the queued increments so they are + counted once. """ if _primary_key(transaction) in existing_primary_keys: return 0.0 - base: Final = await spend_logs_aggregate( + totals: Final = await spend_logs_aggregate( prisma_client=prisma_client, entity_type=transaction["entity_type"], entity_id=transaction["entity_id"], window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc), - exclude_request_ids=transaction["request_ids"], - exclude_started_at=_transaction_started_at(transaction), + batch_started_at=_transaction_started_at(transaction), ) - return float(base or 0.0) + if totals is None: + return 0.0 + return _seed_base(totals=totals, batch_spend=transaction["spend"]) + + +def _seed_base(totals: WindowSeedTotals, batch_spend: float) -> float: + """What the window already held before the increments about to be applied. + + Subtracting the batch's own spend from the full sum keeps every other + request in the seed, including the ones another pod persisted and has not + incremented yet, which a plain cutoff would drop for good if that pod died. + When this batch's own log rows have not landed yet the subtraction takes + spend that was never counted, so the sum before the batch is the floor. + """ + return max(totals.total - batch_spend, totals.before_batch) def _transaction_started_at(transaction: WindowSpendTransaction) -> datetime | None: @@ -241,7 +261,7 @@ def _upsert_params( async def commit_window_spend_updates( prisma_client: "PrismaClient", transactions: Sequence[WindowSpendTransaction], - spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_total_excluding, + spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_seed_totals, ) -> None: """Apply aggregated window increments to LiteLLM_BudgetWindowSpend. diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 641c07914d9..202a95ba29b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -215,11 +215,7 @@ class DBSpendUpdateWriter: start_time: datetime | None, end_time: datetime | None, response_cost: float | None, - ) -> str | None: - """Returns the LiteLLM_SpendLogs request_id this call was recorded - under, so the caller can tell the budget-window writer which log rows - its increments already cover. None when the payload could not be built. - """ + ) -> None: from litellm.proxy.proxy_server import ( disable_spend_logs, litellm_proxy_budget_name, @@ -236,7 +232,7 @@ class DBSpendUpdateWriter: team_id, ) if ProxyUpdateSpend.disable_spend_updates() is True: - return None + return if token is not None and isinstance(token, str) and token.startswith("sk-"): hashed_token = hash_token(token=token) else: @@ -310,7 +306,6 @@ class DBSpendUpdateWriter: ) verbose_proxy_logger.debug("Runs spend update on all tables") - return payload.get("request_id") except Exception: spend_log_error( "Spend tracking - update_database failed. Spend log insertion or daily transaction enqueue " @@ -323,7 +318,7 @@ class DBSpendUpdateWriter: org_id, end_user_id, ) - return None + return async def _enqueue_tool_usage_transaction( self, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 4dd23270bf8..c06f2e04aca 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -46,6 +46,7 @@ from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdate from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendTransaction, WindowSpendUpdateQueue, + to_wire_payload, ) from litellm.secret_managers.main import str_to_bool from litellm.types.caching import ( @@ -298,7 +299,7 @@ class RedisUpdateBuffer: ServiceTypes.REDIS_DAILY_AGENT_SPEND_UPDATE_QUEUE, ), ( - window_spend_update_transactions, + tuple(map(to_wire_payload, window_spend_update_transactions)), REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY, ServiceTypes.REDIS_WINDOW_SPEND_UPDATE_QUEUE, ), @@ -484,7 +485,12 @@ class RedisUpdateBuffer: (daily_end_user_spend_update_transactions, REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY), (daily_agent_spend_update_transactions, REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY), (daily_tag_spend_update_transactions, REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY), - (window_spend_update_transactions, REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY), + ( + None + if window_spend_update_transactions is None + else tuple(map(to_wire_payload, window_spend_update_transactions)), + REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY, + ), ) rpush_list: Final = tuple( diff --git a/litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py index 04dea66165e..372a6666c02 100644 --- a/litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/window_spend_update_queue.py @@ -26,17 +26,12 @@ class WindowSpendTransaction(TypedDict): window_start is an ISO-8601 string rather than a datetime so the transaction survives the JSON round trip through the Redis buffer. - request_ids carries the LiteLLM_SpendLogs ids this spend came from. The - one-time seed for a window that has no row yet subtracts them from its - LiteLLM_SpendLogs aggregate, because the spend log writer flushes on its - own ~2s poll and will usually have persisted these rows before the window - queue flushes; without the exclusion the seed and the increment would each - count them. - - started_at is the earliest request start in the batch. The seed only - subtracts a request_id whose LiteLLM_SpendLogs.startTime is at or after it, - so a client that replays an old id through x-litellm-call-id cannot make the - seed drop the historical row that id already paid for. + started_at is the earliest request start in the batch. The one-time seed for + a window that has no row yet uses it to tell this batch's own + LiteLLM_SpendLogs rows from everything else, because the spend log writer + flushes on its own ~2s poll and will usually have persisted this batch's + rows before the window queue flushes; without that split the seed and the + increment would each count them. """ entity_type: ReadOnly[str] @@ -44,10 +39,37 @@ class WindowSpendTransaction(TypedDict): window_duration: ReadOnly[str] window_start: ReadOnly[str] spend: ReadOnly[float] - request_ids: ReadOnly[Sequence[str]] started_at: ReadOnly[str | None] +class WindowSpendWirePayload(WindowSpendTransaction): + """How an increment is encoded in the shared Redis buffer. + + request_ids is dead weight here: workers built before this field was + dropped index it while merging whatever they pop, and the pop is + destructive, so a leader still running one of those during a rolling deploy + would raise on a payload without the key and lose those increments. It is + always empty, which only makes such a leader seed without exclusions. + + TODO: remove once no supported version reads it, i.e. one release after the + field stopped being written. + """ + + request_ids: ReadOnly[Sequence[str]] + + +def to_wire_payload(transaction: WindowSpendTransaction) -> WindowSpendWirePayload: + return WindowSpendWirePayload( + entity_type=transaction["entity_type"], + entity_id=transaction["entity_id"], + window_duration=transaction["window_duration"], + window_start=transaction["window_start"], + spend=transaction["spend"], + started_at=transaction.get("started_at"), + request_ids=(), + ) + + def to_naive_utc(value: datetime) -> datetime: """LiteLLM_BudgetWindowSpend.window_start is TIMESTAMP(3), which holds naive UTC.""" if value.tzinfo is None: @@ -72,7 +94,6 @@ def build_window_spend_transaction( window_duration: str, window_start: datetime, spend: float, - request_id: str | None = None, started_at: datetime | None = None, ) -> WindowSpendTransaction: return WindowSpendTransaction( @@ -81,7 +102,6 @@ def build_window_spend_transaction( window_duration=window_duration, window_start=to_naive_utc(window_start).isoformat(timespec="microseconds"), spend=spend, - request_ids=() if request_id is None else (request_id,), started_at=None if started_at is None else to_naive_utc(started_at.astimezone(timezone.utc)).isoformat(timespec="microseconds"), @@ -101,7 +121,6 @@ def _merge_window_spend_transactions( window_duration=first["window_duration"], window_start=first["window_start"], spend=math.fsum(payload["spend"] for payload in payloads), - request_ids=tuple(sorted(frozenset(chain.from_iterable(payload["request_ids"] for payload in payloads)))), started_at=min(started_ats) if started_ats else None, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index daa15fb5e5f..d8c8c2f4974 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -433,7 +433,7 @@ class HeadroomGuardrail(CustomGuardrail): payload["model"] = model try: - raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] + raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped url=f"{self.headroom_api_base}/v1/compress", json=payload, headers=self._request_headers(), @@ -570,7 +570,7 @@ class HeadroomGuardrail(CustomGuardrail): params["query"] = query try: - raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] + raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.get is untyped url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}", params=params, headers=self._request_headers(), diff --git a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py index a6ea2e09583..68914a1989e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/hiddenlayer/hiddenlayer.py @@ -156,6 +156,31 @@ def _header_value(headers: Mapping[str, str], key: str, default: str) -> str: return headers.get(key, default) +def _is_image_part(item: object) -> bool: + """Whether a structured-message content part carries an image rather than text.""" + + if not isinstance(item, Mapping): + return False + + part: Final[Mapping[object, object]] = item + return part.get("type") == "image_url" + + +def _scannable_text(content: object) -> str: + """Flatten a structured message's content into the single string the v1 detection endpoint takes. + + Image parts are dropped: the endpoint accepts one string, so an image would only reach it as + its stringified source (a base64 blob or a URL), which is not text the scanner can evaluate. + """ + + if not isinstance(content, list): + return str(content or "") + + parts: Final[Sequence[object]] = content + text_parts: Final = [item for item in parts if not _is_image_part(item)] # mutable-ok: sent as a list repr + return str(text_parts or "") + + def is_saas(host: str) -> bool: """Checks whether the connection is to the SaaS platform""" @@ -270,7 +295,7 @@ class HiddenlayerGuardrail(CustomGuardrail): "messages": [ { "role": last_msg.get("role", "user"), - "content": str(last_msg.get("content", "")), + "content": _scannable_text(last_msg.get("content")), } ] }, diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py index fa1f9f3d36d..0aaba4016cd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/__init__.py @@ -20,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, default_on=litellm_params.default_on, + file_sanitization_fail_open=getattr(litellm_params, "file_sanitization_fail_open", None), ) litellm.logging_callback_manager.add_litellm_callback(_prompt_security_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index 809d5e0fb31..84c4f118b00 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -4,10 +4,12 @@ import os from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional +import httpx from fastapi import HTTPException from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.exceptions import Timeout as LiteLLMTimeout from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -24,6 +26,9 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS: Final = 30.0 + + class PromptSecurityGuardrailMissingSecrets(Exception): pass @@ -63,6 +68,13 @@ class _SanitizeStatusResponse(TypedDict, total=False): metadata: ReadOnly[_SanitizeMetadata] +class _SanitizeResult(TypedDict): + action: ReadOnly[str] + content: ReadOnly[str | None] + metadata: ReadOnly[_SanitizeMetadata] + violations: ReadOnly[Sequence[str]] + + class PromptSecurityGuardrail(CustomGuardrail): @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: @@ -79,6 +91,8 @@ class PromptSecurityGuardrail(CustomGuardrail): user: str | None = None, system_prompt: str | None = None, check_tool_results: bool | None = None, + file_sanitization_timeout: float = _SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS, + file_sanitization_fail_open: bool | None = None, **kwargs, ): kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) @@ -108,6 +122,8 @@ class PromptSecurityGuardrail(CustomGuardrail): # Configuration for file sanitization self.max_poll_attempts = 30 # Maximum number of polling attempts self.poll_interval = 2 # Seconds between polling attempts + self.file_sanitization_timeout = file_sanitization_timeout + self.file_sanitization_fail_open = file_sanitization_fail_open is not False super().__init__(**kwargs) @@ -397,6 +413,39 @@ class PromptSecurityGuardrail(CustomGuardrail): Sanitize file content using Prompt Security API. Returns: dict with keys 'action', 'content', 'metadata' """ + try: + return await asyncio.wait_for( + self._sanitize_file_content(file_data, filename, user_api_key_alias), + timeout=self.file_sanitization_timeout, + ) + except (asyncio.TimeoutError, httpx.TimeoutException, LiteLLMTimeout) as exc: + if not self.file_sanitization_fail_open: + verbose_proxy_logger.error( + "Prompt Security Guardrail: file sanitization for %s timed out with %s; failing closed", + filename, + type(exc).__name__, + ) + raise HTTPException(status_code=408, detail="File sanitization timeout") from exc + + verbose_proxy_logger.error( + "Prompt Security Guardrail: file sanitization for %s timed out with %s; failing open", + filename, + type(exc).__name__, + ) + fail_open_result: Final[_SanitizeResult] = { + "action": "allow", + "content": None, + "metadata": {}, + "violations": (), + } + return fail_open_result + + async def _sanitize_file_content( + self, + file_data: bytes, + filename: str, + user_api_key_alias: str | None, + ) -> _SanitizeResult: headers: Final = {"APP-ID": self.api_key} if user_api_key_alias: headers["X-LiteLLM-Key-Alias"] = user_api_key_alias diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py index d842a9b9b6a..8925cc5b3a6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py @@ -197,7 +197,7 @@ class RepelloAIGuardrail(CustomGuardrail): repelloai_response: RepelloAIAnalyzeResponse | None = None try: verbose_proxy_logger.debug("RepelloAI Argus request: %s", request) - response: Final[HttpxResponse] = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] + response: Final[HttpxResponse] = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped url=endpoint, headers={"X-API-Key": self.repelloai_api_key}, json=request, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index f02901f0e97..47aafda2337 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -587,7 +587,7 @@ async def _update_database_and_spend_counters( model_access_groups: Sequence[str] | None = None, ) -> None: try: - spend_log_request_id = await proxy_logging_obj.db_spend_update_writer.update_database( + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key, response_cost=response_cost, user_id=user_id, @@ -623,7 +623,6 @@ async def _update_database_and_spend_counters( budget_reservation=budget_reservation, end_user_id=end_user_id, tags=request_tags, - request_id=spend_log_request_id, request_started_at=start_time, model_access_groups=model_access_groups, ) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index b5533d548e5..44b0cdcca2e 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -7,7 +7,7 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone -from itertools import groupby +from itertools import chain, groupby from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol @@ -58,10 +58,11 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( ComplexityRouterConfigValidationResponse, RequestComplexityRouterConfig, ShadowEvalDirection, - ShadowEvalJobKeyResponse, ShadowEvalJobResponse, + ShadowEvalJobTargetResponse, ShadowEvalResult, ShadowEvalSlice, + ShadowEvalTargetType, StartShadowEvalRequest, ) @@ -104,10 +105,43 @@ class _VerificationTokenTable(Protocol): async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_VerificationTokenRow]: ... +class _TeamRow(Protocol): + @property + def team_id(self) -> str: ... + + @property + def team_alias(self) -> str | None: ... + + +class _TeamRowsTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ... + + +class _UserRow(Protocol): + @property + def user_id(self) -> str: ... + + @property + def user_email(self) -> str | None: ... + + +class _UserRowsTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_UserRow]: ... + + class _ShadowEvalJobRow(Protocol): @property def id(self) -> str: ... + @property + def group_id(self) -> str: ... + + @property + def target_type(self) -> str: ... + + @property + def target_id(self) -> str: ... + class _ShadowEvalJobTable(Protocol): async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... @@ -138,6 +172,14 @@ def _verification_tokens(prisma_client: "PrismaClient") -> _VerificationTokenTab return prisma_client.db.litellm_verificationtoken +def _team_rows(prisma_client: "PrismaClient") -> _TeamRowsTable: + return prisma_client.db.litellm_teamtable + + +def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable: + return prisma_client.db.litellm_usertable + + def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable: return prisma_client.db.litellm_shadowevaljob @@ -836,7 +878,7 @@ def _validate_judge_is_not_a_candidate( def _is_unique_violation(error: Exception) -> bool: - """Whether a Prisma create failed on a unique index. One active job per key and + """Whether a Prisma create failed on a unique index. One active job per target and direction lives in a partial unique index (raw SQL in the migration; schema.prisma cannot express partial indexes), so the read-then-create check above it is advisory: two concurrent starts pass the read, and the loser must surface as the same 409 @@ -885,7 +927,7 @@ _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT # direction, and mid-deploy rows from old pods price as judge-only until the deploy ends). _SWEEP_FINISHED_JOBS_SQL: Final = """ UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') -WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL +WHERE j.target_type = $2 AND j.target_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns @@ -966,10 +1008,10 @@ WHERE group_id IN ( ) """ -_LIST_LEGS_BY_KEY_SQL: Final = """ +_LIST_LEGS_BY_TARGET_SQL: Final = """ SELECT * FROM "LiteLLM_ShadowEvalJob" WHERE group_id IN ( - SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 + SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE target_type = $2 AND target_id = $3 GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int ) """ @@ -1007,15 +1049,16 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: class _LegRow(BaseModel): """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is - one key's leg of a job; the legs of a job share group_id and identical config, written - together by one create_many. The API's job id is the group id, so leg ids never leave - the server (attempts reference them internally).""" + one target's leg of a job; the legs of a job share group_id and identical config, + written together by one create_many. The API's job id is the group id, so leg ids + never leave the server (attempts reference them internally).""" model_config = ConfigDict(from_attributes=True) id: str group_id: str - api_key_id: str + target_type: ShadowEvalTargetType + target_id: str router_name: str direction: ShadowEvalDirection baseline_model: str | None = None @@ -1068,16 +1111,17 @@ def _group_response( first: Final = legs[0] return ShadowEvalJobResponse( job_id=group_id, - keys=tuple( - ShadowEvalJobKeyResponse( - api_key_id=leg.api_key_id, + targets=tuple( + ShadowEvalJobTargetResponse( + target_type=leg.target_type, + target_id=leg.target_id, max_turns=leg.max_turns, max_budget=leg.max_budget, stopped_at=leg.stopped_at, attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0, spend=round(stats.spend, 6) if stats else 0.0, ) - for leg in sorted(legs, key=lambda leg: leg.api_key_id) + for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id)) ), router_name=first.router_name, direction=first.direction, @@ -1090,34 +1134,85 @@ def _group_response( ) -_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None) +_NO_TARGET_LABELS: Final[tuple[str | None, str | None]] = (None, None) -async def _with_key_labels( +def _target_labels( + key_rows: Sequence[_VerificationTokenRow], + team_rows: Sequence[_TeamRow], + user_rows: Sequence[_UserRow], +) -> Mapping[tuple[str, str], tuple[str | None, str | None]]: + """Display labels by (target_type, target_id): a key's (alias, masked name), a + team's (alias, None), a user's (email, None).""" + return MappingProxyType( + { # mutable-ok: MappingProxyType needs a dict to wrap + key: value + for key, value in chain( + ((("key", row.token), (row.key_alias, row.key_name)) for row in key_rows), + ((("team", row.team_id), (row.team_alias, None)) for row in team_rows), + ((("user", row.user_id), (row.user_email, None)) for row in user_rows), + ) + } + ) + + +def _target_ids_of(responses: Sequence[ShadowEvalJobResponse], target_type: ShadowEvalTargetType) -> tuple[str, ...]: + return tuple( + sorted( + frozenset( + target.target_id + for response in responses + for target in response.targets + if target.target_type == target_type + ) + ) + ) + + +async def _with_target_labels( prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse] ) -> tuple[ShadowEvalJobResponse, ...]: - """Resolve every scoped key's hash to its alias and masked name in one batched read, - so the UI can say whose traffic a job shadows. Deleted keys resolve to None.""" + """Resolve every scoped target's id to a display label in one batched read per kind, + so the UI can say whose traffic a job shadows: a key's alias and masked name, a + team's alias, a user's email. Deleted targets resolve to None.""" if not responses: return () - tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys)) - key_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": tokens}} # mutable-ok: Prisma filter + tokens: Final = _target_ids_of(responses, "key") + team_ids: Final = _target_ids_of(responses, "team") + user_ids: Final = _target_ids_of(responses, "user") + key_rows: Final = ( + await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(tokens)}} # mutable-ok: Prisma filter + ) + if tokens + else () ) - labels: Final[Mapping[str, tuple[str | None, str | None]]] = { - row.token: (row.key_alias, row.key_name) for row in key_rows or () - } + team_rows: Final = ( + await _team_rows(prisma_client).find_many( + where={"team_id": {"in": list(team_ids)}} # mutable-ok: Prisma filter + ) + if team_ids + else () + ) + user_rows: Final = ( + await _user_rows(prisma_client).find_many( + where={"user_id": {"in": list(user_ids)}} # mutable-ok: Prisma filter + ) + if user_ids + else () + ) + labels: Final = _target_labels(key_rows or (), team_rows or (), user_rows or ()) return tuple( response.model_copy( update={ # mutable-ok: pydantic update payload - "keys": tuple( - key.model_copy( + "targets": tuple( + target.model_copy( update={ # mutable-ok: pydantic update payload - "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0], - "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1], + "target_alias": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[0], + "key_name": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[1], } ) - for key in response.keys + for target in response.targets ) } ) @@ -1125,29 +1220,37 @@ async def _with_key_labels( ) -async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None: - """All three stratifications of one job's verdicts. Tier answers "where does the router - do well"; the model stratification groups by whichever model served the real arm, so it - answers "which of the models these keys use today would the router beat" forward, and - "for the turns the router sent to X, did X beat the baseline" in reverse; key answers - "which key's traffic does the router suit". Reads are bounded by the job's own attempts - (<= the sum of its keys' max_turns) via the job_id index.""" +async def _shadow_eval_results( + prisma_client: "PrismaClient", legs: Sequence[_LegRow] +) -> tuple[ShadowEvalResult | None, Mapping[tuple[str, str], ShadowEvalSlice]]: + """One job's stratified verdicts, plus each target's own slice keyed by the + (target_type, target_id) pair so a key, team, and user sharing an id can never + collapse into one entry. Tier answers "where does the router do well"; the model + stratification groups by whichever model served the real arm, so it answers "which + of the models these targets use today would the router beat" forward, and "for the + turns the router sent to X, did X beat the baseline" in reverse; the per-target + slices answer "which target's traffic does the router suit". Reads are bounded by + the job's own attempts (<= the sum of its targets' max_turns) via the job_id index.""" leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or () ) if not by_tier: - return None + return None, MappingProxyType({}) by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or () ) - key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs}) + target_by_leg: Final = MappingProxyType({leg.id: (leg.target_type, leg.target_id) for leg in legs}) by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or () ) - by_key: Final = tuple( - row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload - for row in by_leg + verdicts_by_target: Final[Mapping[tuple[str, str], ShadowEvalSlice]] = MappingProxyType( + { # mutable-ok: MappingProxyType needs a dict to wrap + target_by_leg[slice.group]: slice.model_copy( + update={"group": target_by_leg[slice.group][1]} # mutable-ok: pydantic update payload + ) + for slice in _slices(by_leg) + } ) total_turns: Final = sum(r.turn_count for r in by_tier) funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids) @@ -1155,10 +1258,9 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le # Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert # failed) must read as unknown, not as job-level counts missing a leg's traffic. funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None - return ShadowEvalResult( + result: Final = ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), - by_key=_slices(by_key), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), sampled_real_spend=sum(r.real_spend for r in by_tier), @@ -1168,6 +1270,7 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le shed_count=funnel.shed if funnel is not None else None, withheld_count=funnel.withheld if funnel is not None else None, ) + return result, verdicts_by_target @router.post( @@ -1182,22 +1285,29 @@ async def start_shadow_eval( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: """ - Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against - a second arm, judge the two responses blind, and stratify win rates by tier, by the model - that served the real arm, and by key. + Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic + against a second arm, judge the two responses blind, and stratify win rates by tier, + by the model that served the real arm, and by target. - A forward job answers whether the keys should adopt router_name: it samples the requests - the router did not serve and duplicates them through it. A reverse job answers whether a - key already on the router still gains from it: it samples the requests the router did - serve and duplicates them against baseline_model. A key can hold one active job per - direction, so both questions can run at once. + A target is a virtual key, a team, or a user. Team and user targets match on the + identity every request resolves to at auth time, so they cover JWT-authenticated + traffic, which presents no virtual key; a user target samples that user's traffic + across all their teams, whether it arrives on a JWT or a key they own. - Shadow responses are never served to users. Each key samples until its recorded eval - spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's - window ends, or the job is stopped, so one key running out of budget does not end - sampling for the others; sampling changes propagate to pods within about 10 seconds. - Shadow and judge calls bill to the shadowed key but are excluded from request counts - and auto-router adoption metrics. + A forward job answers whether the targets should adopt router_name: it samples the + requests the router did not serve and duplicates them through it. A reverse job + answers whether a target already on the router still gains from it: it samples the + requests the router did serve and duplicates them against baseline_model. A target + can hold one active job per direction, so both questions can run at once, and a + request matching several jobs' targets (say its key and its team) is sampled by + each, separately budgeted. + + Shadow responses are never served to users. Each target samples until its recorded + eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the + job's window ends, or the job is stopped, so one target running out of budget does + not end sampling for the others; sampling changes propagate to pods within about 10 + seconds. Shadow and judge calls bill to the sampled request's own identity but are + excluded from request counts and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -1206,35 +1316,88 @@ async def start_shadow_eval( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name): raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router") - token_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter - ) - unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))) - if unknown: - raise HTTPException( - status_code=400, - detail=( - f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, " - "the value the key list and key info endpoints report" - ), + token_rows: Final = ( + await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter ) + if data.api_key_ids + else () + ) + team_rows: Final = ( + await _team_rows(prisma_client).find_many( + where={"team_id": {"in": list(data.team_ids)}} # mutable-ok: Prisma filter + ) + if data.team_ids + else () + ) + user_rows: Final = ( + await _user_rows(prisma_client).find_many( + where={"user_id": {"in": list(data.user_ids)}} # mutable-ok: Prisma filter + ) + if data.user_ids + else () + ) + unknown_keys: Final = sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ())) + unknown_teams: Final = sorted(frozenset(data.team_ids) - frozenset(row.team_id for row in team_rows or ())) + unknown_users: Final = sorted(frozenset(data.user_ids) - frozenset(row.user_id for row in user_rows or ())) + unknown_parts: Final = tuple( + part + for part in ( + ( + f"api_key_ids not on this proxy: {', '.join(unknown_keys)}; pass each key's token hash, " + "the value the key list and key info endpoints report" + ) + if unknown_keys + else None, + f"team_ids not on this proxy: {', '.join(unknown_teams)}" if unknown_teams else None, + f"user_ids not on this proxy: {', '.join(unknown_users)}" if unknown_users else None, + ) + if part is not None + ) + if unknown_parts: + raise HTTPException(status_code=400, detail=". ".join(unknown_parts)) # Every model check below runs once per team the job samples for, since that is the # identity the shadow and judge calls carry and therefore what the router selects on. - team_ids: Final = tuple(dict.fromkeys(row.team_id for row in token_rows or ())) + # A user target's traffic can span teams, so it validates unscoped (None); each + # sampled attempt still resolves the judge under its own request's team at eval time. + team_ids: Final = tuple( + dict.fromkeys( + ( + *(row.team_id for row in token_rows or ()), + *data.team_ids, + *((None,) if data.user_ids else ()), + ) + ) + ) _validate_plain_model(llm_router, data.judge_model, "judge_model", team_ids) if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids) _validate_judge_is_not_a_candidate(llm_router, data, team_ids) + requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = ( + *(("key", key) for key in data.api_key_ids), + *(("team", team) for team in data.team_ids), + *(("user", user) for user in data.user_ids), + ) + requested_by_type: Final[tuple[tuple[ShadowEvalTargetType, tuple[str, ...]], ...]] = tuple( + (target_type, ids) + for target_type, ids in (("key", data.api_key_ids), ("team", data.team_ids), ("user", data.user_ids)) + if ids + ) # A job whose window passed or whose budget ran out stopped sampling on its own, - # but its legs still hold their slots in the per-key, per-direction partial unique index - # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. - requested: Final = list(data.api_key_ids) # mutable-ok: query param - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested) + # but its legs still hold their slots in the per-target, per-direction partial unique + # index until stamped; free them so a new eval can start. Sweeping both directions is + # deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id + # that happens to equal a key hash never matches the other kind's slot. + for target_type, ids in requested_by_type: + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) # mutable-ok: query param claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ # mutable-ok: Prisma filter - "api_key_id": {"in": requested}, # mutable-ok: Prisma filter + "OR": [ # mutable-ok: Prisma filter + {"target_type": target_type, "target_id": {"in": list(ids)}} # mutable-ok: Prisma filter + for target_type, ids in requested_by_type + ], "direction": data.direction, "stopped_at": None, }, @@ -1244,7 +1407,7 @@ async def start_shadow_eval( status_code=409, detail=( f"Already in an active {data.direction} shadow eval job: " - + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed)) + + ", ".join(sorted(f"{row.target_type} {row.target_id} (job {row.group_id})" for row in claimed)) + ". Stop it first." ), ) @@ -1268,10 +1431,16 @@ async def start_shadow_eval( # Leg ids are minted here rather than by the DB default so the funnel seed below # writes from the same values with no read-back, which a lagging read replica # (DATABASE_URL_READ_REPLICA) could otherwise return empty. - leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids) + leg_ids: Final = tuple(str(uuid4()) for _ in requested_targets) await _shadow_eval_jobs(prisma_client).create_many( data=[ # mutable-ok: Prisma payload - {**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids) + { # mutable-ok: Prisma payload + **shared_config, + "id": leg_id, + "target_type": target_type, + "target_id": target_id, + } # mutable-ok: Prisma payload + for leg_id, (target_type, target_id) in zip(leg_ids, requested_targets) ] ) except Exception as e: @@ -1280,7 +1449,8 @@ async def start_shadow_eval( raise HTTPException( status_code=409, detail=( - f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first." + f"A requested target was claimed by another {data.direction} shadow eval job concurrently. " + "Stop it first." ), ) from e # Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so @@ -1293,18 +1463,19 @@ async def start_shadow_eval( ) except Exception as seed_err: # noqa: BLE001 # coverage is advisory; the job must still start verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err) - labels: Final = MappingProxyType({row.token: row for row in token_rows}) + labels: Final = _target_labels(token_rows or (), team_rows or (), user_rows or ()) return ShadowEvalJobResponse( job_id=group_id, - keys=tuple( - ShadowEvalJobKeyResponse( - api_key_id=api_key_id, + targets=tuple( + ShadowEvalJobTargetResponse( + target_type=target_type, + target_id=target_id, max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=data.max_budget, - key_alias=labels[api_key_id].key_alias, - key_name=labels[api_key_id].key_name, + target_alias=labels.get((target_type, target_id), _NO_TARGET_LABELS)[0], + key_name=labels.get((target_type, target_id), _NO_TARGET_LABELS)[1], ) - for api_key_id in sorted(data.api_key_ids) + for target_type, target_id in sorted(requested_targets) ), router_name=data.router_name, direction=data.direction, @@ -1324,22 +1495,29 @@ async def start_shadow_eval( ) async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], - api_key_id: Annotated[ - str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others") + target_type: Annotated[ + ShadowEvalTargetType | None, Query(description="Kind of target to filter on; requires target_id") + ] = None, + target_id: Annotated[ + str | None, Query(description="Filter to jobs that shadow this target, alone or alongside others") ] = None, limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> tuple[ShadowEvalJobResponse, ...]: - """List shadow eval jobs, newest first, each key with its attempt count so status is - accurate. Judged counts, spend, and results ride the detail endpoint only.""" + """List shadow eval jobs, newest first, each target with its attempt count so status + is accurate. Judged counts, spend, and results ride the detail endpoint only.""" from litellm.proxy.proxy_server import prisma_client _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + filter_type: Final = target_type if isinstance(target_type, str) else None + filter_id: Final = target_id if isinstance(target_id, str) else None + if (filter_type is None) != (filter_id is None): + raise HTTPException(status_code=400, detail="target_type and target_id filter together; pass both or neither") legs: Final = _LEG_ROWS.validate_python( ( - await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id) - if api_key_id + await _query_raw(prisma_client, _LIST_LEGS_BY_TARGET_SQL, limit, filter_type, filter_id) + if filter_type and filter_id else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit) ) or () @@ -1354,7 +1532,7 @@ async def list_shadow_eval_jobs( by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True ) counts: Final = await _leg_attempt_counts(prisma_client, legs) - return await _with_key_labels( + return await _with_target_labels( prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first) ) @@ -1391,16 +1569,25 @@ async def get_shadow_eval_job( where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter order={"created_at": "desc"}, # mutable-ok: Prisma order ) - labeled: Final = await _with_key_labels( + labeled: Final = await _with_target_labels( prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),) ) + results, verdicts_by_target = await _shadow_eval_results(prisma_client, legs) return labeled[0].model_copy( update={ # mutable-ok: pydantic update payload "judged_count": totals[0].judged_count if totals else 0, "error_count": totals[0].error_count if totals else 0, "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, "last_error": latest_error.error if latest_error else None, - "results": await _shadow_eval_results(prisma_client, legs), + "results": results, + "targets": tuple( + target.model_copy( + update={ # mutable-ok: pydantic update payload + "verdicts": verdicts_by_target.get((target.target_type, target.target_id)) + } + ) + for target in labeled[0].targets + ), } ) @@ -1415,8 +1602,8 @@ async def stop_shadow_eval_job( job_id: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: - """Stop an active shadow eval job, every key it scopes at once. Attempts are kept; - sampling halts within ~10s. Keys that already stopped on their own budget keep the + """Stop an active shadow eval job, every target it scopes at once. Attempts are kept; + sampling halts within ~10s. Targets that already stopped on their own budget keep the stopped_at they earned. The statement is the whole state machine: it claims the job only while a leg still samples inside the window with no stop recorded, so a racing operator, a same-instant budget spend, and a repeat stop all read the same 400 with @@ -1443,5 +1630,5 @@ async def stop_shadow_eval_job( current: Final = _group_response(job_id, legs, counts) if claimed == 0: raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") - labeled: Final = await _with_key_labels(prisma_client, (current,)) + labeled: Final = await _with_target_labels(prisma_client, (current,)) return labeled[0] diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 9ef3d2defef..d2d87331d55 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -626,11 +626,7 @@ async def update_end_user( # get non default values for key non_default_values: Final = dict[str, object]() for k, v in data_json.items(): - if v is not None and v not in ( - [], - {}, - 0, - ): # models default to [], spend defaults to 0, we should not reset these values + if v is not None and ((isinstance(v, bool) and k in data.fields_set()) or v not in ([], {}, 0)): non_default_values[k] = v ## Get end user table data ## diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 9c4c9948c10..c08ca5b7783 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1027,6 +1027,14 @@ async def user_info_v2( This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem where the old endpoint loaded all keys and teams into memory. + Note on `spend`: this is the user's running budget counter, which the budget reset job + resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default, + or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT + lifetime or per-period historical spend. For historical spend over a date range, use + `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend + records that only ever accumulate and are never reset. The two values are expected to + diverge once a budget reset has occurred within the queried period. + Access control: - Proxy admins can query any user - Team admins can query users within their teams @@ -2726,6 +2734,11 @@ async def get_user_daily_activity( Meant to optimize querying spend data for analytics for a user. + Reads daily spend records that only ever accumulate and are never affected by budget + resets. Their total can legitimately exceed the `spend` field returned by + `/v2/user/info`, which is a running budget counter that every budget reset sets back + to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). + Returns: (by date) - spend @@ -2839,6 +2852,11 @@ async def get_user_daily_activity_aggregated( """ Aggregated analytics for a user's daily activity without pagination. Returns the same response shape as the paginated endpoint with page metadata set to single-page. + + Reads daily spend records that only ever accumulate and are never affected by budget + resets. Their total can legitimately exceed the `spend` field returned by + `/v2/user/info`, which is a running budget counter that every budget reset sets back + to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). """ from litellm.proxy.proxy_server import prisma_client diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 52a192f8537..26b65dd48cf 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -19,13 +19,13 @@ import re import secrets import traceback from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence -from contextlib import AbstractAsyncContextManager from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeVar, cast import fastapi import yaml from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -155,6 +155,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: + import prisma from prisma import Prisma from prisma import models as prisma_models @@ -182,6 +183,14 @@ class _TxTables(Protocol): litellm_proxymodeltable: TableActions[object] +class _ModelParamsUpdate(TypedDict): + litellm_params: ReadOnly["prisma.Json"] + + +class _ModelRowWhere(TypedDict): + model_id: ReadOnly[str] + + class _ConfigTableActions(Protocol): """Config table surface this module needs; the shared repository seam exposes no ``update``.""" @@ -273,12 +282,6 @@ def _env_vars_param_value(param: _EnvVarsParam) -> Mapping[str, str] | None: return param.param_value -def _tx_tables_context( - open_tx: Callable[[], AbstractAsyncContextManager[_TxTables]], -) -> AbstractAsyncContextManager[_TxTables]: - return open_tx() - - async def _check_custom_key_allowed(custom_key_value: str | None) -> None: """Raise 403 if custom API keys are disabled and a custom key was provided.""" if custom_key_value is None: @@ -4484,27 +4487,29 @@ async def _rotate_master_key( if models: decrypted_models: Final = proxy_config.decrypt_model_list_from_db(new_models=models) verbose_proxy_logger.debug("ABLE TO DECRYPT MODELS - len(decrypted_models): %s", len(decrypted_models)) - new_models: Final[list[dict[str, object]]] = [] - for model in decrypted_models: - new_model = await _add_model_to_db( - model_params=Deployment(**model), - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - new_encryption_key=new_master_key, - should_create_model_in_db=False, - ) - if new_model: - _dumped = dict[str, object](_as_object_dict(new_model.model_dump(exclude_none=True))) - _dumped["litellm_params"] = prisma.Json(_dumped["litellm_params"]) - _dumped["model_info"] = prisma.Json(_dumped["model_info"]) - new_models.append(_dumped) - verbose_proxy_logger.debug("Resetting proxy model table") - async with _tx_tables_context(prisma_client.db.tx) as tx: - await tx.litellm_proxymodeltable.delete_many() - verbose_proxy_logger.debug("Creating %s models", len(new_models)) - await tx.litellm_proxymodeltable.create_many( - data=new_models, - ) + reencrypted_models: Final = tuple( + [ + reencrypted + for model in decrypted_models + if ( + reencrypted := await _add_model_to_db( + model_params=Deployment(**model), + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + new_encryption_key=new_master_key, + should_create_model_in_db=False, + ) + ) + ] + ) + verbose_proxy_logger.debug("Re-encrypting litellm_params on %s model rows", len(reencrypted_models)) + async with prisma_client.db.tx(timeout=timedelta(minutes=2)) as tx_ctx: + tx: Final[_TxTables] = tx_ctx + for reencrypted_model in reencrypted_models: + await tx.litellm_proxymodeltable.update_many( + data=_ModelParamsUpdate(litellm_params=prisma.Json(reencrypted_model.litellm_params)), + where=_ModelRowWhere(model_id=reencrypted_model.model_id), + ) await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") # 3. process config table try: diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 08346983f32..c2f5dbb4032 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -252,7 +252,7 @@ async def add_team_callbacks( Use this if if you want different teams to have different success/failure callbacks Parameters: - - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add + - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of: - "success": Callback for successful LLM calls - "failure": Callback for failed LLM calls @@ -268,6 +268,8 @@ async def add_team_callbacks( - langsmith_api_key: The API key for the Langsmith callback - langsmith_project: The project for the Langsmith callback - langsmith_base_url: The base URL for the Langsmith callback + - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400 + - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key Example curl: ``` diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 525e7099b89..1d28ad1914f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -6,6 +6,8 @@ Provider-specific Pass-Through Endpoints Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. """ +from __future__ import annotations + import hmac import json import os @@ -28,6 +30,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.handle_jwt import JWTHandler @@ -51,6 +54,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( create_websocket_passthrough_route, websocket_passthrough_request, ) +from litellm.proxy.utils import ProxyLogging as ProxyLoggingType from litellm.proxy.utils import is_known_model from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, @@ -70,13 +74,17 @@ from litellm.utils import ProviderConfigManager from .passthrough_endpoint_router import PassthroughEndpointRouter if TYPE_CHECKING: + from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.router import Router + ProxyConfig = _ProxyConfig # rebind-ok: conditional type alias +else: + ProxyConfig = Any # rebind-ok: runtime fallback + vertex_llm_base: Final = VertexBase() router: Final = APIRouter() openai_passthrough_router: Final = APIRouter() default_vertex_config: Final = None - passthrough_endpoint_router: Final = PassthroughEndpointRouter() @@ -495,8 +503,14 @@ async def milvus_proxy_route( request_body: Final = await get_request_body(request) # check collectionName - collection_name: Final = cast(str | None, request_body.get("collectionName")) - extra_headers = {} + _raw_collection_name: Final = request_body.get("collectionName") + if _raw_collection_name is not None and not isinstance(_raw_collection_name, str): + raise HTTPException( + status_code=400, + detail=f"collectionName must be a string. Got {type(_raw_collection_name).__name__}", + ) + collection_name: str | None = _raw_collection_name # rebind-ok: locally scoped conversion + extra_headers = {} # mutable-ok: dict for extra headers; rebind-ok: reassigned later from credentials base_target_url: str | None = None if not collection_name: raise HTTPException( @@ -1273,7 +1287,7 @@ def _resolve_vertex_model_from_router( vertex_location: Current vertex location (may be from URL) Returns: - Tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location) + tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location) with resolved values from router config """ if not llm_router: @@ -1702,7 +1716,7 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict: def get_vertex_pass_through_handler( - call_type: Literal["discovery", "aiplatform"], + call_type: Literal["discovery", "aiplatform"], # noqa: UP037 ) -> BaseVertexAIPassThroughHandler: if call_type == "discovery": return VertexAIDiscoveryPassThroughHandler() @@ -1726,7 +1740,7 @@ def _override_vertex_params_from_router_credentials( vertex_location: Current vertex location (from URL) Returns: - Tuple of (vertex_project, vertex_location) with overridden values if applicable + tuple of (vertex_project, vertex_location) with overridden values if applicable """ if router_credentials is None: return vertex_project, vertex_location @@ -1893,12 +1907,12 @@ async def _prepare_vertex_auth_headers( authenticated them is stripped on the credential-less branch Returns: - Tuple containing: + tuple containing: - headers: dict - Authentication headers to use - - base_target_url: Optional[str] - Updated base target URL + - base_target_url: str | None - Updated base target URL - headers_passed_through: bool - Whether headers were passed through from request - - vertex_project: Optional[str] - Updated vertex project ID - - vertex_location: Optional[str] - Updated vertex location + - vertex_project: str | None - Updated vertex project ID + - vertex_location: str | None - Updated vertex location """ vertex_llm_base: Final = VertexBase() headers_passed_through = False @@ -2546,7 +2560,7 @@ def _vertex_publisher_model_suffix(model: str) -> str: return f"{VERTEX_PUBLISHER_MODEL_PREFIX}{model.rsplit('/', 1)[-1]}" -def _get_llm_router() -> "Router | None": +def _get_llm_router() -> Router | None: from litellm.proxy.proxy_server import llm_router return llm_router @@ -2586,7 +2600,7 @@ def _resolve_vertex_live_credentials( def _build_vertex_live_setup_model_rewriter( vertex_project: str | None, vertex_location: str | None, - llm_router: "Router | None", + llm_router: Router | None, ) -> Callable[[str], str] | None: """ Rewrite the ``setup`` frame's model into the full Vertex resource path the Live API requires. @@ -2606,7 +2620,7 @@ def _build_vertex_live_setup_model_rewriter( return rewrite -def _resolve_alias_to_upstream_model(setup_model: str, llm_router: "Router | None") -> str: +def _resolve_alias_to_upstream_model(setup_model: str, llm_router: Router | None) -> str: """ The Live SDK wraps whatever the caller typed as ``models/``, so a gateway alias arrives prefixed """ @@ -2796,6 +2810,238 @@ def create_generic_websocket_passthrough_endpoint( ) +@router.api_route( + "/gigachat/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route methods + tags=["Gigachat Pass-through", "pass-through"], # mutable-ok: FastAPI route tags +) +async def gigachat_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """ + [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + ## check for streaming + request_body: Final[dict[str, object]] = await get_request_body(request) + is_router_model = False # rebind-ok: conditionally set to True when model uses router + + raw_model: Final = request_body.get("model") + model: Final = raw_model if isinstance(raw_model, str) else None + if model: + is_router_model = is_passthrough_request_using_router_model( + request_body, llm_router + ) # rebind-ok: conditionally set to True + elif any(word in endpoint for word in ("completions", "embeddings")): + raise HTTPException( + status_code=400, detail={"error": "Model is required in request body"} + ) # mutable-ok: HTTPException detail dict + + # If router model, use dedicated router passthrough handler + # This uses the same common processing path as non-router models + if model and is_router_model and llm_router: + return await handle_gigachat_passthrough_router_model( + model=model, + endpoint=endpoint, + request=request, + request_body=request_body, + fastapi_response=fastapi_response, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + + verbose_proxy_logger.debug( + "Gigachat passthrough: Using direct Gigachat model '%s' for endpoint '%s'", model, endpoint + ) + + from litellm.llms.gigachat.authenticator import get_access_token + from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL + + base_target_url: Final = get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL + request_path: Final = httpx.URL(endpoint).path + encoded_endpoint: Final = request_path if request_path.startswith("/") else f"/{request_path}" + + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint) + ) + + is_streaming_request: Final = await is_streaming_request_fn(request) + + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={"Authorization": f"Bearer {get_access_token()}"}, + is_streaming_request=is_streaming_request, + ) + return await endpoint_func( + request, + fastapi_response, + user_api_key_dict, + ) + + +async def handle_gigachat_passthrough_router_model( + model: str, + endpoint: str, + request: Request, + request_body: dict, + fastapi_response: Response, + llm_router: litellm.Router, + user_api_key_dict: UserAPIKeyAuth, + proxy_logging_obj: ProxyLoggingType, + general_settings: dict, + proxy_config: ProxyConfig, + select_data_generator: Callable, + user_model: str | None, + user_temperature: float | None, + user_request_timeout: float | None, + user_max_tokens: int | None, + user_api_base: str | None, + version: str | None, +) -> Response | StreamingResponse: + """ + Handle Gigachat passthrough for router models (models defined in config.yaml). + + Uses the same common processing path as non-router models to ensure + metadata and hooks are properly initialized. + + Args: + model: The router model name (e.g., "gigachat/gigachat-2") + endpoint: The Gigachat endpoint path (e.g., "/chat/completions") + request: The FastAPI request object + request_body: The parsed request body + llm_router: The LiteLLM router instance + user_api_key_dict: The user API key authentication dictionary + proxy_logging_obj: Proxy logging + general_settings: Proxy general settings + proxy_config: Proxy config + select_data_generator: Select data generator function + (additional args for common processing) + + Returns: + Response or StreamingResponse depending on endpoint type + """ + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + # Detect streaming based on request body + is_streaming: Final = request_body.get("stream", False) # pyright: ignore[reportUnknownVariableType] # request_body is dict[Unknown, Unknown] + + data: dict[str, Any] = await _read_request_body( + request=request + ) # mutable-ok: mutated in place by proxy pipeline; pyright: ignore[reportExplicitAny] # Any needed for proxy pipeline + if user_api_key_dict is not None: + auth_metadata: Final = { + metadata_key: value + for metadata_key, value in ( + ("user_api_key_user_id", getattr(user_api_key_dict, "user_id", None)), + ("user_api_key_team_id", getattr(user_api_key_dict, "team_id", None)), + ("user_api_key_org_id", getattr(user_api_key_dict, "org_id", None)), + ("agent_id", getattr(user_api_key_dict, "agent_id", None)), + ) + if value is not None + } + existing_metadata: Final = data.get("metadata") + data["metadata"] = { + **(existing_metadata if isinstance(existing_metadata, dict) else {}), + **auth_metadata, + } + + verbose_proxy_logger.debug( + "Gigachat router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming + ) + + # Use the common processing path (same as non-router models) + # This ensures all metadata, hooks, and logging are properly initialized + + data["model"] = model + data["method"] = request.method + data["endpoint"] = endpoint + data["json"] = request_body + data["custom_llm_provider"] = "gigachat" + + # Remove sensitive keys from data + keys: Final = [ # mutable-ok: list of keys to remove from data + "gigachat_auth_url", + "gigachat_access_token", + "gigachat_scope", + "api_base", + "api_key", + ] + for key in keys: + data.pop(key, None) + + client: Final = get_async_httpx_client( + llm_provider=LlmProviders.GIGACHAT, + params={ # mutable-ok: httpx client params + "timeout": httpx.Timeout(timeout=600.0, connect=5.0), + }, + ) + + data["client"] = client + base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) + + # Use the common passthrough processing to handle metadata and hooks + # This also handles all response formatting (streaming/non-streaming) and exceptions + try: + result = await base_llm_response_processor.base_passthrough_process_llm_request( # rebind-ok: assigned once in try block + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=model, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: # noqa: BLE001 # Safe catch-all for handle exception + # Use common exception handling + raise await base_llm_response_processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + ) + else: + if isinstance(result, StreamingResponse): + if result.headers.get("Content-Type") is None: + result.headers["Content-Type"] = "text/event-stream; charset=utf-8" + + return result + + @router.api_route( "/watsonx/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 09d3dedaafa..ddd27f0fc2a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -6,9 +6,10 @@ import posixpath import traceback from base64 import b64encode from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from itertools import groupby -from typing import Any, Final, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, TypedDict, cast from urllib.parse import urlencode, urlparse import httpx @@ -47,6 +48,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_no_callback_env_reference from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -78,7 +80,10 @@ from litellm.proxy.common_utils.http_parsing_utils import ( from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above +) from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -90,7 +95,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import Usage +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging @@ -99,6 +104,9 @@ from .upstream_usage_headers import ( apply_upstream_reported_usage, ) +if TYPE_CHECKING: + from litellm.proxy.proxy_server import ProxyConfig + router: Final = APIRouter() pass_through_endpoint_logging: Final = PassThroughEndpointLogging() @@ -752,6 +760,67 @@ def _build_passthrough_failure_request_payload( return request_payload +@dataclass(frozen=True, slots=True) +class _TeamCallbackWiring: + success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg + + +def _resolve_team_callback_wiring( + user_api_key_dict: UserAPIKeyAuth, + proxy_config: "ProxyConfig", + route_description: str, +) -> _TeamCallbackWiring: + """Resolve key/team dynamic logging callbacks for a passthrough request. + + Mirrors add_litellm_data_to_request: callback_vars are unpacked top-level + (read by initialize_standard_callback_dynamic_params) and also stamped on + the proxy-owned trusted-vars field (read by get_trusted_callback_params). + + Fails open: a callback resolution or validation error is logged at error + level and the request proceeds without dynamic callbacks, since a broken + logging config must not fail the customer's upstream call (and the + websocket is already accepted by the time this runs on that path). The + env-reference check runs here because the deprecated callback_settings + branch skips AddTeamCallback validation, and Logging.__init__ would + otherwise reject the vars mid-request. + """ + try: + callback_settings_obj: Final = _get_dynamic_logging_metadata( + user_api_key_dict=user_api_key_dict, proxy_config=proxy_config + ) + if callback_settings_obj and callback_settings_obj.callback_vars: + for ( + item + ) in callback_settings_obj.callback_vars.items(): # rebind-ok: dict.items iteration for env-ref validation + validate_no_callback_env_reference(item[0], item[1], source="key/team callback metadata") + except Exception: # noqa: BLE001 - a broken logging config must never fail the passthrough request + verbose_proxy_logger.exception( + "%s: failed to resolve team logging callbacks, continuing without them", + route_description, + ) + return _TeamCallbackWiring() + if callback_settings_obj is None: + return _TeamCallbackWiring() + callback_vars: Final = callback_settings_obj.callback_vars + success_callbacks: Final = callback_settings_obj.success_callback + failure_callbacks: Final = callback_settings_obj.failure_callback + logging_kwargs: Final = ( + None + if not callback_vars + else { # mutable-ok: Logging arg + **callback_vars, + TRUSTED_CALLBACK_VARS_FIELD: callback_vars, + } + ) + return _TeamCallbackWiring( + success_callbacks=None if success_callbacks is None else [*success_callbacks], # mutable-ok: Logging arg + failure_callbacks=None if failure_callbacks is None else [*failure_callbacks], # mutable-ok: Logging arg + logging_kwargs=logging_kwargs, + ) + + async def _log_passthrough_upstream_failure( response: httpx.Response, user_api_key_dict: UserAPIKeyAuth, @@ -845,7 +914,7 @@ async def pass_through_request( from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( PassthroughGuardrailHandler, ) - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj ######################################################### # Initialize variables @@ -930,6 +999,11 @@ async def pass_through_request( # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" start_time: Final = datetime.now() + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="pass_through_endpoint", + ) logging_obj = Logging( model=passthrough_model, messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], @@ -938,6 +1012,9 @@ async def pass_through_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="1245", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Store passthrough guardrails config on logging_obj for field targeting @@ -2022,7 +2099,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream """ from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) @@ -2055,6 +2132,11 @@ async def websocket_passthrough_request( upstream_headers[header_name] = header_value # Initialize logging object similar to HTTP passthrough + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="websocket_passthrough", + ) logging_obj: Final = Logging( model="unknown", messages=[{"role": "user", "content": "WebSocket connection"}], @@ -2063,6 +2145,9 @@ async def websocket_passthrough_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="websocket_passthrough", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Create passthrough logging payload diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index da2e09bd7f6..acdd0f99295 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2658,7 +2658,6 @@ async def increment_spend_counters( budget_reservation: dict | None = None, end_user_id: str | None = None, tags: list[str] | None = None, - request_id: str | None = None, request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, ): @@ -2733,7 +2732,6 @@ async def increment_spend_counters( window_duration=duration, window_start=key_window_start, increment=cost, - request_id=request_id, request_started_at=request_started_at, ) @@ -2777,7 +2775,6 @@ async def increment_spend_counters( window_duration=duration, window_start=team_window_start, increment=cost, - request_id=request_id, request_started_at=request_started_at, ) @@ -3005,16 +3002,15 @@ async def _enqueue_window_spend_row_update( window_duration: str, window_start: datetime | None, increment: float, - request_id: str | None, request_started_at: datetime | None, ) -> None: """Queue this request's cost against the LiteLLM_BudgetWindowSpend row for the window, so enforcement can read a maintained total instead of aggregating LiteLLM_SpendLogs. - request_id is the LiteLLM_SpendLogs id this cost was recorded under and - request_started_at its startTime; the flush uses them to keep the one-time - seed from counting a request that its increment already covers. + request_started_at is this request's LiteLLM_SpendLogs startTime; the flush + stops the one-time seed there so a request its increment already covers is + not counted twice. Enqueued even when the cache increment was skipped for a reserved counter: the reservation only pre-charged the counter, and the row still owes the @@ -3035,7 +3031,6 @@ async def _enqueue_window_spend_row_update( window_duration=window_duration, window_start=window_start, spend=increment, - request_id=request_id, started_at=request_started_at, ) ) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 4652719a23b..a746e9af326 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1318,6 +1318,68 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "GIGACHAT", + "provider_display_name": "GigaChat", + "litellm_provider": "gigachat", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "gigachat_scope", + "label": "Scope", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "select", + "options": [ + "GIGACHAT_API_PERS", + "GIGACHAT_API_B2B", + "GIGACHAT_API_CORP" + ], + "default_value": "GIGACHAT_API_PERS" + }, + { + "key": "gigachat_auth_url", + "label": "Auth URL", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "gigachat_access_token", + "label": "Access token", + "placeholder": null, + "tooltip": "Disable OAuth, provide value to authorization.", + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "GigaChat-2" + }, { "provider": "GITHUB", "provider_display_name": "Github", diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 5e56e822484..aa7595ed13d 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,14 +1,18 @@ import asyncio import json import time -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Mapping +from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args from uuid import uuid4 import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import JSONResponse +from openai.types.responses.response_create_params import ResponseInputParam from starlette.websockets import WebSocket, WebSocketDisconnect +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ModifyResponseException @@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_set_request_parsed_body, ) -from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.llms.openai import ( + REASONING_EFFORT, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, +) from litellm.types.responses.main import DeleteResponseResult +from litellm.types.utils import TokenCountResponse if TYPE_CHECKING: from litellm.router import Router @@ -35,7 +44,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _user_api_key_auth_dep: Final = Depends(user_api_key_auth) -_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags +_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags _TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( { @@ -1017,6 +1026,152 @@ async def compact_response( ) +class _ResponsesApiErrorDetail(TypedDict): + message: ReadOnly[str] + type: ReadOnly[str] + param: ReadOnly[str | None] + code: ReadOnly[str | None] + + +class _ResponsesApiErrorBody(TypedDict): + error: ReadOnly[_ResponsesApiErrorDetail] + + +class _ResponsesInputTokensResult(TypedDict): + object: ReadOnly[str] + input_tokens: ReadOnly[int] + + +class _TokenCountPayload(TypedDict): + model: ReadOnly[str] + messages: ReadOnly[tuple[Mapping[str, object], ...]] + tools: ReadOnly[object] + + +class _TokenCounter(Protocol): + def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ... + + +def _proxy_token_counter() -> _TokenCounter: + from litellm.proxy.proxy_server import token_counter + + return token_counter + + +_token_counter_dep: Final = Depends(_proxy_token_counter) + + +def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: + body: Final[_ResponsesApiErrorBody] = { + "error": { + "message": message, + "type": "invalid_request_error", + "param": param, + "code": code, + } + } + return JSONResponse(status_code=400, content=body) + + +def _missing_responses_param_response(param: str) -> JSONResponse: + return _responses_invalid_request_response( + message=f"Missing required parameter: '{param}'.", + param=param, + code="missing_required_parameter", + ) + + +def _responses_input_as_token_count_messages( + input_value: str | ResponseInputParam, + instructions: str | None, +) -> tuple[Mapping[str, object], ...]: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions} + transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=input_value, + responses_api_request=request_params, + ) + return tuple( + message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed + ) + + +@router.post( + "/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +@router.post( + "/openai/v1/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) +async def responses_input_tokens( + request: Request, + token_counter: _TokenCounter = _token_counter_dep, +): + """ + Count the input tokens of a Responses API request without calling the model. + + Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + + ```bash + curl -X POST http://localhost:4000/v1/responses/input_tokens \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o", + "input": "Hello, how are you?" + }' + ``` + + Returns: `{"object": "response.input_tokens", "input_tokens": }` + """ + data: Final = await _read_request_body(request=request) + model_name: Final = data.get("model") + input_value: Final = data.get("input") + if not isinstance(model_name, str) or not model_name: + return _missing_responses_param_response("model") + if input_value is None: + return _missing_responses_param_response("input") + if isinstance(input_value, (str, list)) and not input_value: + return _responses_invalid_request_response( + message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + param=None, + code="missing_required_parameter", + ) + + try: + payload: Final[_TokenCountPayload] = { + "model": model_name, + "messages": _responses_input_as_token_count_messages( + input_value=input_value, + instructions=data.get("instructions"), + ), + "tools": data.get("tools"), + } + token_request: Final = TokenCountRequest.model_validate(payload) + except Exception as e: + return _responses_invalid_request_response( + message=f"Invalid request for token counting: {e}", param=None, code=None + ) + + token_response: Final = await token_counter(request=token_request, call_endpoint=True) + result: Final[_ResponsesInputTokensResult] = { + "object": "response.input_tokens", + "input_tokens": token_response.total_tokens, + } + return result + + @router.post( "/v1/responses/{response_id}/cancel", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index f755c6d478b..b03122966c0 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -144,10 +144,8 @@ async def background_streaming_task( # Process streaming response following OpenAI events format # https://platform.openai.com/docs/api-reference/responses-streaming - output_items: Final = dict[str, _OutputItem]() # Track output items by ID - accumulated_text: Final = dict[ - tuple[str, int], str - ]() # Track accumulated text deltas by (item_id, content_index) + output_items: Final = dict[str, _OutputItem]() + accumulated_text: Final = dict[tuple[str, int], str]() # ResponsesAPIResponse fields to extract from response.completed usage_data = None @@ -262,7 +260,6 @@ async def background_streaming_task( if "content" in delta_item: content_list = delta_item["content"] if content_index < len(content_list): - # Update existing content part with accumulated text content_entry = content_list[content_index] if isinstance(content_entry, dict): content_entry["text"] = accumulated_text[key] diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 60223265211..01a607b68a9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 8b2a5dd9312..2d113cfe355 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -172,7 +172,14 @@ async def reserve_budget_for_request( ) -> dict | None: if valid_token is None or not RouteChecks.is_llm_api_route(route=route): return None - if route in {"/models", "/v1/models", "/utils/token_counter"}: + if route in { + "/models", + "/v1/models", + "/utils/token_counter", + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + }: return None if get_model_from_request(request_body, route, llm_router=llm_router) is None: return None diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 43709e4e6ff..9f718b7d20d 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -30,7 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import ( request_model_access_groups_from_litellm_params, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes -from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload +from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsRouterMetadata from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token from litellm.types.utils import ( @@ -93,6 +93,24 @@ def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) return hash_token(stripped) +def _get_router_metadata_for_spend_log( + metadata: Mapping[str, object] | None, + requested_model: str | None, + selected_model: str | None, + selected_provider: str | None, + router_correlation_id: str | None, +) -> SpendLogsRouterMetadata | None: + model_info: Final = metadata.get("model_info") if metadata is not None else None + if not isinstance(model_info, Mapping) or model_info.get("internal_router_model") is not True: + return None + return SpendLogsRouterMetadata( + requested_model=requested_model or None, + selected_model=selected_model or None, + selected_provider=selected_provider or None, + router_correlation_id=router_correlation_id, + ) + + def _get_spend_logs_metadata( metadata: dict | None, applied_guardrails: list[str] | None = None, @@ -109,6 +127,7 @@ def _get_spend_logs_metadata( cost_breakdown: CostBreakdown | None = None, litellm_call_id: str | None = None, autorouter_savings: float | None = None, + router_metadata: SpendLogsRouterMetadata | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -148,13 +167,17 @@ def _get_spend_logs_metadata( autorouter_savings=autorouter_savings, litellm_gateway_injected_cache=None, litellm_call_id=litellm_call_id, + router_metadata=router_metadata, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " + str(list(metadata.keys())) ) # Filter the metadata dictionary to include only the specified keys - clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__}) + clean_metadata: Final = SpendLogsMetadata( + **{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"}, + router_metadata=router_metadata, + ) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") _already_redacted: Final = ( @@ -375,6 +398,20 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs hidden_params: Final = standard_logging_payload.get("hidden_params", {}) litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + custom_llm_provider: Final = ( + kwargs.get("custom_llm_provider") + or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") + or None + ) + raw_model: Final = cast(str, kwargs.get("model") or "") + model_name: Final = ( + standard_logging_payload.get("model") if standard_logging_payload is not None else None + ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + litellm_call_id: Final = cast( + str | None, + kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), + ) + # clean up litellm metadata clean_metadata = _get_spend_logs_metadata( metadata, @@ -433,9 +470,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs autorouter_savings=( standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None ), - litellm_call_id=cast( - str | None, - kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), + litellm_call_id=litellm_call_id, + router_metadata=_get_router_metadata_for_spend_log( + metadata=metadata, + requested_model=_model_group, + selected_model=model_name, + selected_provider=custom_llm_provider, + router_correlation_id=litellm_call_id, ), ) @@ -480,15 +521,6 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs # Extract agent_id for A2A requests (set directly on model_call_details) agent_id: Final[str | None] = kwargs.get("agent_id") or metadata.get("agent_id") - custom_llm_provider: Final = ( - kwargs.get("custom_llm_provider") - or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") - or None - ) - raw_model: Final = cast(str, kwargs.get("model") or "") - model_name: Final = ( - standard_logging_payload.get("model") if standard_logging_payload is not None else None - ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) try: payload: Final[SpendLogsPayload] = SpendLogsPayload( diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py index 07f9f346d08..eff8ad1b8cb 100644 --- a/litellm/rag/ingestion/vertex_ai_ingestion.py +++ b/litellm/rag/ingestion/vertex_ai_ingestion.py @@ -186,7 +186,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): base_url: Final = get_vertex_base_url(self.location) url: Final = f"{base_url}/v1beta1/projects/{self.project_id}/locations/{self.location}/ragCorpora" - # Build request body with camelCase keys (Vertex AI API format) vector_db_config: Final = self.vector_store_config.get("vector_db_config") embedding_model: Final = self.vector_store_config.get("embedding_model") embedding_model_config: Final = ( @@ -447,7 +446,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): # Add max embedding requests per minute if specified max_embedding_qpm: Final = self.vector_store_config.get("max_embedding_requests_per_min") - # Build request body with camelCase keys (Vertex AI API format) chunking_config: Final = ( {"chunkSize": chunk_size or 1024, "chunkOverlap": chunk_overlap or 200} if chunk_size or chunk_overlap diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index f39df38d069..4a416f61e1b 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig: file_dict["file_id"] = file_id if item.get("file_data"): file_dict["file_data"] = item["file_data"] + if item.get("filename"): + file_dict["filename"] = item["filename"] new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict} if "cache_control" in item: diff --git a/litellm/router.py b/litellm/router.py index c93c1753f0e..edff8294c3e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6486,6 +6486,8 @@ class Router: **kwargs, ) elif call_type == "allm_passthrough_route": + if client: + kwargs["client"] = client return await self._ageneric_api_call_with_fallbacks( original_function=original_function, passthrough_on_no_deployment=True, @@ -9152,7 +9154,8 @@ class Router: if _deployment_on_router is not None: # deployment with this model_id exists on the router if ( - deployment.litellm_params == _deployment_on_router.litellm_params + deployment.model_name == _deployment_on_router.model_name + and deployment.litellm_params == _deployment_on_router.litellm_params and deployment.model_info == _deployment_on_router.model_info ): # No need to update diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 63ba760ff66..bc8df67cc28 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -154,6 +154,9 @@ model_list: # Fallback model if tier cannot be determined default_model: gpt-4o + + # Replace a routed model that cannot take image input (default: false) + modality_routing: true ``` ## Usage @@ -178,6 +181,25 @@ response = litellm.completion( ## Special Behaviors +### Modality-based capability routing + +The classifier reads text alone, so a request carrying an image can classify cheap and land on a +text-only model, which rejects it with a provider 400 no fallback catches. With +`modality_routing: true`, one gate inspects every decided placement: when the routed model is +explicitly declared `supports_vision: false` (deployment `model_info` first, the model cost map +otherwise; unmapped names stay routable, and a multi-deployment group must accept on every +deployment), the request is re-placed on the nearest HIGHER tier holding a capable model, with +routing plugins still applied to the re-pick, then on `default_model` (never on plugin routers +and never for a plan-floored decision), and otherwise rejected with a clear 400 naming the +router. The walk only ever goes up, so a plan-mode floor cannot be undercut; a router whose only +vision model sits below the decided tier gets the 400 and an actionable message instead. + +A same-tier re-pick keeps the decision's cause and adds `modality:image` to `signals`; a tier +change or default takeover records `cause: modality_escalation` with the displaced placement +(`modality_escalated_from:` or `modality_displaced_default_model`). Escalations are never +pinned by session affinity, and a KEPT session pin bypasses the gate entirely: a session pinned +to a text-only model keeps it even when an image arrives. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 2f4305756e9..be7653902a7 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -30,6 +30,7 @@ from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata +from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( @@ -479,6 +480,33 @@ def _last_human_ask_index( ) +def _newest_turn_is_human_ask( + messages: Sequence[Mapping[str, object]] | None, + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, +) -> bool: + """Whether the request's newest turn carries a real human ask, i.e. this is a new ask rather + than an agent loop's continuation traffic. + + Anchored on `_last_human_ask_index` so every surface's plumbing reads as a continuation: + chat-completions tool turns are role=tool, Messages-surface tool_result turns flatten to empty + human text, and a hybrid turn carrying an ask alongside a tool_result still counts as an ask. + Compared against the newest non-system message rather than the raw tail, because Claude Code + appends a system-role reminder after the human turn; that trailing plumbing is neither an ask + nor loop traffic and must not turn a fresh ask into a continuation. An unreadable request (no + messages) is treated as a continuation: there is no ask to classify, which is the same reading + `_extract_current_ask_and_system_prompt` gives it downstream. + """ + if not messages: + return False + newest_non_system: Final = next( + (index for index in range(len(messages) - 1, -1, -1) if messages[index].get("role") != "system"), + None, + ) + if newest_non_system is None: + return False + return _last_human_ask_index(messages, marker_pairs) == newest_non_system + + def _iter_system_scope_texts( body_system: object, messages: Sequence[Mapping[str, object]], @@ -706,11 +734,25 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo of the three: an agent names the conversation on its first turn, so the cheapest tier would be the pin every session starts with, and the real work that follows would run there for the whole TTL. It describes what that one call is, never what the session's traffic looks like. + + A context-window escalation describes the prompt's size, not the session's complexity, and + size shrinks again the moment the client compacts: pinning the escalated tier would hold the + session on the big-window model long after the oversized context that forced it is gone. The + gate re-fires per request, so leaving these unpinned costs nothing but the classifier call. + + A modality escalation is transient the same way: it describes what this one call carries (an + image), not what the session's traffic looks like, and pinning it would hold every following + text turn on the vision-capable model the image forced. """ - return decision is None or decision.get("cause") not in ( - "default_model_fallback", - "plan_mode", - "housekeeping", + return decision is None or ( + decision.get("cause") + not in ( + "default_model_fallback", + "plan_mode", + "housekeeping", + "modality_escalation", + ) + and not decision.get("context_escalated") ) @@ -759,6 +801,39 @@ class ClassificationOutcome(NamedTuple): classifier_cost: float | None = None +def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]: + return models if fit_filter is None else tuple(model for model in models if model in fit_filter) + + +def _apply_context_placement( + tier: ComplexityTier | str, signals: tuple[str, ...], placement: _ContextWindowPlacement | None +) -> tuple[ComplexityTier | str, tuple[str, ...], ComplexityTier | str | None]: + """(final tier, signals, original tier when the gate escalated, else None).""" + if placement is None: + return tier, signals, None + if _tier_name(placement.tier) == _tier_name(tier): + return placement.tier, signals, None + return placement.tier, (*signals, "context_escalation"), tier + + +def _window_can_hold(window: int | None, needed: int, buffer: float) -> bool: + return window is None or needed <= int(window * buffer) + + +def _group_provably_fits(facts: tuple[int | None, bool], needed: int, buffer: float) -> bool: + window, has_unknown = facts + return window is not None and not has_unknown and needed <= int(window * buffer) + + +class _ContextWindowPlacement(NamedTuple): + """Where the context-window gate placed the request: the placement tier, the subset of its + pool the pick may use, and every configured group not provably misfit (the adaptive filter).""" + + tier: ComplexityTier | str + allowed_models: tuple[str, ...] + holdable_models: frozenset[str] + + class _SessionAffinityPin(NamedTuple): model: str tier: ComplexityTier | None @@ -1195,6 +1270,7 @@ class ComplexityRouter(CustomLogger): classifier_cost: float | None = None, conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, + context_escalation_original_tier: ComplexityTier | str | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1244,6 +1320,12 @@ class ComplexityRouter(CustomLogger): decision["classifier_model"] = classifier_model if classifier_cost is not None: decision["classifier_cost"] = classifier_cost + if context_escalation_original_tier is not None: + # The pair travels together: the flag says the gate moved the request off its + # decided tier on prompt size, and the original tier names where the decision + # (classifier, keyword rule, or session pin) had placed it before physics did. + decision["context_escalated"] = True + decision["context_escalation_original_tier"] = _tier_name(context_escalation_original_tier) if tier_litellm_params: masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): @@ -1644,7 +1726,7 @@ class ComplexityRouter(CustomLogger): return entry.litellm_params if entry is not None else MappingProxyType({}) @staticmethod - def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str: + def _pick_from_tier_value(model: str | Sequence[str], tier_key: str) -> str: if isinstance(model, str): return model if not model: @@ -1660,15 +1742,21 @@ class ComplexityRouter(CustomLogger): raw_messages: list[dict[str, Any]] | None, resolved_messages: list[dict[str, Any]] | None, request_kwargs: dict, + allowed_models: tuple[str, ...] | None = None, ) -> str: if not self.config.plugins: + if allowed_models is not None: + return self._pick_from_tier_value(allowed_models, _tier_name(tier)) return self.get_model_for_tier(tier) from litellm.types.router import RoutingContext tier_key: Final = _tier_name(tier) metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs) - pool: Final = tuple(self._tier_pools().get(tier_key, ())) + full_pool: Final = tuple(self._tier_pools().get(tier_key, ())) + pool: Final = ( + tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool + ) if not pool: # Nothing for the plugins to filter. Falling through would raise the # plugin-filtering error below and send the operator hunting for a policy @@ -1762,6 +1850,7 @@ class ComplexityRouter(CustomLogger): request_kwargs: dict[str, Any] | None = None, hard_floor: ComplexityTier | str | None = None, hard_ceiling: ComplexityTier | str | None = None, + fit_filter: frozenset[str] | None = None, ) -> str: """hard_floor excludes every candidate whose tiers all sit below it, turning this pick's soft floors (a distance penalty a high-scoring cheap model can outweigh) into a hard @@ -1774,7 +1863,10 @@ class ComplexityRouter(CustomLogger): tier because that is all it is worth, so a bandit trading cost for quality has nothing to win and must not reach above it. Without it the distance penalty is the only thing holding the tier, and a deployment that lowers tier_distance_penalty silently gets the expensive - model back while the routing decision still reads as the cheapest tier.""" + model back while the routing decision still reads as the cheapest tier. + + fit_filter excludes candidates the context-window gate proved cannot hold the prompt, + in every phase including cold start and the tier fallbacks.""" from litellm.router_strategy.adaptive_router.bandit import ( normalized_cost, thompson_sample, @@ -1785,12 +1877,12 @@ class ComplexityRouter(CustomLogger): if adaptive is None or not isinstance(classified_tier, ComplexityTier): # Custom tier names have no severity index; adaptive is rejected alongside # tier_definitions, so this guard is the contract for any future caller. - return self.get_model_for_tier(classified_tier) + return self._fitting_tier_fallback(classified_tier, fit_filter) request_type: Final = classify_prompt(user_message) classified_idx: Final = TIER_SEVERITY_ORDER.index(classified_tier) pools: Final = self._tier_pools() - classified_candidates: Final = tuple(pools.get(_tier_name(classified_tier), ())) + classified_candidates: Final = _allowed(tuple(pools.get(_tier_name(classified_tier), ())), fit_filter) cold_start_candidates: Final = tuple( model for model in classified_candidates if adaptive._cells[(request_type, model)].total_samples == 0 ) @@ -1820,9 +1912,9 @@ class ComplexityRouter(CustomLogger): if self.config.adaptive_eligible == "classified_tier": candidates = list(classified_candidates) if not candidates: - return self.get_model_for_tier(classified_tier) + return self._fitting_tier_fallback(classified_tier, fit_filter) else: - candidates = list(adaptive.config.available_models) + candidates = list(_allowed(tuple(adaptive.config.available_models), fit_filter)) all_costs: Final = [adaptive.model_to_cost.get(m, 0.0) for m in candidates] quality_weight: Final = self.config.adaptive_weights.quality @@ -1869,7 +1961,7 @@ class ComplexityRouter(CustomLogger): best_score = score best_model = model if best_model is None: - return self.get_model_for_tier(classified_tier) + return self._fitting_tier_fallback(classified_tier, fit_filter) if request_kwargs is not None: metadata = request_kwargs.setdefault("metadata", {}) if isinstance(metadata, dict): @@ -1886,6 +1978,12 @@ class ComplexityRouter(CustomLogger): } return best_model + def _fitting_tier_fallback(self, classified_tier: ComplexityTier | str, fit_filter: frozenset[str] | None) -> str: + fitting: Final = _allowed(tuple(self._tier_pools().get(_tier_name(classified_tier), ())), fit_filter) + if fit_filter is not None and fitting: + return self._pick_from_tier_value(fitting, _tier_name(classified_tier)) + return self.get_model_for_tier(classified_tier) + def _resolve_plan_mode_floor(self) -> ComplexityTier | str | None: """The configured floor as an active tier: the built-in enum member, or the defined name itself for a custom tier set; None when the feature is off.""" @@ -1956,6 +2054,163 @@ class ComplexityRouter(CustomLogger): return None return name if self.config.has_custom_tiers else ComplexityTier(name) + def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None: + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + + deployment_model_info: Final = deployment.get("model_info") + declared: Final = ( + deployment_model_info.get("max_input_tokens") if isinstance(deployment_model_info, Mapping) else None + ) + if isinstance(declared, int): + return declared + litellm_params: Final = deployment.get("litellm_params") + params: Final = litellm_params if isinstance(litellm_params, Mapping) else EMPTY_MAPPING + provider_override: Final = params.get("custom_llm_provider") + # get_router_model_info resolves the provider, and get_llm_provider runs the OAuth device + # flow for github_copilot/chatgpt, so a metadata question must never reach it for those. + if declared_authenticating_provider( + str(params.get("model") or ""), provider_override if isinstance(provider_override, str) else None + ): + return None + try: + model_info: Final = self.litellm_router_instance.get_router_model_info( + deployment=cast(dict, deployment), # cast-ok: router deployments are plain dicts + received_model_name=group, + ) + window: Final = model_info.get("max_input_tokens") + except Exception: # noqa: BLE001 # best-effort: an unmappable deployment must not hide the others + return None + return window if isinstance(window, int) else None + + def _group_window_facts(self, group: str) -> tuple[int | None, bool]: + """(smallest declared context window across the group's deployments, whether any deployment + declares none). The core router picks a deployment within the group without a fit check, so + the group is only as safe as its smallest member.""" + list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None) + deployments: Final = list_models(model_name=group) if callable(list_models) else None + if not isinstance(deployments, list) or not deployments: + return (None, True) + windows: Final = tuple( + window for deployment in deployments if (window := self._deployment_window(group, deployment)) is not None + ) + return (min(windows) if windows else None, len(windows) < len(deployments)) + + @staticmethod + def _out_of_band_request_text(request_kwargs: Mapping[str, object]) -> str: + """Prompt content the resolved message list never carries: the Responses API's + `instructions`, the /v1/messages top-level `system` block, and tool definitions. + A coding agent's context is dominated by these.""" + import json + + instructions: Final = request_kwargs.get("instructions") + proxy_request: Final = request_kwargs.get("proxy_server_request") + body: Final = proxy_request.get("body") if isinstance(proxy_request, Mapping) else None + system: Final = body.get("system") if isinstance(body, Mapping) else None + tools: Final = ( + body.get("tools") if isinstance(body, Mapping) and body.get("tools") else request_kwargs.get("tools") + ) + tools_text = "" + if tools: + try: + tools_text = json.dumps(tools, default=str) + except (TypeError, ValueError): + tools_text = str(tools) + return ( + (instructions if isinstance(instructions, str) else "") + + (str(system) if system is not None else "") + + tools_text + ) + + def _request_byte_upper_bound( + self, resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object] + ) -> int: + """UTF-8 byte length of all prompt content. BPE emits at least one byte per token in every + script, so the token count never exceeds this and 'bytes fit' soundly skips counting.""" + content_bytes: Final = sum(len(str(m.get("content") or "").encode()) for m in resolved_messages or ()) + return content_bytes + len(self._out_of_band_request_text(request_kwargs).encode()) + + async def _counted_request_tokens( + self, resolved_messages: Sequence[Mapping[str, object]], request_kwargs: Mapping[str, object] + ) -> int | None: + """Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the + event loop; None when counting fails, and the gate then leaves the placement alone.""" + import litellm + from litellm.litellm_core_utils.asyncify import asyncify + + out_of_band: Final = self._out_of_band_request_text(request_kwargs) + try: + counted: Final = await asyncify(litellm.token_counter)( + messages=cast(list, resolved_messages) # cast-ok: token_counter only iterates the sequence + ) + return counted + (await asyncify(litellm.token_counter)(text=out_of_band) if out_of_band else 0) + except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request + verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e) + return None + + async def _context_window_placement( + self, + tier: ComplexityTier | str, + resolved_messages: Sequence[Mapping[str, object]] | None, + request_kwargs: Mapping[str, object], + pool_override: tuple[str, ...] | None = None, + ) -> _ContextWindowPlacement | None: + """Correct a decided placement whose models provably cannot hold the prompt, or None + (the placement stands). Only a real tokenizer count ever moves a request, escalation + lands only on groups whose every deployment declares a fitting window, and a group + with no resolvable window is never moved on faith in either direction.""" + if not self.config.enable_context_window_escalation or not resolved_messages: + return None + pools: Final = self._tier_pools() + pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) + if not pool: + return None + facts: Final = MappingProxyType({group: self._group_window_facts(group) for group in pool}) + known_windows: Final = tuple(window for window, _ in facts.values() if window is not None) + if not known_windows: + return None + buffer: Final = self.config.context_window_escalation_buffer + if self._request_byte_upper_bound(resolved_messages, request_kwargs) <= int(min(known_windows) * buffer): + return None + needed: Final = await self._counted_request_tokens(resolved_messages, request_kwargs) + if needed is None: + return None + return self._placement_for_tokens(tier=tier, pool=pool, pools=pools, facts=facts, needed=needed) + + def _placement_for_tokens( + self, + *, + tier: ComplexityTier | str, + pool: tuple[str, ...], + pools: Mapping[str, list[str]], + facts: Mapping[str, tuple[int | None, bool]], + needed: int, + ) -> _ContextWindowPlacement | None: + buffer: Final = self.config.context_window_escalation_buffer + in_tier: Final = tuple(group for group in pool if _window_can_hold(facts[group][0], needed, buffer)) + if in_tier and len(in_tier) == len(pool): + return None + holdable: Final = frozenset( + group + for tier_pool in pools.values() + for group in tier_pool + if _window_can_hold(self._group_window_facts(group)[0], needed, buffer) + ) + if in_tier: + return _ContextWindowPlacement(tier=tier, allowed_models=in_tier, holdable_models=holdable) + for name in self.config.tier_names()[self._active_tier_severity(tier) + 1 :]: + proven = tuple( + group + for group in pools.get(name, ()) + if _group_provably_fits(self._group_window_facts(group), needed, buffer) + ) + if proven: + return _ContextWindowPlacement( + tier=name if self.config.has_custom_tiers else ComplexityTier(name), + allowed_models=proven, + holdable_models=holdable, + ) + return None + def _apply_plan_mode_floor(self, tier: ComplexityTier | str) -> ComplexityTier | str: """The higher of the decided tier and the plan-mode floor; identity when the floor is unset.""" floor: Final = self._resolve_plan_mode_floor() @@ -2025,6 +2280,175 @@ class ComplexityRouter(CustomLogger): return pinned_model return self.get_model_for_tier(escalated_tier) + def _model_accepts_image_input(self, model_name: str) -> bool: + """Whether a routed model or pool entry can serve an image request. + + Resolved through the deployments that would actually serve the name; a name with no + deployment on the router is served by the SDK directly and is checked against the model + cost map itself. Only an explicit supports_vision false excludes, a deployment-level + model_info override first and the map otherwise, so unmapped custom names stay routable. + + A multi-deployment group must accept on EVERY deployment: the router picks a deployment + inside the group after this gate runs, so a mixed group marked eligible could still hand + the image to its text-only member and fail with the exact 400 the gate exists to prevent. + """ + from litellm.utils import is_vision_explicitly_disabled + + def deployment_accepts(deployment: Mapping[str, Any]) -> bool: + declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision") + if declared is not None: + return declared is True + litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name + return not is_vision_explicitly_disabled(litellm_model) + + deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name) + if not deployments: + return not is_vision_explicitly_disabled(model_name) + return all(deployment_accepts(deployment) for deployment in deployments) + + def _modality_eligible_models(self) -> frozenset[str]: + """Every configured pool entry, plus default_model, that can serve an image request.""" + names: Final = frozenset(entry for pool in self._tier_pools().values() for entry in pool) | frozenset( + name for name in (self.config.default_model,) if name + ) + return frozenset(name for name in names if self._model_accepts_image_input(name)) + + async def _gate_response_modality( + self, + response: PreRoutingHookResponse, + messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + resolved_messages: Sequence[Mapping[str, object]] | None, + request_kwargs: dict, # mutable-ok: same shape the hook receives + ) -> PreRoutingHookResponse: + """Replace a routed model that cannot accept this request's image input. + + The single modality owner, applied to the decided response at the hook's exits so every + routing path is covered uniformly. A KEPT session pin is exempt by design (its cause); + replacement picks and every other path are just responses. The re-placement walks + UPWARD-ONLY from the decision's tier (so a plan-mode floor can never be undercut), picks + through `_pick_model_for_tier` so routing plugins still apply, then falls to + default_model (never on plugin routers, and never on a plan-floored decision, since + default_model carries no tier guarantee), else raises the clear 400. The rewritten + decision keeps its cause on a same-tier repick and becomes modality_escalation when the + tier moved or default_model took over, with the displaced placement in signals. + """ + decision: Final = response.routing_decision + if ( + not self.config.modality_routing + or not resolved_messages + or response.model is None + or (decision is not None and decision.get("cause") == "session_affinity_pin") + or not request_contains_image_content(resolved_messages) + or self._model_accepts_image_input(response.model) + ): + return response + eligible: Final = self._modality_eligible_models() + names: Final = self.config.tier_names() + pools: Final = self._tier_pools() + decided: Final = decision.get("tier") if decision is not None else None + start: Final = names.index(decided) if isinstance(decided, str) and decided in names else 0 + capable: Final = next( + (name for name in names[start:] if any(entry in eligible for entry in pools.get(name, ()))), None + ) + if capable is not None: + new_tier: ComplexityTier | str | None = capable if self.config.has_custom_tiers else ComplexityTier(capable) + repick_messages: Final = list(resolved_messages) # mutable-ok: the pick's param is list-typed + new_model = await self._pick_model_for_tier( + new_tier, + messages, + repick_messages, # pyright: ignore[reportArgumentType] # hook-resolved message dicts; the pick only reads them + request_kwargs, + allowed_models=tuple(entry for entry in pools.get(capable, ()) if entry in eligible), + ) + elif self._modality_default_model_usable(request_kwargs, resolved_messages, eligible): + new_tier = None + new_model = self._placed_default_model() + else: + import litellm + + raise litellm.BadRequestError( + message=( + f"Auto-router {self.model_name} received a request with image input, but no model " + f"at or above the decided tier accepts images and modality_routing is enabled. " + f"Tiers checked: {', '.join(names[start:])}. Add a vision-capable model to a tier, " + f"or set a vision-capable default_model, or remove the image content." + ), + model=self.model_name, + llm_provider="", + ) + self._restamp_adaptive_choice(request_kwargs, response.model, new_model) + same_tier: Final = capable is not None and decided == capable + base_cause: Final = (decision.get("cause") if decision is not None else None) or "default_fallback" + displaced_default: Final = decided is None and response.model == self.config.default_model + markers: Final = ( + "modality:image", + *((f"modality_escalated_from:{decided}",) if not same_tier and isinstance(decided, str) else ()), + *(("modality_displaced_default_model",) if not same_tier and displaced_default else ()), + ) + old_signals: Final = tuple(decision.get("signals") or ()) if decision is not None else () + new_decision: Final = self._build_routing_decision( + routed_model=new_model, + cause=base_cause if same_tier else "modality_escalation", + tier=new_tier, + score=decision.get("score") if decision is not None else None, + signals=(*old_signals, *markers), + matched_keyword=decision.get("matched_keyword") if decision is not None else None, + escalation_keyword=decision.get("escalation_keyword") if decision is not None else None, + escalated=bool(decision.get("escalated", False)) if decision is not None else False, + classifier_model=decision.get("classifier_model") if decision is not None else None, + classifier_cost=decision.get("classifier_cost") if decision is not None else None, + conversation_continuing=bool(decision.get("conversation_continuing", True)) + if decision is not None + else True, + tier_litellm_params=self._litellm_params_for_model(new_tier, new_model), + context_escalation_original_tier=( + decision.get("context_escalation_original_tier") if decision is not None else None + ), + ) + from litellm.types.router import PreRoutingHookResponse as HookResponse + + return HookResponse( + model=new_model, + messages=response.messages, + litellm_params=self._litellm_params_for_model(new_tier, new_model), + routing_decision=new_decision, + ) + + def _modality_default_model_usable( + self, + request_kwargs: Mapping[str, object], + resolved_messages: Sequence[Mapping[str, object]] | None, + eligible: frozenset[str], + ) -> bool: + """default_model may serve a gated request only when it is configured, plugin-free + (it is never checked against the plugin pipeline), capability-eligible, and the turn + carries no plan-mode sentinel. The sentinel is re-detected here rather than read off + the decision record, because the record only marks turns the floor RAISED; a sentinel + turn already at or above the floor keeps its ordinary cause, and default_model carries + no tier the floor could vouch for on any sentinel turn.""" + return ( + bool(self.config.default_model) + and not self.config.plugins + and self.config.default_model in eligible + and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None + ) + + def _placed_default_model(self) -> str: + """The default_model behind a usable-default verdict; the raise is the type-level + proof, not a reachable path.""" + model: Final = self.config.default_model + if model is None: + raise ValueError(f"Auto-router {self.model_name}: modality gate routed to an unset default_model") + return model + + @staticmethod + def _restamp_adaptive_choice(request_kwargs: Mapping[str, object], old_model: str, new_model: str) -> None: + """The adaptive feedback loop reads its chosen-model marker from request metadata; a + gate rewrite must move the marker with the model or rewards land on the displaced one.""" + metadata: Final = request_kwargs.get("metadata") + if isinstance(metadata, dict) and metadata.get("adaptive_router_chosen_model") == old_model: + metadata["adaptive_router_chosen_model"] = new_model + def _lexical_tier_override(self, user_message: str) -> KeywordOverride | None: """When keyword_tier_rules match literally, the most-severe matched tier wins. @@ -2247,14 +2671,18 @@ class ComplexityRouter(CustomLogger): @property def _uses_tier_pin(self) -> bool: - return bool(self.config.session_affinity and not self.config.plugins) + """classification_mode 'user_turn' implies the tier pin machinery: the pin write after each + pinnable classification is what gives a continuation a held decision to replay.""" + return bool( + (self.config.session_affinity or self.config.classification_mode == "user_turn") and not self.config.plugins + ) @property def _uses_deployment_pin(self) -> bool: - """session_affinity implies the deployment pin: a session frozen onto one model + """The tier pin implies the deployment pin: a session frozen onto one model group but load-balanced across its deployments would still go cache-cold, which is the exact failure both flags exist to prevent.""" - return bool((self.config.deployment_affinity or self.config.session_affinity) and not self.config.plugins) + return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin def _with_session_deployment_affinity( self, response: PreRoutingHookResponse | None @@ -2282,6 +2710,11 @@ class ComplexityRouter(CustomLogger): pins the model chosen on the session's first turn and reuses it for every later turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`. + When `classification_mode` is 'user_turn', the same pin is replayed only on + continuation turns (an agent loop's tool traffic); a new human ask always falls + through to classification, so the session can still move tiers between asks. + With both knobs on, session_affinity's pin-first behavior wins. + Skipped entirely when `plugins` are configured: reusing a stale pin would bypass the plugin pipeline on every turn after the first, since a pinned model was never re-checked against a policy plugin whose decision can change between turns (e.g. a @@ -2305,7 +2738,13 @@ class ComplexityRouter(CustomLogger): session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None - if cache_key is not None: + # In 'user_turn' mode a held pin is replayed only on continuation turns; a new human + # ask falls through and re-classifies. session_affinity restores pin-first for asks too. + pin_replay_allowed: Final = bool(self.config.session_affinity) or not _newest_turn_is_human_ask( + resolved_messages, self._reminder_markers + ) + + if cache_key is not None and pin_replay_allowed: pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key) pinned_pin: Final = _parse_session_affinity_pin(pinned_value) if pinned_pin is not None: @@ -2339,6 +2778,26 @@ class ComplexityRouter(CustomLogger): session_model: Final = routed_model if plan_floored and pinned_tier is not None: routed_model = self.get_model_for_tier(self._apply_plan_mode_floor(pinned_tier)) + pin_source_tier: Final = self._tier_for_model(routed_model) + pin_placement: Final = ( + await self._context_window_placement( + pin_source_tier, resolved_messages, request_kwargs, pool_override=(routed_model,) + ) + if pin_source_tier is not None + else None + ) + pin_context_original_tier: Final = ( + pin_source_tier + if pin_placement is not None + and pin_source_tier is not None + and _tier_name(pin_placement.tier) != _tier_name(pin_source_tier) + else None + ) + if pin_placement is not None and pin_context_original_tier is not None: + # The stored pin below keeps the session's own model on purpose. + routed_model = self._pick_from_tier_value( + pin_placement.allowed_models, _tier_name(pin_placement.tier) + ) # Refresh the TTL on every hit so an active session doesn't lose its # pin mid-conversation just because it outlives the original write. await self.litellm_router_instance.cache.async_set_cache( @@ -2354,36 +2813,47 @@ class ComplexityRouter(CustomLogger): kwargs_metadata: Final = request_kwargs.setdefault("metadata", {}) if isinstance(kwargs_metadata, dict): kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = routed_model + replay_cause: Final[RoutingDecisionCause] = ( + "session_affinity_pin" if self.config.session_affinity else "user_turn_continuation" + ) cause: RoutingDecisionCause = ( - "plan_mode" - if plan_floored - else ("session_affinity_escalation" if escalated else "session_affinity_pin") + "plan_mode" if plan_floored else ("session_affinity_escalation" if escalated else replay_cause) ) verbose_router_logger.info( "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model ) - routed_pin_tier: Final = self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier + routed_pin_tier: Final = ( + pin_placement.tier + if pin_placement is not None and pin_context_original_tier is not None + else (self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier) + ) session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( - PreRoutingHookResponse( - model=routed_model, - messages=messages if has_original_messages else None, - litellm_params=session_tier_litellm_params, - routing_decision=self._build_routing_decision( - routed_model=routed_model, - cause=cause, - tier=routed_pin_tier, - matched_keyword=pin_plan_sentinel if plan_floored else None, - escalation_keyword=pin_escalation_keyword, - escalated=escalated, - conversation_continuing=conversation_continuing, - tier_litellm_params=session_tier_litellm_params, + await self._gate_response_modality( + PreRoutingHookResponse( + model=routed_model, + messages=messages if has_original_messages else None, + litellm_params=session_tier_litellm_params, + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause=cause, + tier=routed_pin_tier, + matched_keyword=pin_plan_sentinel if plan_floored else None, + escalation_keyword=pin_escalation_keyword, + escalated=escalated, + conversation_continuing=conversation_continuing, + tier_litellm_params=session_tier_litellm_params, + context_escalation_original_tier=pin_context_original_tier, + ), ), + messages, + resolved_messages, + request_kwargs, ) ) - response: Final = await self._classify_and_route( + routed_response: Final = await self._classify_and_route( model=model, request_kwargs=request_kwargs, messages=messages, @@ -2392,6 +2862,11 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, resolved_messages=resolved_messages, ) + response: Final = ( + await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs) + if routed_response is not None + else None + ) # Sentinel presence, not the plan_mode cause, gates the pin write: a plan-mode turn # classified at or above the floor keeps its ordinary cause, yet on an adaptive router # the hard floor constrained its pick, so pinning it would carry a plan-mode-shaped @@ -2573,6 +3048,8 @@ class ComplexityRouter(CustomLogger): plan_floored: Final = tier != pre_floor_tier if plan_floored: signals = (*signals, "plan_mode_floor") + context_placement: Final = await self._context_window_placement(tier, resolved_messages, request_kwargs) + tier, signals, context_original_tier = _apply_context_placement(tier, signals, context_placement) score_repr: Final = f"{score:.3f}" if score is not None else "n/a" fallback_model: Final = self.config.default_model if not self.config.plugins else None # A sentinel-carrying request skips the failure exit below, whether or not the floor @@ -2619,8 +3096,15 @@ class ComplexityRouter(CustomLogger): # the cheapest tier would then contradict the floor and bound the pick below the tier # the decision reports. housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None + # A context-escalated tier becomes the hard floor: a floor the bandit can slide + # under is not a floor. routed_model = self._soft_floor_pick( - tier, user_message, request_kwargs, hard_floor=plan_floor, hard_ceiling=housekeeping_ceiling + tier, + user_message, + request_kwargs, + hard_floor=tier if context_original_tier is not None else plan_floor, + hard_ceiling=housekeeping_ceiling, + fit_filter=context_placement.holdable_models if context_placement is not None else None, ) adaptive: Final = self._ensure_adaptive_router() if adaptive is not None: @@ -2637,7 +3121,13 @@ class ComplexityRouter(CustomLogger): routed_model, ) else: - routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs) + routed_model = await self._pick_model_for_tier( + tier, + messages, + resolved_messages, + request_kwargs, + allowed_models=context_placement.allowed_models if context_placement is not None else None, + ) verbose_router_logger.info( "ComplexityRouter: routing decision cause=%s, tier=%s, score=%s, signals=%s, routed_model=%s", outcome.cause, @@ -2690,5 +3180,6 @@ class ComplexityRouter(CustomLogger): classifier_model=classifier_model, classifier_cost=outcome.classifier_cost, tier_litellm_params=tier_litellm_params, + context_escalation_original_tier=context_original_tier, ), ) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 335de11e669..70aeecb31c6 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -823,6 +823,44 @@ class ComplexityRouterConfig(BaseModel): ), ) + enable_context_window_escalation: bool = Field( + default=True, + description=( + "Escalate a request off a tier whose models provably cannot hold its prompt, before " + "dispatch. The classifier scores complexity and never prompt size, so a long agentic " + "session whose newest ask is trivial lands on a small-window tier and the provider " + "rejects it with a context-window 400 that nothing retries. When every model of the " + "decided tier has a declared window smaller than the estimated prompt, the request " + "moves to the lowest configured tier with a model whose declared window fits; when " + "only some of the tier's models fit, the pick is restricted to those and the tier " + "keeps the request. Models with no resolvable window are never escalated away from " + "and never escalated onto. Set false to dispatch on complexity alone, as before." + ), + ) + context_window_escalation_buffer: float = Field( + default=0.95, + gt=0, + le=1, + description=( + "Fraction of a model's declared context window the estimated prompt must fit within. " + "The token count is an estimate, so fitting against the full window would dispatch " + "prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that " + "drift plus the response tokens." + ), + ) + modality_routing: bool = Field( + default=False, + description=( + "Route image-bearing requests only to models that can accept image input. The " + "classifier reads text alone, so an image request whose text classifies cheap " + "otherwise lands on a text-only model and fails with a provider 400. When enabled, " + "a routed model explicitly declared supports_vision false (deployment model_info " + "or the model cost map; unmapped names stay routable) is replaced by the nearest " + "HIGHER tier holding a capable model, then default_model, else a clear 400. A kept " + "session-affinity pin still wins even when an image arrives." + ), + ) + # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching semantic_keyword_matching: bool = Field( default=False, @@ -839,6 +877,21 @@ class ComplexityRouterConfig(BaseModel): description="Minimum cosine similarity for a semantic keyword match", ) + classification_mode: Literal["every_request", "user_turn"] = Field( + default="every_request", + description=( + "When to run the complexity classifier. 'every_request' (the default) classifies every " + "inference request, including the tool-result continuation turns of an agentic loop. " + "'user_turn' classifies only requests whose newest turn is a new human ask and replays " + "the session's held routing decision on continuation turns, which cuts classifier " + "spend and eliminates mid-loop model switches. Continuations with no held decision to " + "replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike " + "session_affinity, a new human ask always re-classifies, so a session can still move " + "tiers between asks. Suppressed when plugins are configured, for the same reason " + "session_affinity is: a replayed decision would bypass the plugin pipeline." + ), + ) + # Session affinity: pin the first turn's routed model for the rest of the session session_affinity: bool = Field( default=False, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a6115640d78..fcade835cce 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2162,6 +2162,42 @@ class OpenAIRealtimeDoneEvent(TypedDict): type: Literal["response.done"] +class OpenAIRealtimeInputAudioBufferSpeechEvent(TypedDict): + type: ReadOnly[Literal["input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + + +class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict): + type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + content_index: ReadOnly[int] + delta: ReadOnly[str] + + +class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict): + type: ReadOnly[Literal["conversation.item.input_audio_transcription.completed"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + content_index: ReadOnly[int] + transcript: ReadOnly[str] + + +class OpenAIRealtimeUsageTokenDetails(TypedDict): + audio_tokens: ReadOnly[int] + text_tokens: ReadOnly[int] + cached_tokens: NotRequired[ReadOnly[int]] + + +class OpenAIRealtimeResponseUsage(TypedDict): + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + input_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] + output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2199,6 +2235,9 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeOutputItemDone | OpenAIRealtimeFunctionCallArgumentsDone | OpenAIRealtimeDoneEvent + | OpenAIRealtimeInputAudioBufferSpeechEvent + | OpenAIRealtimeInputAudioTranscriptionDelta + | OpenAIRealtimeInputAudioTranscriptionCompleted ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index bde3f5f9e7e..dfc45ccf9bf 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -245,6 +245,8 @@ ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"] ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"] +ShadowEvalTargetType: TypeAlias = Literal["key", "team", "user"] + DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" # Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that @@ -253,16 +255,37 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 class StartShadowEvalRequest(BaseModel): - """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" + """Start duplicating one or more targets' traffic for blind comparison against an auto-router. + + A target is a virtual key, a team, or a user; each becomes its own leg with its own + budget and stop state. Team and user targets match on the identity every request + carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover + JWT-authenticated traffic, which presents no virtual key at all.""" api_key_ids: tuple[str, ...] = Field( - min_length=1, + default=(), max_length=100, description=( - "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " - "keys' traffic; requests made with any other key are not sampled. Each key carries its own " - "max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 " - "keys per job, which also bounds every read the job's endpoints make." + "Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job " + "needs at least one target and at most 100, which also bounds every read the job's endpoints make. " + "Each target carries its own max_budget spend budget, so one exhausting its budget leaves the " + "others sampling." + ), + ) + team_ids: tuple[str, ...] = Field( + default=(), + max_length=100, + description=( + "Teams whose traffic will be shadowed, matched on the team every authenticated request resolves " + "to, so a team's JWT-auth and virtual-key traffic are both sampled" + ), + ) + user_ids: tuple[str, ...] = Field( + default=(), + max_length=100, + description=( + "Users whose traffic will be shadowed, matched on the user every authenticated request resolves " + "to across all their teams: JWT requests carrying their subject claim and virtual keys they own" ), ) router_name: str = Field(description="The auto-router under evaluation, in either direction") @@ -285,7 +308,7 @@ class StartShadowEvalRequest(BaseModel): shadow_percentage: float = Field( ge=0.1, le=100.0, - description="Percentage of the key's requests to duplicate through the router", + description="Percentage of each target's requests to duplicate through the router", ) judge_model: str = Field( default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL, @@ -306,9 +329,9 @@ class StartShadowEvalRequest(BaseModel): ge=0.01, le=10_000, description=( - "Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " - "the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval " - "spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight " + "Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " + "the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval " + "spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight " "samples can overshoot the cap by one sampling cache window" ), ) @@ -319,7 +342,7 @@ class StartShadowEvalRequest(BaseModel): """Pydantic ignores unknown fields, so a caller still sending max_turns would silently run on the default dollar budget instead of the bound they asked for.""" if isinstance(values, Mapping) and "max_turns" in values: - raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend") + raise ValueError("max_turns was replaced by max_budget, the per-target USD cap on the eval's own spend") return values @field_validator("shadow_percentage") @@ -327,12 +350,21 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) - @field_validator("api_key_ids") + @field_validator("api_key_ids", "team_ids", "user_ids") @classmethod - def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]: - """A key named twice would collide with itself on the one-active-per-(key, direction) index.""" + def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]: + """A target named twice would collide with itself on the one-active-per-(target, direction) index.""" return tuple(dict.fromkeys(value)) + @model_validator(mode="after") + def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest": + total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids) + if total < 1: + raise ValueError("at least one target is required: pass api_key_ids, team_ids, or user_ids") + if total > 100: + raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids") + return self + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -343,8 +375,9 @@ class StartShadowEvalRequest(BaseModel): class ShadowEvalSlice(BaseModel): - """Judge outcomes for one slice of a job's verdicts (a router tier, or one of the - models that served the real arm).""" + """Judge outcomes for one slice of a job's verdicts: a router tier, one of the + models that served the real arm, or one scoped target (embedded on that target's + own entry, so slices never need re-joining to a target by id).""" group: str turn_count: int @@ -395,12 +428,6 @@ class ShadowEvalResult(BaseModel): "and in reverse the models the router itself picked" ) ) - by_key: tuple[ShadowEvalSlice, ...] = Field( - description=( - "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job " - "scopes but has not judged a turn for yet are absent rather than reported as zero" - ), - ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float sampled_real_spend: float = Field( @@ -436,27 +463,28 @@ class ShadowEvalResult(BaseModel): ) -class ShadowEvalJobKeyResponse(BaseModel): - """One key a job shadows, with its own budget and stop state.""" +class ShadowEvalJobTargetResponse(BaseModel): + """One target a job shadows (a key, team, or user), with its own budget and stop state.""" - api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") + target_type: ShadowEvalTargetType = Field(description="What kind of entity this entry scopes") + target_id: str = Field(description="The hashed virtual key, team id, or user id whose traffic this entry scopes") max_turns: int = Field( description=( - "This key's sample-count ceiling: the whole budget for jobs created before max_budget " + "This target's sample-count ceiling: the whole budget for jobs created before max_budget " "existed, and the error-loop safety valve otherwise" ) ) max_budget: float | None = Field( default=None, description=( - "This key's own USD budget for the eval's shadow and judge spend, independent of its " + "This target's own USD budget for the eval's shadow and judge spend, independent of its " "siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds" ), ) stopped_at: datetime | None = Field( default=None, description=( - "When this key's slot was stamped free, whether its own budget ran out, the window closed, " + "When this target's slot was stamped free, whether its own budget ran out, the window closed, " "or an operator stopped the job; status is derived, so a spent budget reads completed even " "while this is still unset" ), @@ -464,45 +492,53 @@ class ShadowEvalJobKeyResponse(BaseModel): attempt_count: int | None = Field( default=None, description=( - "This key's sampled attempts so far, judged and errored alike, the same count the sampler " + "This target's sampled attempts so far, judged and errored alike, the same count the sampler " "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at " - "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" + "once the target is stamped, so in-flight attempts landing after a stop never reclassify it" ), ) spend: float | None = Field( default=None, description=( - "This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets " + "This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets " "against max_budget; populated on list and detail responses and frozen at stopped_at " "exactly like attempt_count" ), ) + verdicts: "ShadowEvalSlice | None" = Field( + default=None, + description="This target's own judged-verdict slice; detail endpoint only, None until a turn is judged", + ) + @property def budget_spent(self) -> bool: over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns) - key_alias: str | None = Field( + target_alias: str | None = Field( default=None, - description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted", + description=( + "Display label resolved from the target's own row at read time: the key's alias, the team's " + "alias, or the user's email; None when unset or deleted" + ), ) key_name: str | None = Field( default=None, - description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias", + description="Masked display name (sk-...) for key targets, resolved at read time; None for teams and users", ) class ShadowEvalJobResponse(BaseModel): - """A shadow-eval job over one or more keys, each with its own budget and stop state; - status is derived from stopped_by, the keys' stop and budget state, and ends_at, + """A shadow-eval job over one or more targets, each with its own budget and stop state; + status is derived from stopped_by, the targets' stop and budget state, and ends_at, never stored, so no writer anywhere can produce an inconsistent one. Aggregate fields are populated by the detail endpoint only and stay None on list responses.""" job_id: str - keys: tuple[ShadowEvalJobKeyResponse, ...] = Field( + targets: tuple[ShadowEvalJobTargetResponse, ...] = Field( min_length=1, - description="The keys whose traffic this job evaluates, and only those keys', each with its own budget", + description="The targets whose traffic this job evaluates, and only theirs, each with its own budget", ) router_name: str direction: ShadowEvalDirection = "forward" @@ -531,8 +567,8 @@ class ShadowEvalJobResponse(BaseModel): def status(self) -> ShadowEvalStatus: """Three recorded facts, no history-guessing: a stop is stopped_by (the migration backfills it for every job that displayed stopped when the column arrived, so the - pre-column population is closed), completion is the window passing or every key - spending its budget, and anything else is running. The all-keys-stamped fallback + pre-column population is closed), completion is the window passing or every target + spending its budget, and anything else is running. The all-targets-stamped fallback covers only stops written by pre-column pods during a rolling deploy.""" if self.stopped_by is not None: return "stopped" @@ -540,8 +576,8 @@ class ShadowEvalJobResponse(BaseModel): self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) ): return "completed" - if all(key.budget_spent for key in self.keys): + if all(target.budget_spent for target in self.targets): return "completed" - if all(key.stopped_at is not None for key in self.keys): + if all(target.stopped_at is not None for target in self.targets): return "stopped" return "running" diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py b/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py index 6e64f0f47a5..94f8161f44e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/prompt_security.py @@ -12,6 +12,10 @@ class PromptSecurityGuardrailConfigModel(GuardrailConfigModel): default=None, description="The API base for the Prompt Security guardrail. If not provided, the `PROMPT_SECURITY_API_BASE` environment variable is used.", ) + file_sanitization_fail_open: bool = Field( + default=True, + description="Whether file sanitization timeouts allow the original file through instead of blocking the request.", + ) @staticmethod def ui_friendly_name() -> str: diff --git a/litellm/types/router.py b/litellm/types/router.py index 97bd93f3f47..ab6c807ba20 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -189,6 +189,11 @@ class ModelInfo(MirroredPricingParams): # router-wide default. enable_tag_filtering: bool | None = None + # when True, calls routed to this deployment persist a router_metadata block + # (requested model group, selected model + provider, router correlation id) + # in the spend log row's metadata. Set it on every deployment of the group. + internal_router_model: bool | None = None + def __init__(self, id: str | int | None = None, **params) -> None: if id is None: id = str(uuid.uuid4()) # Generate a UUID if id is None or not provided diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4bf8289d725..55a32989b1c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2840,8 +2840,17 @@ RoutingDecisionCause = Literal[ # never called. The matched sentinel rides in matched_keyword. Distinct from the keyword causes, # which are operator-authored rules; these sentinels ship with the router. "housekeeping", + # modality_routing replaced the decided placement: the request carries an image and the + # routed model does not accept image input, so the nearest higher capable tier or + # default_model served instead. The displaced placement rides in signals. + "modality_escalation", "session_affinity_pin", "session_affinity_escalation", + # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new + # human ask), so the session's held routing decision was replayed and the classifier was never + # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning + # every turn including new asks; this cause only appears when session_affinity is off. + "user_turn_continuation", "default_fallback", "keyword", "quality_tier", @@ -2881,6 +2890,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str classifier_cost: float escalated: bool + context_escalated: bool # writable-ok: Pydantic warns on ReadOnly TypedDict fields + context_escalation_original_tier: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries reasoning_override_min_score: float # writable-ok: Pydantic warns on ReadOnly TypedDict fields conversation_continuing: bool @@ -2907,6 +2918,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "classifier_model", "classifier_cost", "escalated", + "context_escalated", + "context_escalation_original_tier", "tier_boundaries", "reasoning_override_min_score", "conversation_continuing", diff --git a/litellm/utils.py b/litellm/utils.py index d1a8309e8b3..e2b6059db06 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2662,10 +2662,19 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, ``_supports_factory`` so caching, fallback, and normalisation improvements apply here automatically. """ + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + model = model.removeprefix( + f"{declared}/" + ) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow + custom_llm_provider = declared # rebind-ok: same + else: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) val: Final = model_info.get(key) if val is False: @@ -2753,6 +2762,15 @@ def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> ) +def is_vision_explicitly_disabled(model: str, custom_llm_provider: str | None = None) -> bool: + """True only when supports_vision is explicitly declared false for the model. + + The opt-out mirror of :func:`supports_vision`: a missing declaration reads as not + disabled, so unknown or newly added models stay eligible for image routing. + """ + return _is_explicitly_disabled_factory(model, custom_llm_provider, "supports_vision") + + def supports_vision(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports vision and return a boolean value. @@ -8832,6 +8850,12 @@ class ProviderConfigManager: ) return AzurePassthroughConfig() + elif LlmProviders.GIGACHAT == provider: + from litellm.llms.gigachat.passthrough.transformation import ( + GigaChatPassthroughConfig, + ) + + return GigaChatPassthroughConfig() elif LlmProviders.WATSONX == provider: from litellm.llms.watsonx.passthrough.transformation import ( WatsonxPassthroughConfig, diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 6335e6f6bd8..c6d1b0cc46e 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -35,7 +35,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/ # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \ + apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done @@ -56,7 +56,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ --extra extra_proxy \ - --python python3 + --python python3.13 # Stage 2 — copy source and install the project + workspace members. COPY . . @@ -65,7 +65,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ --extra extra_proxy \ - --python python3 + --python python3.13 COPY migrations/run.py /app/run.py @@ -87,7 +87,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python3 nodejs libsndfile libatomic && break; \ + apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 05c1cfd3179..718e6c489fd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -553,6 +553,26 @@ "supports_response_schema": true, "supports_vision": true }, + "amazon.nova-sonic-v1:0": { + "input_cost_per_audio_token": 3.4e-06, + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.36e-05, + "output_cost_per_token": 2.4e-07, + "supports_audio_input": true, + "supports_audio_output": true + }, + "amazon.nova-2-sonic-v1:0": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2.75e-06, + "supports_audio_input": true, + "supports_audio_output": true + }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, @@ -3726,7 +3746,7 @@ "output_cost_per_token": 0, "litellm_provider": "azure_ai", "mode": "chat", - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-services/", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" }, "azure/eu/gpt-4o-2024-08-06": { @@ -5328,7 +5348,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", - "source": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/gpt-realtime-whisper", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -9107,7 +9127,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/Cohere-embed-v3-multilingual": { @@ -9118,7 +9138,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/cohere.cohere-embed-v3-english-offer?tab=PlansAndPrice", "supports_embedding_image_input": true }, "azure_ai/FLUX-1.1-pro": { @@ -9134,7 +9154,7 @@ "litellm_provider": "azure_ai", "mode": "image_generation", "output_cost_per_image": 0.04, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/images/generations" ] @@ -9467,7 +9487,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9481,7 +9501,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.04e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true, "supports_vision": true @@ -9494,7 +9514,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 7.1e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.llama-3-3-70b-instruct-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -9543,7 +9563,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { @@ -9554,7 +9574,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { @@ -9566,7 +9586,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.1e-07, - "source": "https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, "azure_ai/Phi-3-medium-128k-instruct": { @@ -9756,7 +9776,7 @@ "supported_endpoints": [ "/v1/ocr" ], - "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", @@ -9967,7 +9987,7 @@ "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, - "source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice", "supported_endpoints": [ "/v1/embeddings" ], @@ -10151,7 +10171,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, - "source": "https://azure.microsoft.com/en-us/products/ai-services/ai-foundry/models/jais-30b-chat" + "source": "https://ai.azure.com/catalog/models/jais-30b-chat" }, "azure_ai/jamba-instruct": { "input_cost_per_token": 5e-07, @@ -10208,7 +10228,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10231,7 +10251,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10243,7 +10263,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", "supports_function_calling": true, "supports_tool_choice": true }, @@ -10280,7 +10300,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", + "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice", "supports_function_calling": true }, "azure_ai/mistral-small": { @@ -12315,7 +12335,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "claude-haiku-4-5-20251001": { "deprecation_date": "2026-10-15", @@ -17767,7 +17788,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "prompt_cache_min_tokens": 1024 + "prompt_cache_min_tokens": 1024, + "deprecation_date": "2027-01-08" }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -19564,6 +19586,61 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "friendliai/zai-org/GLM-5.3-Flash": { + "litellm_provider": "friendliai", + "supports_reasoning": true, + "supports_function_calling": true, + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 3e-08, + "supports_prompt_caching": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "mode": "chat", + "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks", + "source": "https://api.friendli.ai/serverless/v1/models", + "supports_vision": true, + "supports_image_input": true, + "supports_video_input": true + }, + "friendliai/zai-org/GLM-5.3": { + "litellm_provider": "friendliai", + "supports_reasoning": true, + "supports_function_calling": true, + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.26e-06, + "output_cost_per_token": 3.96e-06, + "cache_read_input_token_cost": 2.34e-07, + "supports_prompt_caching": true, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "mode": "chat", + "comment": "Flagship GLM model for long-horizon coding, agents, and complex project delivery", + "source": "https://api.friendli.ai/serverless/v1/models", + "supports_vision": false, + "supports_image_input": false + }, "ft:babbage-002": { "deprecation_date": "2026-10-23", "input_cost_per_token": 1.6e-06, @@ -22843,7 +22920,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22897,7 +22974,7 @@ "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -22962,7 +23039,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23021,7 +23098,7 @@ "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23226,7 +23303,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23310,7 +23387,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23373,7 +23450,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -23430,7 +23507,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -24342,7 +24419,7 @@ "supports_response_schema": true, "supports_vision": true }, - "gigachat/GigaChat-2-Lite": { + "gigachat/GigaChat-2": { "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 128000, @@ -24404,6 +24481,15 @@ "output_cost_per_token": 0.0, "output_vector_size": 2560 }, + "gigachat/GigaEmbeddings-3B-2025-09": { + "input_cost_per_token": 0.0, + "litellm_provider": "gigachat", + "max_input_tokens": 4096, + "max_tokens": 4096, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 2048 + }, "gmi/anthropic/claude-opus-4.5": { "input_cost_per_token": 5e-06, "litellm_provider": "gmi", @@ -25327,7 +25413,7 @@ "supports_vision": true }, "gpt-4o-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 2.5e-06, "litellm_provider": "openai", @@ -25650,7 +25736,7 @@ "supports_vision": true }, "gpt-4o-mini-audio-preview": { - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 1.5e-07, "litellm_provider": "openai", @@ -25688,7 +25774,7 @@ "gpt-4o-mini-realtime-preview": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -25787,7 +25873,8 @@ "output_cost_per_token": 5e-06, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-4o-mini-tts": { "input_cost_per_token": 2.5e-06, @@ -25809,7 +25896,7 @@ }, "gpt-4o-realtime-preview": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25828,7 +25915,7 @@ }, "gpt-4o-realtime-preview-2024-12-17": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25847,7 +25934,7 @@ }, "gpt-4o-realtime-preview-2025-06-03": { "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2027-01-20", + "deprecation_date": "2026-05-07", "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -25926,7 +26013,8 @@ "output_cost_per_token": 1e-05, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, @@ -27077,7 +27165,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/gpt-5.6-cyber", + "source": "https://developers.openai.com/api/docs/models/gpt-5.6-cyber", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27116,7 +27204,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-red-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-red-latest", "supports_computer_use": true, "supports_parallel_function_calling": true }, @@ -27156,7 +27244,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "source": "https://platform.openai.com/docs/models/daybreak-blue-latest", + "source": "https://developers.openai.com/api/docs/models/daybreak-blue-latest", "supports_parallel_function_calling": true }, "chat-latest": { @@ -27168,7 +27256,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, - "source": "https://platform.openai.com/docs/models/chat-latest", + "source": "https://developers.openai.com/api/docs/models/chat-latest", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -30463,7 +30551,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30504,7 +30592,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30545,7 +30633,7 @@ "search_context_size_low": 0.0025, "search_context_size_medium": 0.0025 }, - "source": "https://dev.meta.ai/docs/getting-started/pricing-rate-limits", + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses", @@ -30578,7 +30666,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30594,7 +30682,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text" ], @@ -30610,7 +30698,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -30627,7 +30715,7 @@ "max_output_tokens": 4028, "max_tokens": 4028, "mode": "chat", - "source": "https://llama.developer.meta.com/docs/models", + "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ "text", "image" @@ -32572,7 +32660,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { "max_tokens": 164000, @@ -32584,7 +32672,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { "max_tokens": 128000, @@ -32595,7 +32683,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { "max_tokens": 128000, @@ -32606,7 +32694,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { "max_tokens": 128000, @@ -32617,7 +32705,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { "max_tokens": 128000, @@ -32629,7 +32717,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, @@ -32640,7 +32728,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { "max_tokens": 128000, @@ -32650,7 +32738,7 @@ "output_cost_per_token": 6e-08, "litellm_provider": "nebius", "mode": "chat", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { "max_tokens": 128000, @@ -32661,7 +32749,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { "max_tokens": 128000, @@ -32672,7 +32760,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { "max_tokens": 128000, @@ -32683,7 +32771,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { "max_tokens": 128000, @@ -32694,7 +32782,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { "max_tokens": 128000, @@ -32705,7 +32793,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { "max_tokens": 128000, @@ -32716,7 +32804,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { "max_tokens": 131072, @@ -32727,7 +32815,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { "max_tokens": 262144, @@ -32738,7 +32826,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { "max_tokens": 32768, @@ -32749,7 +32837,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-30B-A3B": { "max_tokens": 32768, @@ -32760,7 +32848,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { "max_tokens": 32768, @@ -32771,7 +32859,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { "max_tokens": 32768, @@ -32782,7 +32870,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { "max_tokens": 32768, @@ -32794,7 +32882,7 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { "max_tokens": 128000, @@ -32805,7 +32893,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { "max_tokens": 128000, @@ -32816,7 +32904,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { "max_tokens": 32768, @@ -32827,7 +32915,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { "max_tokens": 131072, @@ -32839,7 +32927,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { "max_tokens": 131072, @@ -32851,7 +32939,7 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { "max_tokens": 131072, @@ -32862,7 +32950,7 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-en-icl": { "max_tokens": 32768, @@ -32871,7 +32959,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { "max_tokens": 8192, @@ -32880,7 +32968,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { "max_tokens": 32768, @@ -32889,7 +32977,7 @@ "output_cost_per_token": 0.0, "litellm_provider": "nebius", "mode": "embedding", - "source": "https://nebius.com/prices-ai-studio" + "source": "https://nebius.com/prices" }, "nvidia.nemotron-nano-12b-v2": { "input_cost_per_token": 2e-07, @@ -33630,7 +33718,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33643,7 +33731,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33656,7 +33744,7 @@ "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true @@ -33713,7 +33801,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": true, "supports_response_schema": false, "supports_native_streaming": true, @@ -33727,7 +33815,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_function_calling": false, "supports_response_schema": false, "supports_native_streaming": true @@ -33738,7 +33826,7 @@ "max_input_tokens": 512, "mode": "embedding", "output_vector_size": 1024, - "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { @@ -34635,7 +34723,7 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2e-07, - "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", + "source": "https://openrouter.ai/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, "openrouter/deepseek/deepseek-chat": { @@ -34870,7 +34958,7 @@ "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -34911,7 +34999,7 @@ "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, "rpm": 2000, - "source": "https://ai.google.dev/pricing/gemini-3", + "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -35996,7 +36084,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/deepseek-r1-distill-llama-70b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36010,7 +36098,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llama-3-1-8b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36023,7 +36111,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-1-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": false, "supports_tool_choice": false @@ -36036,7 +36124,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/meta-llama-3-3-70b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36049,7 +36137,7 @@ "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-7b-instruct-v0-3", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36062,7 +36150,7 @@ "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-nemo-instruct-2407", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true @@ -36075,7 +36163,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mistral-small-3-2-24b-instruct-2506", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -36089,7 +36177,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mixtral-8x7b-instruct-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36102,7 +36190,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-coder-32b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -36115,7 +36203,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen2-5-vl-72b-instruct", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36129,7 +36217,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/qwen3-32b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -36143,7 +36231,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-120b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36157,7 +36245,7 @@ "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/gpt-oss-20b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_reasoning": true, "supports_response_schema": true, @@ -36171,7 +36259,7 @@ "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/llava-next-mistral-7b", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false, @@ -36185,7 +36273,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, - "source": "https://endpoints.ai.cloud.ovh.net/models/mamba-codestral-7b-v0-1", + "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", "supports_function_calling": false, "supports_response_schema": true, "supports_tool_choice": false @@ -38321,7 +38409,7 @@ "source": "https://docs.mistral.ai/capabilities/code_generation/" }, "text-embedding-004": { - "deprecation_date": "2026-01-14", + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38946,7 +39034,7 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, - "source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B", + "source": "https://www.together.ai/models/qwen3-5-397b-a17b", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -43183,7 +43271,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43199,7 +43287,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 5e-07, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43216,7 +43304,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -43232,7 +43320,7 @@ "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 6e-06, - "source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)", + "source": "https://docs.x.ai/developers/models", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -43352,7 +43440,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43366,7 +43454,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43381,7 +43469,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -43396,7 +43484,7 @@ "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.15, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/veo/3-1-generate", "supported_modalities": [ "text" ], @@ -44100,7 +44188,8 @@ "output_cost_per_second": 0.0001, "supported_endpoints": [ "/v1/audio/transcriptions" - ] + ], + "deprecation_date": "2027-02-26" }, "xai/grok-3": { "cache_read_input_token_cost": 2e-07, @@ -45088,7 +45177,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2", "supported_modalities": [ "text" ], @@ -45100,7 +45189,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.3, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -45112,7 +45201,7 @@ "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.5, - "source": "https://azure.microsoft.com/en-us/products/ai-services/video-generation", + "source": "https://ai.azure.com/catalog/models/sora-2-pro", "supported_modalities": [ "text" ], @@ -49322,7 +49411,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-realtime-whisper", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -50574,7 +50663,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50612,7 +50701,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50650,7 +50739,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -50688,7 +50777,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "source": "https://www.volcengine.com/docs/82379/1330310", + "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": false, @@ -51437,7 +51526,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { "max_tokens": 131072, @@ -51450,7 +51539,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { "max_tokens": 131072, @@ -51463,7 +51552,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { "max_tokens": 131072, @@ -51476,7 +51565,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { "max_tokens": 163840, @@ -51489,7 +51578,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": true, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { "max_tokens": 1000192, @@ -51502,7 +51591,7 @@ "supports_function_calling": true, "supports_assistant_prefill": true, "supports_reasoning": false, - "source": "https://pinstripes.io/pricing" + "source": "https://pinstripes.io/" }, "darkbloom/gemma-4-26b": { "input_cost_per_token": 3e-08, @@ -51606,7 +51695,7 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-transcribe", "supported_endpoints": [ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" @@ -51624,7 +51713,7 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "openai", "mode": "audio_transcription", - "source": "https://platform.openai.com/docs/models/gpt-live-transcribe", + "source": "https://developers.openai.com/api/docs/models/gpt-live-transcribe", "supported_endpoints": [ "/v1/realtime", "/v1/realtime/transcription_sessions" @@ -51645,7 +51734,7 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", - "source": "https://platform.openai.com/docs/models/gpt-realtime-translate", + "source": "https://developers.openai.com/api/docs/models/gpt-realtime-translate", "supported_modalities": [ "audio" ], @@ -51673,7 +51762,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_mid_conversation_system": true, @@ -51712,7 +51801,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "source": "https://docs.claude.com/en/docs/about-claude/models/overview", + "source": "https://platform.claude.com/docs/en/about-claude/models/overview", "supports_adaptive_thinking": true, "thinking_always_on": true, "supports_assistant_prefill": false, diff --git a/osv-scanner.toml b/osv-scanner.toml index 7ab450945f5..5b0339bdcd0 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -2,3 +2,8 @@ id = "GHSA-w8v5-vhqr-4h9v" ignoreUntil = 2026-09-09 reason = "diskcache has no fixed release published; remove this entry once one exists" + +[[IgnoredVulns]] +id = "GHSA-h7x2-h6g9-p789" +ignoreUntil = 2026-09-14 +reason = "mlflow has no fixed release published; remove this entry once one exists" diff --git a/pyproject.toml b/pyproject.toml index 34c1fec1c11..f33bf68822a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,11 @@ cli = [ ] extra_proxy = [ "prisma>=0.11.0,<1.0", + # Used by ProxyExtrasDBManager.spend_logs_is_partitioned() to detect a + # partitioned LiteLLM_SpendLogs and keep schema reconciliation from + # fighting its composite primary key. + "psycopg>=3.2,<4.0", + "psycopg-binary>=3.2,<4.0", "azure-identity>=1.25.2,<2.0", "azure-keyvault-secrets>=4.10.0,<5.0", # Not in PyPI proxy extra. @@ -262,7 +267,7 @@ healthcheck = [ ] [build-system] -requires = ["maturin==1.9.4"] +requires = ["maturin==1.15.0"] build-backend = "maturin" [tool.maturin] @@ -270,6 +275,9 @@ manifest-path = "litellm-rust/crates/python-bridge/Cargo.toml" module-name = "litellm.rust_bridge._native" python-source = "." bindings = "pyo3" +features = ["extension-module"] +profile = "release" +editable-profile = "dev" include = ["litellm/proxy/_experimental/out/**"] exclude = [ "litellm/proxy/enterprise", diff --git a/schema.prisma b/schema.prisma index 60223265211..01a607b68a9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/scripts/auto-close-duplicates.test.ts b/scripts/auto-close-duplicates.test.ts new file mode 100644 index 00000000000..b49bf05cbc2 --- /dev/null +++ b/scripts/auto-close-duplicates.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, test } from "bun:test"; + +import { + CLOSED_MARKER, + REOPEN_COMMENT, + candidateNumbers, + duplicateTarget, + normalizeTitle, + pendingNotice, + readConfig, + reopenTarget, + sweepClosedIssue, + sweepIssue, + type Comment, + type GitHubApi, + type Issue, + type Reaction, + type SweepConfig, +} from "./auto-close-duplicates"; + +const NOW = new Date("2026-09-04T09:00:00Z"); +const DAY_MS = 24 * 60 * 60 * 1000; +const daysAgo = (days: number): string => new Date(NOW.getTime() - days * DAY_MS).toISOString(); + +const issue = (number: number, title: string, overrides: Partial = {}): Issue => ({ + number, + title, + state: "open", + user: { login: "reporter" }, + ...overrides, +}); + +const notice = (candidates: readonly number[], createdAt: string, overrides: Partial = {}): Comment => ({ + id: 900, + body: `\n**Potential duplicate detected**`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + ...overrides, +}); + +const humanComment = (createdAt: string, body = "It is not the same thing", login = "reporter"): Comment => ({ + id: 901, + body, + created_at: createdAt, + user: { type: "User", login }, +}); + +const config: SweepConfig = { repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }; + +describe("normalizeTitle", () => { + test("drops the template prefix, case, and punctuation", () => { + expect(normalizeTitle("[Bug]: Gemma 4-e4b fails on Vertex!")).toBe("gemma 4 e4b fails on vertex"); + expect(normalizeTitle("[Feature]: ")).toBe(""); + }); +}); + +describe("candidateNumbers", () => { + test("reads only the marker field, keeps older issues, sorted ascending and deduplicated", () => { + const body = "\n- #1 - see #1 (100% similar)"; + expect(candidateNumbers(body, 35)).toEqual([10, 30]); + }); + + test("returns nothing without the marker", () => { + expect(candidateNumbers("- #1 - looks like #1", 35)).toEqual([]); + }); +}); + +describe("pendingNotice", () => { + test("waits out the grace period from the latest notice", () => { + const fresh = pendingNotice(issue(35, "t"), [notice([10], daysAgo(2.9))], config); + expect(fresh.kind).toBe("skip"); + const aged = pendingNotice(issue(35, "t"), [notice([10], daysAgo(3.1))], config); + expect(aged.kind).toBe("pending"); + const reposted = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(6)), notice([10], daysAgo(1), { id: 902 })], + config, + ); + expect(reposted.kind).toBe("skip"); + }); + + test("an objection posted before a re-posted notice still keeps the issue open", () => { + const verdict = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(10)), humanComment(daysAgo(7)), notice([10], daysAgo(4), { id: 902 })], + config, + ); + expect(verdict).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + }); + + test("a zero-day grace period acts on the notice at once", () => { + const verdict = pendingNotice(issue(35, "t"), [notice([10], daysAgo(0.01))], { ...config, graceDays: 0 }); + expect(verdict.kind).toBe("pending"); + }); + + test("a human reply after the notice keeps the issue open, a bot reply does not", () => { + const human = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5)), humanComment(daysAgo(4))], config); + expect(human).toEqual({ kind: "skip", reason: "someone replied after the notice" }); + const bot = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(5)), { id: 903, body: "triage", created_at: daysAgo(4), user: { type: "Bot", login: "triage[bot]" } }], + config, + ); + expect(bot.kind).toBe("pending"); + }); + + test("a human quoting the marker is not a notice", () => { + const quoted = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5), { user: { type: "User", login: "reporter" } })], config); + expect(quoted).toEqual({ kind: "skip", reason: "carries no duplicate notice" }); + }); + + test("never closes an issue twice: a reopened issue is left alone", () => { + const reopened = pendingNotice( + issue(35, "t"), + [notice([10], daysAgo(9)), { id: 904, body: `Closed automatically\n\n${CLOSED_MARKER}`, created_at: daysAgo(5), user: { type: "Bot", login: "github-actions[bot]" } }], + config, + ); + expect(reopened).toEqual({ kind: "skip", reason: "was reopened after an automatic close" }); + }); + + test("skips pull requests and issues whose only candidates are newer", () => { + expect(pendingNotice(issue(35, "t", { pull_request: {} }), [notice([10], daysAgo(5))], config).kind).toBe("skip"); + expect(pendingNotice(issue(35, "t"), [notice([40], daysAgo(5))], config)).toEqual({ + kind: "skip", + reason: "no candidate is older than this issue", + }); + }); +}); + +describe("duplicateTarget", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + + test("closes only against the earliest open issue with the identical normalized title", () => { + const verdict = duplicateTarget( + reporter, + [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex"), issue(20, "[bug]: gemma 4-e4b fails on vertex"), issue(30, "[Bug]: Gemma 4-e4b fails on Vertex")], + [], + ); + expect(verdict).toEqual({ kind: "close", duplicateOf: 20 }); + }); + + test("a near miss in the title is not a duplicate", () => { + const verdict = duplicateTarget(reporter, [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex")], []); + expect(verdict).toEqual({ kind: "skip", reason: "no older open issue has the identical title" }); + }); + + test("bare template titles never match each other", () => { + const verdict = duplicateTarget(issue(35, "[Bug]: "), [issue(10, "[Bug]: ")], []); + expect(verdict.kind).toBe("skip"); + expect(verdict.kind === "skip" && verdict.reason).toContain("too short"); + }); + + test("a closed candidate or a pull request is never the target", () => { + expect(duplicateTarget(reporter, [issue(10, reporter.title, { state: "closed" })], []).kind).toBe("skip"); + expect(duplicateTarget(reporter, [issue(10, reporter.title, { pull_request: {} })], []).kind).toBe("skip"); + }); + + test("a thumbs down on the notice keeps the issue open", () => { + const verdict = duplicateTarget(reporter, [issue(10, reporter.title)], [{ content: "+1" }, { content: "-1" }]); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + }); +}); + +describe("sweepIssue", () => { + const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex"); + const original = issue(10, "[Bug]: Gemma 4-e4b fails on Vertex"); + + function fakeApi( + comments: readonly Comment[] = [notice([10], daysAgo(5))], + reactionsByNotice: Readonly> = {}, + ): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + const reactionsPath = path.match(/^\/repos\/BerriAI\/litellm\/issues\/comments\/(\d+)\/reactions/); + if (reactionsPath) { + return (reactionsByNotice[Number(reactionsPath[1])] ?? []) as T; + } + if (path === "/repos/BerriAI/litellm/issues/10") { + return original as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + test("a dry run reports the close and writes nothing", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, { ...config, dryRun: true }, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes).toEqual([]); + }); + + test("a thumbs down on an earlier notice still keeps the issue open", async () => { + const { api, writes } = fakeApi([notice([10], daysAgo(9)), notice([10], daysAgo(5), { id: 902 })], { 900: [{ content: "-1" }] }); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" }); + expect(writes).toEqual([]); + }); + + test("a real run comments, labels, then closes with the duplicate reason", async () => { + const { api, writes } = fakeApi(); + const verdict = await sweepIssue(api, config, reporter); + expect(verdict).toEqual({ kind: "close", duplicateOf: 10 }); + expect(writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([ + "POST /repos/BerriAI/litellm/issues/35/comments", + "POST /repos/BerriAI/litellm/issues/35/labels", + "PATCH /repos/BerriAI/litellm/issues/35", + ]); + expect(writes[0]).toContain("duplicate of #10"); + expect(writes[0]).toContain("unanswered for 3 days"); + expect(writes[0]).toContain(CLOSED_MARKER); + expect(writes[1]).toContain('{"labels":["duplicate"]}'); + expect(writes[2]).toContain('{"state":"closed","state_reason":"duplicate"}'); + }); +}); + +describe("reopenTarget", () => { + const closedByBot = (overrides: Partial = {}): Issue => + issue(35, "t", { state: "closed", closed_by: { type: "Bot" }, ...overrides }); + const closeMarker = (createdAt: string): Comment => ({ + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: createdAt, + user: { type: "Bot", login: "github-actions[bot]" }, + }); + + test("a reporter reply after the automatic close reopens", () => { + const verdict = reopenTarget(closedByBot(), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "reopen" }); + }); + + test("an issue closed by a person stays closed", () => { + const verdict = reopenTarget(closedByBot({ closed_by: { type: "User" } }), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1)), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "was closed by a person" }); + }); + + test("without the automatic-close marker nothing reopens", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "carries no automatic-close marker" }); + }); + + test("a maintainer reply alone does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [ + closeMarker(daysAgo(2)), + humanComment(daysAgo(1), "Confirmed duplicate", "maintainer"), + ]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a reporter comment from before the close does not reopen", () => { + const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(3)), closeMarker(daysAgo(2))]); + expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" }); + }); + + test("a pull request never reopens", () => { + const verdict = reopenTarget(closedByBot({ pull_request: {} }), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]); + expect(verdict).toEqual({ kind: "skip", reason: "is a pull request" }); + }); +}); + +describe("sweepClosedIssue", () => { + function fakeApi(issueBody: Issue, comments: readonly Comment[]): { readonly api: GitHubApi; readonly writes: readonly string[] } { + const writes: string[] = []; + const api: GitHubApi = { + request: async (method: string, path: string, body?: object): Promise => { + if (method !== "GET") { + writes.push(`${method} ${path} ${JSON.stringify(body)}`); + return {} as T; + } + if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) { + return comments as T; + } + if (path === "/repos/BerriAI/litellm/issues/35") { + return issueBody as T; + } + throw new Error(`unexpected GET ${path}`); + }, + }; + return { api, writes }; + } + + const closedByBot = issue(35, "t", { state: "closed", closed_by: { type: "Bot" } }); + const closeMarker: Comment = { + id: 905, + body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`, + created_at: daysAgo(2), + user: { type: "Bot", login: "github-actions[bot]" }, + }; + + test("a real run unlabels, reopens, then explains", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, config, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([ + "DELETE /repos/BerriAI/litellm/issues/35/labels/duplicate undefined", + 'PATCH /repos/BerriAI/litellm/issues/35 {"state":"open"}', + `POST /repos/BerriAI/litellm/issues/35/comments {"body":"${REOPEN_COMMENT}"}`, + ]); + }); + + test("a dry run reports the reopen and writes nothing", async () => { + const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]); + const verdict = await sweepClosedIssue(api, { ...config, dryRun: true }, 35); + expect(verdict).toEqual({ kind: "reopen" }); + expect(writes).toEqual([]); + }); +}); + +describe("readConfig", () => { + test("defaults to a real run with a 3-day grace period", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm" }, NOW); + expect(parsed).toEqual({ token: "t", repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW }); + }); + + test("honors DRY_RUN and GRACE_PERIOD_DAYS overrides", () => { + const parsed = readConfig( + { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", DRY_RUN: "true", GRACE_PERIOD_DAYS: "0" }, + NOW, + ); + expect(parsed.dryRun).toBe(true); + expect(parsed.graceDays).toBe(0); + }); + + test("an empty GRACE_PERIOD_DAYS, as a schedule run renders it, means the default", () => { + const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "" }, NOW); + expect(parsed.graceDays).toBe(3); + }); + + test("refuses a missing token, a malformed repository, or a bad grace period", () => { + expect(() => readConfig({ GITHUB_REPOSITORY: "o/r" }, NOW)).toThrow("GITHUB_TOKEN"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "litellm" }, NOW)).toThrow("owner/repo"); + expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "-1" }, NOW)).toThrow( + "GRACE_PERIOD_DAYS", + ); + }); +}); diff --git a/scripts/auto-close-duplicates.ts b/scripts/auto-close-duplicates.ts new file mode 100644 index 00000000000..c595104d886 --- /dev/null +++ b/scripts/auto-close-duplicates.ts @@ -0,0 +1,300 @@ +#!/usr/bin/env bun + +declare const process: { readonly env: Readonly> }; + +export interface Issue { + readonly number: number; + readonly title: string; + readonly state: string; + readonly user: { readonly login: string }; + readonly closed_by?: { readonly type: string } | null; + readonly pull_request?: unknown; +} + +export interface Comment { + readonly id: number; + readonly body: string; + readonly created_at: string; + readonly user: { readonly type: string; readonly login: string }; +} + +export interface Reaction { + readonly content: string; +} + +export interface GitHubApi { + readonly request: (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object) => Promise; +} + +export interface SweepConfig { + readonly repo: string; + readonly graceDays: number; + readonly dryRun: boolean; + readonly now: Date; +} + +export type NoticeVerdict = + | { readonly kind: "pending"; readonly notices: readonly Comment[]; readonly candidates: readonly number[] } + | { readonly kind: "skip"; readonly reason: string }; + +export type CloseVerdict = + | { readonly kind: "close"; readonly duplicateOf: number } + | { readonly kind: "skip"; readonly reason: string }; + +export type ReopenVerdict = + | { readonly kind: "reopen" } + | { readonly kind: "skip"; readonly reason: string }; + +export const FLAG_LABEL = "potential-duplicate"; +export const CLOSED_MARKER = ""; +export const DEFAULT_GRACE_DAYS = 3; +export const REOPEN_COMMENT = + "Reopened automatically: the reporter replied after the duplicate close, so this needs a human look."; +const NOTICE_MARKER = //; +const MIN_TITLE_WORDS = 3; +const PAGE_SIZE = 100; +const DAY_MS = 24 * 60 * 60 * 1000; +const REOPEN_LOOKBACK_DAYS = 30; + +const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason }); + +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/^\s*\[[^\]]*\]\s*:?/, "") + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +export function candidateNumbers(noticeBody: string, issueNumber: number): readonly number[] { + const field = noticeBody.match(NOTICE_MARKER); + if (!field) { + return []; + } + const older = field[1] + .split(",") + .filter((value) => value !== "") + .map(Number) + .filter((candidate) => candidate < issueNumber); + return [...new Set(older)].sort((a, b) => a - b); +} + +export function pendingNotice( + issue: Issue, + comments: readonly Comment[], + config: Pick, +): NoticeVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (comments.some((comment) => comment.body.includes(CLOSED_MARKER))) { + return skip("was reopened after an automatic close"); + } + const notices = comments.filter((comment) => comment.user.type === "Bot" && NOTICE_MARKER.test(comment.body)); + const first = notices[0]; + const latest = notices[notices.length - 1]; + if (first === undefined || latest === undefined) { + return skip("carries no duplicate notice"); + } + const ageDays = (config.now.getTime() - new Date(latest.created_at).getTime()) / DAY_MS; + if (ageDays < config.graceDays) { + return skip(`notice is ${ageDays.toFixed(1)} days old, grace period is ${config.graceDays}`); + } + const firstNoticeAt = new Date(first.created_at); + if (comments.some((comment) => comment.user.type !== "Bot" && new Date(comment.created_at) > firstNoticeAt)) { + return skip("someone replied after the notice"); + } + const candidates = candidateNumbers(latest.body, issue.number); + if (candidates.length === 0) { + return skip("no candidate is older than this issue"); + } + return { kind: "pending", notices, candidates }; +} + +export function duplicateTarget( + issue: Issue, + candidates: readonly Issue[], + reactions: readonly Reaction[], +): CloseVerdict { + if (reactions.some((reaction) => reaction.content === "-1")) { + return skip("someone gave the notice a thumbs down"); + } + const title = normalizeTitle(issue.title); + if (title.split(" ").length < MIN_TITLE_WORDS) { + return skip(`title "${issue.title}" is too short to match on`); + } + const original = candidates.find( + (candidate) => + candidate.state === "open" && candidate.pull_request === undefined && normalizeTitle(candidate.title) === title, + ); + if (original === undefined) { + return skip("no older open issue has the identical title"); + } + return { kind: "close", duplicateOf: original.number }; +} + +export function reopenTarget(issue: Issue, comments: readonly Comment[]): ReopenVerdict { + if (issue.pull_request !== undefined) { + return skip("is a pull request"); + } + if (issue.closed_by?.type !== "Bot") { + return skip("was closed by a person"); + } + const marker = comments.find((comment) => comment.body.includes(CLOSED_MARKER)); + if (marker === undefined) { + return skip("carries no automatic-close marker"); + } + const markerAt = new Date(marker.created_at); + if (!comments.some((comment) => comment.user.login === issue.user.login && new Date(comment.created_at) > markerAt)) { + return skip("the reporter has not replied since the close"); + } + return { kind: "reopen" }; +} + +export function closingComment(duplicateOf: number, graceDays: number): string { + return `Closed automatically as a duplicate of #${duplicateOf}. Its title is identical to that older open issue and the duplicate notice above went unanswered for ${graceDays} days. If this is wrong, comment here with how it differs from #${duplicateOf} and this issue will be reopened automatically within a day. + +${CLOSED_MARKER}`; +} + +async function listAll(api: GitHubApi, path: string, page = 1): Promise { + const separator = path.includes("?") ? "&" : "?"; + const batch = await api.request("GET", `${path}${separator}per_page=${PAGE_SIZE}&page=${page}`); + return batch.length < PAGE_SIZE ? batch : [...batch, ...(await listAll(api, path, page + 1))]; +} + +async function closeAsDuplicate( + api: GitHubApi, + config: SweepConfig, + issueNumber: number, + duplicateOf: number, +): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("POST", `${issuePath}/comments`, { body: closingComment(duplicateOf, config.graceDays) }); + await api.request("POST", `${issuePath}/labels`, { labels: ["duplicate"] }); + await api.request("PATCH", issuePath, { state: "closed", state_reason: "duplicate" }); +} + +async function reopenForReporter(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issuePath = `/repos/${config.repo}/issues/${issueNumber}`; + await api.request("DELETE", `${issuePath}/labels/duplicate`); + await api.request("PATCH", issuePath, { state: "open" }); + await api.request("POST", `${issuePath}/comments`, { body: REOPEN_COMMENT }); +} + +export async function sweepClosedIssue(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise { + const issue = await api.request("GET", `/repos/${config.repo}/issues/${issueNumber}`); + const comments = await listAll(api, `/repos/${config.repo}/issues/${issueNumber}/comments`); + const verdict = reopenTarget(issue, comments); + if (verdict.kind === "reopen" && !config.dryRun) { + await reopenForReporter(api, config, issueNumber); + } + return verdict; +} + +export async function sweepIssue(api: GitHubApi, config: SweepConfig, issue: Issue): Promise { + const comments = await listAll(api, `/repos/${config.repo}/issues/${issue.number}/comments`); + const pending = pendingNotice(issue, comments, config); + if (pending.kind === "skip") { + return pending; + } + const reactions = ( + await Promise.all( + pending.notices.map((notice) => listAll(api, `/repos/${config.repo}/issues/comments/${notice.id}/reactions`)), + ) + ).flat(); + const candidates = await Promise.all( + pending.candidates.map((candidate) => api.request("GET", `/repos/${config.repo}/issues/${candidate}`)), + ); + const verdict = duplicateTarget(issue, candidates, reactions); + if (verdict.kind === "close" && !config.dryRun) { + await closeAsDuplicate(api, config, issue.number, verdict.duplicateOf); + } + return verdict; +} + +function describe(issue: Issue, verdict: CloseVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issue.number}: skipped, ${verdict.reason}`; + } + return `#${issue.number}: ${dryRun ? "would close" : "closed"} as a duplicate of #${verdict.duplicateOf}`; +} + +export async function sweep(api: GitHubApi, config: SweepConfig): Promise { + const issues = await listAll(api, `/repos/${config.repo}/issues?state=open&labels=${FLAG_LABEL}`); + console.log(`${issues.length} open issues carry the ${FLAG_LABEL} label in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepIssue(api, config, issue); + console.log(describe(issue, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +function describeReopen(issueNumber: number, verdict: ReopenVerdict, dryRun: boolean): string { + if (verdict.kind === "skip") { + return `#${issueNumber}: skipped, ${verdict.reason}`; + } + return `#${issueNumber}: ${dryRun ? "would reopen" : "reopened"} for the reporter's reply`; +} + +export async function reopenSweep(api: GitHubApi, config: SweepConfig): Promise { + const since = new Date(config.now.getTime() - REOPEN_LOOKBACK_DAYS * DAY_MS).toISOString(); + const closedPath = `/repos/${config.repo}/issues?state=closed&labels=duplicate,${FLAG_LABEL}&since=${encodeURIComponent(since)}`; + const issues = await listAll(api, closedPath); + console.log(`${issues.length} recently closed issues carry the duplicate and ${FLAG_LABEL} labels in ${config.repo}${config.dryRun ? " (dry run)" : ""}`); + return issues.reduce>(async (previous, issue) => { + const verdicts = await previous; + const verdict = await sweepClosedIssue(api, config, issue.number); + console.log(describeReopen(issue.number, verdict, config.dryRun)); + return [...verdicts, verdict]; + }, Promise.resolve([])); +} + +export function readConfig(env: Readonly>, now: Date): SweepConfig & { readonly token: string } { + const token = env.GITHUB_TOKEN; + const repo = env.GITHUB_REPOSITORY; + if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) { + throw new Error("GITHUB_TOKEN and GITHUB_REPOSITORY (owner/repo) are required"); + } + const rawGraceDays = env.GRACE_PERIOD_DAYS?.trim(); + const graceDays = rawGraceDays === undefined || rawGraceDays === "" ? DEFAULT_GRACE_DAYS : Number(rawGraceDays); + if (!Number.isFinite(graceDays) || graceDays < 0) { + throw new Error(`GRACE_PERIOD_DAYS must be a non-negative number, got "${env.GRACE_PERIOD_DAYS}"`); + } + return { token, repo, graceDays, dryRun: env.DRY_RUN === "true", now }; +} + +export function githubApi(token: string): GitHubApi { + return { + request: async (method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object): Promise => { + const response = await fetch(`https://api.github.com${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "litellm-auto-close-duplicates", + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`${method} ${path} failed: ${response.status} ${response.statusText}`); + } + return (await response.json()) as T; + }, + }; +} + +if (import.meta.main) { + const { token, ...config } = readConfig(process.env, new Date()); + const api = githubApi(token); + const closeVerdicts = await sweep(api, config); + const reopenVerdicts = await reopenSweep(api, config); + const closed = closeVerdicts.filter((verdict) => verdict.kind === "close").length; + const reopened = reopenVerdicts.filter((verdict) => verdict.kind === "reopen").length; + console.log( + `${config.dryRun ? "Would close" : "Closed"} ${closed} of ${closeVerdicts.length} flagged issues, ${config.dryRun ? "would reopen" : "reopened"} ${reopened}`, + ); +} diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index b91c404b2eb..3652378503e 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -33,6 +33,13 @@ EXCLUDED_ROLLOUT_FLAGS = { "LITELLM_RUST", } +# Internal infrastructure tuning parameters for streaming/queue management +# These are advanced settings with sensible defaults that most users should not modify +EXCLUDED_INTERNAL_TUNING_VARS = { + "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", + "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", +} + EXCLUDED_TERMINAL_VARS = { "TERM", "TERM_PROGRAM", @@ -50,7 +57,9 @@ EXCLUDED_TERMINAL_VARS = { "ALACRITTY_SOCKET", } -EXCLUDED_KEYS = frozenset(EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS) +EXCLUDED_KEYS = frozenset( + EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS | EXCLUDED_INTERNAL_TUNING_VARS +) # Directories to skip (dependencies, venvs, caches) - only scan litellm source SKIP_DIRS = { diff --git a/tests/e2e/management/test_model_tag_accessgroup_e2e.py b/tests/e2e/management/test_model_tag_accessgroup_e2e.py index e6a187ae105..eb3a6093c69 100644 --- a/tests/e2e/management/test_model_tag_accessgroup_e2e.py +++ b/tests/e2e/management/test_model_tag_accessgroup_e2e.py @@ -180,6 +180,12 @@ class ModelBlockBody(BaseModel): model_id: str +class ModelBlockResponse(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + model_id: str + blocked: bool + + class ModelInfoBlockDetail(BaseModel): id: str | None = None blocked: bool | None = None @@ -245,11 +251,6 @@ class TestModelRoutes: def test_block_then_unblock_persists_to_model_info( self, client: ManagementClient, resources: ResourceManager ) -> None: - """The blocked flag's persistence is read back from /model/info, not from the - /model/block response: that route currently returns a non-2xx serialization - envelope even though the DB write lands, so the /model/info read-back is the - authoritative persistence contract and keeps this test valid once the - response shape is fixed.""" model_name = f"e2e-mgmt-model-block-{unique_marker()}" model_id = _create_db_model(client, resources, model_name) @@ -257,27 +258,25 @@ class TestModelRoutes: f"{model_name!r} already reports blocked in /model/info before /model/block ran" ) - _ = client.proxy.transport.send( - "/model/block", - headers=client.proxy.transport.master, - json=ModelBlockBody(model_id=model_id), - ) - _ = _poll( - client.proxy, - lambda: True if _model_blocked_flag(client, model_id) is True else None, - f"/model/info never reported {model_name!r} blocked after /model/block", - ) - - _ = client.proxy.transport.send( - "/model/unblock", - headers=client.proxy.transport.master, - json=ModelBlockBody(model_id=model_id), - ) - _ = _poll( - client.proxy, - lambda: True if _model_blocked_flag(client, model_id) is not True else None, - f"/model/info never cleared blocked for {model_name!r} after /model/unblock", - ) + for action, expected in (("block", True), ("unblock", False)): + response = unwrap( + client.proxy.transport.post( + f"/model/{action}", + headers=client.proxy.transport.master, + json=ModelBlockBody(model_id=model_id), + response_type=ModelBlockResponse, + ) + ) + assert response.model_id == model_id + assert response.blocked is expected + _ = _poll( + client.proxy, + lambda want=expected: True + if _model_blocked_flag(client, model_id) is want + else None, + f"/model/info never reported blocked={expected} for {model_name!r} " + f"after /model/{action}", + ) class TestTagRoutes: diff --git a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts index b8424b06115..26e34dd2fe5 100644 --- a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts @@ -22,8 +22,7 @@ test.describe("Internal User", () => { const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); await teamSelect.click(); await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); - const dropdown = page.locator('[data-slot="combobox-content"]:visible'); - await expect(dropdown.getByText(E2E_TEAM_CRUD_ALIAS).first()).toBeVisible({ timeout: 5_000 }); + await expect(page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first()).toBeVisible({ timeout: 5_000 }); }); test("Team info page omits the Settings tab for non-admin members", async ({ page }) => { @@ -43,12 +42,12 @@ test.describe("Internal User", () => { // Anchor on the user's own seeded key so the absence check below cannot // pass vacuously against an empty table. - await expect(page.locator("table tbody").getByText(E2E_INTERNAL_USER_KEY_ALIAS).first()).toBeVisible({ + await expect(page.getByRole("row").filter({ hasText: E2E_INTERNAL_USER_KEY_ALIAS }).first()).toBeVisible({ timeout: 10_000, }); // The litellm-dashboard team is the proxy's internal bookkeeping team — // its keys must never leak into an internal user's Virtual Keys table. - await expect(page.locator("table tbody").getByText("litellm-dashboard")).toHaveCount(0); + await expect(page.getByRole("row").filter({ hasText: "litellm-dashboard" })).toHaveCount(0); }); }); diff --git a/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts b/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts index c44305187f1..653e096b713 100644 --- a/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUserNoTeam.spec.ts @@ -30,16 +30,13 @@ test.describe("Internal User with no team memberships", () => { const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); await teamSelect.click(); - const dropdown = page.locator('[data-slot="combobox-content"]:visible').first(); - await expect(dropdown).toBeVisible({ timeout: 5_000 }); - // Wait for the settled-empty state, not a transient one. The dropdown shows // "Loading teams…" while teams load and only swaps in "No teams found" once // the request resolves with nothing (team_dropdown.tsx passes both copies to // PaginatedSearchSelect). Asserting on it means a regression where teams DO // load for this user fails here instead of racing a one-shot count() against // an in-flight request. - await expect(dropdown.getByText("No teams found")).toBeVisible({ timeout: 10_000 }); - await expect(dropdown.getByRole("option")).toHaveCount(0); + await expect(page.getByText("No teams found")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByRole("option")).toHaveCount(0); }); }); diff --git a/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts b/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts index 68319154554..49e27a36673 100644 --- a/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts @@ -21,13 +21,10 @@ test.describe("Internal User with team memberships", () => { const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); await teamSelect.click(); - const dropdown = page.locator('[data-slot="combobox-content"]:visible').first(); - await expect(dropdown).toBeVisible({ timeout: 5_000 }); - // Both seeded memberships render, and nothing else does — proving the // dropdown is scoped to the user's teams rather than empty or unfiltered. - await expect(dropdown.getByText(E2E_TEAM_CRUD_ALIAS, { exact: true })).toBeVisible({ timeout: 10_000 }); - await expect(dropdown.getByText(E2E_TEAM_ORG_ALIAS, { exact: true })).toBeVisible(); - await expect(dropdown.getByRole("option")).toHaveCount(2); + await expect(page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS })).toBeVisible({ timeout: 10_000 }); + await expect(page.getByRole("option", { name: E2E_TEAM_ORG_ALIAS })).toBeVisible(); + await expect(page.getByRole("option")).toHaveCount(2); }); }); diff --git a/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts b/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts index 4de86c46398..dd40976341d 100644 --- a/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts +++ b/tests/e2e/ui/tests/internal-viewer/internalViewer.spec.ts @@ -59,9 +59,9 @@ test.describe("Internal Viewer", () => { await expect(page.getByRole("button", { name: /Create New Key/i })).toHaveCount(0); // Open the viewer's own key info page - const keyRow = page.locator("tr", { hasText: E2E_VIEWER_KEY_ALIAS }); + const keyRow = page.getByRole("row").filter({ hasText: E2E_VIEWER_KEY_ALIAS }); await expect(keyRow).toBeVisible({ timeout: 10_000 }); - await keyRow.locator("button").first().click(); + await keyRow.getByRole("button", { name: E2E_VIEWER_KEY_ALIAS }).click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); // None of the destructive / mutating actions should render diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts index 56a3d0f0109..610a88c6cd0 100644 --- a/tests/e2e/ui/tests/logs/logs.spec.ts +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -11,12 +11,11 @@ import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -/** - * Walking up from the label is the only stable handle: the header carries no role, test id or class, - * and its copy button is icon-only with a hover-only tooltip. - */ -const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator => - drawer.getByText(label, { exact: true }).locator("xpath=../../.."); +const sectionToggle = (drawer: Locator, label: "Input" | "Output"): Locator => + drawer.getByRole("button", { name: new RegExp(`^${label}\\b`) }); + +const sectionCopy = (drawer: Locator, label: "Input" | "Output"): Locator => + drawer.getByRole("button", { name: `Copy ${label.toLowerCase()}` }); /** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */ const requestLogsRows = (page: PlaywrightPage): Locator => @@ -95,14 +94,14 @@ test.describe("Logs page", () => { await expect(drawer).toBeVisible({ timeout: 20_000 }); // Copy request: the Input card's copy button puts the prompt on the clipboard. - await sectionHeader(drawer, "Input").getByRole("button").click(); + await sectionCopy(drawer, "Input").click(); await expect(page.getByText("Input copied")).toBeVisible({ timeout: 10_000, }); expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt); // Copy response: the Output card's copy button puts the completion on it. - await sectionHeader(drawer, "Output").getByRole("button").click(); + await sectionCopy(drawer, "Output").click(); await expect(page.getByText("Output copied")).toBeVisible({ timeout: 10_000, }); @@ -125,24 +124,15 @@ test.describe("Logs page", () => { timeout: 20_000, }); - // The body collapses via `max-height: 0; overflow: hidden`, which zeroes its own bounding - // box, so the wrapper reads as hidden while the clipped text node inside it does not. - const header = sectionHeader(drawer, "Input"); - const body = header.locator("xpath=following-sibling::div[1]"); - await expect(header.locator(".lucide-chevron-up")).toBeVisible(); - await expect(body).toBeVisible(); + const toggle = sectionToggle(drawer, "Input"); + await expect(toggle).toHaveAttribute("aria-expanded", "true"); + await expect(drawer.getByText(prompt, { exact: false })).toBeVisible(); - await header.click(); - await expect(header.locator(".lucide-chevron-down")).toBeVisible({ - timeout: 10_000, - }); - await expect(body).toBeHidden({ timeout: 10_000 }); + await toggle.click(); + await expect(toggle).toHaveAttribute("aria-expanded", "false", { timeout: 10_000 }); - await header.click(); - await expect(header.locator(".lucide-chevron-up")).toBeVisible({ - timeout: 10_000, - }); - await expect(body).toBeVisible({ timeout: 10_000 }); + await toggle.click(); + await expect(toggle).toHaveAttribute("aria-expanded", "true", { timeout: 10_000 }); await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({ timeout: 10_000, }); diff --git a/tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts b/tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts index 46799c8a18f..aa7cdf82498 100644 --- a/tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts @@ -73,7 +73,7 @@ test.describe("MCP Servers - edit and delete", () => { test("Deleting a server removes it", async ({ page }) => { expect(await findServerByName(page, serverName), `created server ${serverName} exists`).toBeTruthy(); - const card = page.getByTestId("mcp-servers-grid").locator("div").filter({ hasText: serverName }).first(); + const card = page.getByTestId("mcp-servers-grid").getByRole("button", { name: serverName }); await card.getByRole("button", { name: "Server actions" }).click(); await page.getByRole("menuitem", { name: "Delete" }).click(); diff --git a/tests/e2e/ui/tests/migration/migratedPages.spec.ts b/tests/e2e/ui/tests/migration/migratedPages.spec.ts index 3ad4b217d08..547330190bd 100644 --- a/tests/e2e/ui/tests/migration/migratedPages.spec.ts +++ b/tests/e2e/ui/tests/migration/migratedPages.spec.ts @@ -35,17 +35,12 @@ async function expectRendered(page: Page) { */ async function clickSidebar(page: Page, segment: string) { const link = sidebar(page).locator(`a[href$="/ui/${segment}"]`).first(); + const collapsedGroups = sidebar(page).getByRole("button", { expanded: false }); for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) { - // A collapsed group is a menu item with a group-toggle button but no - // rendered submenu yet; clicking the toggle expands it. - const collapsedGroup = sidebar(page) - .locator( - '[data-slot="sidebar-menu-item"]:has(> [data-slot="sidebar-menu-button"]):not(:has(> [data-slot="sidebar-menu-sub"])) > [data-slot="sidebar-menu-button"]', - ) - .first(); - if (!(await collapsedGroup.isVisible().catch(() => false))) break; - await collapsedGroup.click(); - await page.waitForTimeout(250); + const stillCollapsed = await collapsedGroups.count(); + if (stillCollapsed === 0) break; + await collapsedGroups.first().click(); + await expect(collapsedGroups).toHaveCount(stillCollapsed - 1); } await link.click(); } diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index dad716b4c83..84d1c01b452 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -188,7 +188,7 @@ test.describe("Add Model", () => { await expect(resultsModal).toBeHidden({ timeout: 5_000 }); const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => { - await page.getByRole("button", { name: "Add Model" }).last().click(); + await page.getByTestId("add-model-btn").click(); }); expect(created.model_name, "the model is created under the name that was typed").toBe(publicName); expect(created.litellm_params?.api_base, "the api base survives the form").toBe(MOCK_LLM_BASE); @@ -254,7 +254,7 @@ test.describe("Add Model", () => { // Click Add Model button by its text const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => { - await page.getByRole("button", { name: "Add Model" }).last().click(); + await page.getByTestId("add-model-btn").click(); }); // The form sends custom_llm_provider separately from the name, so both halves have to arrive. expect(created.model_name, "the selected model is what goes on the wire").toBe("claude-haiku-4-5"); @@ -267,11 +267,9 @@ test.describe("Add Model", () => { // Navigate to All Models tab await page.getByRole("tab", { name: "All Models" }).click(); await page.waitForLoadState("networkidle"); - await page.waitForTimeout(2000); // Search for the model we just added await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5"); - await page.waitForTimeout(1000); // Verify the model appears in the results count (not "Showing 0 results") await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, { @@ -279,8 +277,9 @@ test.describe("Add Model", () => { }); // Verify the model name appears in the table body - const tableBody = page.locator("table tbody"); - await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 }); + await expect(page.getByRole("row").filter({ hasText: "claude-haiku-4-5" })).not.toHaveCount(0, { + timeout: 15_000, + }); // A row proves the name is there, not what the deployment routes to. const stored = await findDeploymentByName(page, "claude-haiku-4-5"); @@ -333,11 +332,11 @@ test.describe("Add Model", () => { const teamDropdown = page.getByTestId("team-dropdown").getByRole("combobox"); await expect(teamDropdown).toBeVisible({ timeout: 5_000 }); await teamDropdown.click(); - const teamOption = page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ID).first(); + const teamOption = page.getByRole("option", { name: E2E_TEAM_CRUD_ID }).first(); await expect(teamOption).toBeVisible({ timeout: 5_000 }); await teamOption.click(); - await page.getByRole("button", { name: "Add Model" }).last().click(); + await page.getByTestId("add-model-btn").click(); // Scope to the toast container so a stale toast can't satisfy this. await expect(page.locator("[data-sonner-toast]").getByText("created successfully").last()).toBeVisible({ @@ -347,12 +346,9 @@ test.describe("Add Model", () => { // The Models table renders team-scoped models with the team id in the row. await page.getByRole("tab", { name: "All Models" }).click(); await page.waitForLoadState("networkidle"); - // networkidle fires before the table finishes re-rendering. - await page.waitForTimeout(2000); await page.getByPlaceholder("Search model names").fill("cohere"); - await page.waitForTimeout(1000); - + // Clearer failure than timing out on a row assertion when the table is empty. await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, { timeout: 15_000, @@ -361,7 +357,7 @@ test.describe("Add Model", () => { // Pin to one row carrying both the name and the team, so the sibling test's // team-less cohere row can't satisfy it. const teamCohereRow = page - .locator("table tbody tr") + .getByRole("row") .filter({ hasText: "cohere/" }) .filter({ hasText: E2E_TEAM_CRUD_ID }); await expect(teamCohereRow).toHaveCount(1, { timeout: 15_000 }); @@ -387,7 +383,7 @@ test.describe("Add Model", () => { // Click Add Model button by its text const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => { - await page.getByRole("button", { name: "Add Model" }).last().click(); + await page.getByTestId("add-model-btn").click(); }); // A wildcard with the star stripped becomes a plain "cohere" deployment that matches nothing. expect(created.model_name, "the wildcard route goes on the wire intact").toBe("cohere/*"); @@ -398,11 +394,9 @@ test.describe("Add Model", () => { // Navigate to All Models tab await page.getByRole("tab", { name: "All Models" }).click(); await page.waitForLoadState("networkidle"); - await page.waitForTimeout(2000); // Search for the wildcard model await page.getByPlaceholder("Search model names").fill("cohere"); - await page.waitForTimeout(1000); // Verify the model appears in the results count (not "Showing 0 results") await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, { @@ -410,8 +404,7 @@ test.describe("Add Model", () => { }); // Verify the wildcard model appears in the table body (wildcard models show as "cohere/*") - const tableBody = page.locator("table tbody"); - await expect(tableBody.getByText("cohere/").first()).toBeVisible({ timeout: 15_000 }); + await expect(page.getByRole("row").filter({ hasText: "cohere/" })).not.toHaveCount(0, { timeout: 15_000 }); // "cohere/" in the table also matches a plain cohere deployment; require the wildcard exactly. const stored = await findDeploymentByName(page, "cohere/*"); diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts index 1d080ec82b8..51df50a2e68 100644 --- a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts @@ -17,49 +17,54 @@ async function openTemplateSelect(page: PlaywrightPage) { return trigger; } -function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) { +async function boxes(trigger: Locator, options: Locator) { + const triggerBox = await trigger.boundingBox(); + const optionsBox = await options.boundingBox(); + return triggerBox && optionsBox ? { triggerBox, optionsBox } : null; +} + +const clippedPopup = (page: PlaywrightPage) => page.locator('[data-slot="select-content"]'); + +function pollOptionsOpenBelowTrigger(trigger: Locator, options: Locator) { return expect.poll(async () => { - const triggerBox = await trigger.boundingBox(); - const popupBox = await popup.boundingBox(); - if (!triggerBox || !popupBox) return null; - return popupBox.y - (triggerBox.y + triggerBox.height); + const box = await boxes(trigger, options); + return box && box.optionsBox.y >= box.triggerBox.y + box.triggerBox.height; }); } -function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) { +function pollOptionsCoverTrigger(trigger: Locator, options: Locator) { return expect.poll(async () => { - const triggerBox = await trigger.boundingBox(); - const popupBox = await popup.boundingBox(); - if (!triggerBox || !popupBox) return null; - return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y; + const box = await boxes(trigger, options); + return ( + box && + box.optionsBox.y < box.triggerBox.y + box.triggerBox.height && + box.optionsBox.y + box.optionsBox.height > box.triggerBox.y + ); }); } test.describe("Auto Router template select anchoring", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - test("opens the options below the trigger rather than over it", async ({ page }) => { + test("opens the options below the trigger when there is room below it", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); const trigger = await openTemplateSelect(page); + await trigger.scrollIntoViewIfNeeded(); await trigger.click(); - const popup = page.locator('[data-slot="select-content"]'); - await expect(popup).toBeVisible(); + await expect(page.getByRole("listbox")).toBeVisible(); - // Item-aligned mode reports "none" and puts the active item over the trigger. - await expect(popup).toHaveAttribute("data-side", "bottom"); - await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0); + await pollOptionsOpenBelowTrigger(trigger, clippedPopup(page)).toBe(true); }); - test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => { + test("keeps the trigger uncovered when the options open with no room below it", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 560 }); const trigger = await openTemplateSelect(page); await trigger.scrollIntoViewIfNeeded(); await trigger.click(); - const popup = page.locator('[data-slot="select-content"]'); - await expect(popup).toBeVisible(); + await expect(page.getByRole("listbox")).toBeVisible(); - await pollPopupOverlapsTrigger(trigger, popup).toBe(false); + await pollOptionsCoverTrigger(trigger, clippedPopup(page)).toBe(false); }); }); diff --git a/tests/e2e/ui/tests/modelsPage/responsiveHeader.spec.ts b/tests/e2e/ui/tests/modelsPage/responsiveHeader.spec.ts index 6ad1ccb8451..aabdf18d427 100644 --- a/tests/e2e/ui/tests/modelsPage/responsiveHeader.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/responsiveHeader.spec.ts @@ -7,9 +7,7 @@ test.describe("Models and Endpoints responsive header", () => { viewport: { width: 900, height: 720 }, }); - test("keeps the refresh action on the same row as the tabs", async ({ - page, - }) => { + test("keeps the refresh action on the same row as the tabs", async ({ page }) => { await page.goto("/ui"); await page .getByRole("complementary") @@ -26,8 +24,8 @@ test.describe("Models and Endpoints responsive header", () => { expect(tabsBox).not.toBeNull(); expect(refreshBox).not.toBeNull(); - const tabsCenterY = tabsBox!.y + tabsBox!.height / 2; const refreshCenterY = refreshBox!.y + refreshBox!.height / 2; - expect(Math.abs(tabsCenterY - refreshCenterY)).toBeLessThanOrEqual(2); + const sharesARow = refreshCenterY > tabsBox!.y && refreshCenterY < tabsBox!.y + tabsBox!.height; + expect(sharesARow, "refresh wrapped onto its own row below the tabs").toBe(true); }); }); diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index 0c38641dcc7..f5bee68f245 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -43,7 +43,7 @@ test.describe("Proxy Admin - Keys", () => { const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); await teamSelect.click(); await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); - await page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ALIAS).first().click(); + await page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first().click(); // Select models — the popup is portaled to the body, so scope options to the page. await page.getByRole("combobox", { name: "Select models" }).click(); @@ -74,10 +74,9 @@ test.describe("Proxy Admin - Keys", () => { const before = await findKeyByAlias(page, E2E_REGENERATE_KEY_ALIAS); expect(before?.token, `seeded key ${E2E_REGENERATE_KEY_ALIAS} has a token`).toBeTruthy(); - // Key IDs are rendered as buttons in the table - const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS }); + const keyRow = page.getByRole("row").filter({ hasText: E2E_REGENERATE_KEY_ALIAS }); await expect(keyRow).toBeVisible({ timeout: 10_000 }); - await keyRow.locator("button").first().click(); + await keyRow.getByRole("button", { name: E2E_REGENERATE_KEY_ALIAS }).click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); @@ -109,9 +108,9 @@ test.describe("Proxy Admin - Keys", () => { const before = await findKeyByAlias(page, E2E_UPDATE_LIMITS_KEY_ALIAS); expect(before, `seeded key ${E2E_UPDATE_LIMITS_KEY_ALIAS} exists`).toBeTruthy(); - const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS }); + const keyRow = page.getByRole("row").filter({ hasText: E2E_UPDATE_LIMITS_KEY_ALIAS }); await expect(keyRow).toBeVisible({ timeout: 10_000 }); - await keyRow.locator("button").first().click(); + await keyRow.getByRole("button", { name: E2E_UPDATE_LIMITS_KEY_ALIAS }).click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); @@ -147,9 +146,9 @@ test.describe("Proxy Admin - Keys", () => { await navigateToPage(page, Page.ApiKeys); await dismissFeedbackPopup(page); - const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS }); + const keyRow = page.getByRole("row").filter({ hasText: E2E_DELETE_KEY_ALIAS }); await expect(keyRow).toBeVisible({ timeout: 10_000 }); - await keyRow.locator("button").first().click(); + await keyRow.getByRole("button", { name: E2E_DELETE_KEY_ALIAS }).click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); diff --git a/tests/e2e/ui/tests/settings/scim.spec.ts b/tests/e2e/ui/tests/settings/scim.spec.ts new file mode 100644 index 00000000000..d7dd4248f50 --- /dev/null +++ b/tests/e2e/ui/tests/settings/scim.spec.ts @@ -0,0 +1,53 @@ +import { test, expect, Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; + +async function createScimTokenViaUi(page: PlaywrightPage, alias: string): Promise { + await navigateToPage(page, Page.AdminPanel); + await page.getByRole("tab", { name: "SCIM" }).click(); + + await expect(page.getByText("SCIM Tenant URL")).toBeVisible(); + await expect(page.locator("input[disabled]").first()).toHaveValue(/\/scim\/v2$/); + + await page.getByLabel("Token Name").fill(alias); + await page.getByRole("button", { name: "Create SCIM Token" }).click(); + + await expect(page.getByText(/copy this token now/i)).toBeVisible({ timeout: 15_000 }); + const token = await page.locator('input[type="password"]').inputValue(); + expect(token, "the one-time token panel shows a usable virtual key").toMatch(/^sk-/); + return token; +} + +test.describe("Admin Settings - SCIM", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Create SCIM Token shows the token once and offers to create another", async ({ page }) => { + await createScimTokenViaUi(page, `e2e-scim-ui-${Date.now()}`); + + await page.getByRole("button", { name: "Create Another Token" }).click(); + await expect(page.getByRole("button", { name: "Create SCIM Token" })).toBeVisible(); + await expect(page.getByText(/copy this token now/i)).toBeHidden(); + }); + + test("a UI-minted SCIM token authorizes the SCIM API", async ({ page, request }) => { + test.skip(!process.env.LITELLM_LICENSE, "LITELLM_LICENSE not set in test env — /scim/v2 is premium-gated"); + + const token = await createScimTokenViaUi(page, `e2e-scim-api-${Date.now()}`); + + const denied = await request.get(`${rootPath()}/scim/v2/Groups`, { + headers: { Authorization: "Bearer sk-not-a-real-key" }, + }); + expect(denied.status(), "an unknown key must not reach SCIM").toBe(401); + + const res = await request.get(`${rootPath()}/scim/v2/Groups`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(res.status(), `SCIM Groups listing failed: ${await res.text()}`).toBe(200); + const body = await res.json(); + expect(body.schemas, "SCIM answers with a ListResponse").toContain("urn:ietf:params:scim:api:messages:2.0:ListResponse"); + expect(Array.isArray(body.Resources), "SCIM ListResponse carries a Resources array").toBe(true); + }); +}); diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index f93cca75347..5116cb5df19 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -142,7 +142,7 @@ test.describe("Team Admin", () => { const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); await teamSelect.click(); await page.keyboard.type(E2E_TEAM_CRUD_ALIAS); - await page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_CRUD_ALIAS).first().click(); + await page.getByRole("option", { name: E2E_TEAM_CRUD_ALIAS }).first().click(); // Models — pick "All Team Models". The popup is portaled to the body, so // scope the option lookup to the page. diff --git a/tests/e2e/ui/tests/usage/usagePage.spec.ts b/tests/e2e/ui/tests/usage/usagePage.spec.ts index 8fa59beb905..f61ab018e1b 100644 --- a/tests/e2e/ui/tests/usage/usagePage.spec.ts +++ b/tests/e2e/ui/tests/usage/usagePage.spec.ts @@ -51,20 +51,19 @@ test.describe("Usage page", () => { const card = await openUsage(page); // Table view (the default): the key is listed by its alias. - const row = card.locator("tbody tr").filter({ hasText: alias }); + const row = card.getByRole("row").filter({ hasText: alias }); await expect(row, `${alias} missing from Top Virtual Keys`).toHaveCount(1, { timeout: 30_000, }); // Chart view swaps the table out for the bar chart, and back. await card.getByText("Chart View", { exact: true }).click(); - await expect(card.locator("tbody tr")).toHaveCount(0, { timeout: 10_000 }); + await expect(card.getByRole("table")).toHaveCount(0, { timeout: 10_000 }); await card.getByText("Table View", { exact: true }).click(); await expect(row).toHaveCount(1, { timeout: 10_000 }); - // Clicking the Key ID cell fetches key info and opens the detail panel. // The alias is already in the row behind the modal, so match the panel's own controls. - await row.locator("td").first().click(); + await row.getByRole("button", { name: token }).click(); const keyInfo = page.getByRole("tab", { name: "Overview", exact: true }); await expect(keyInfo, "key info panel did not open").toBeVisible({ timeout: 20_000, diff --git a/tests/e2e/ui/tests/users/searchUsers.spec.ts b/tests/e2e/ui/tests/users/searchUsers.spec.ts index e87218b5a5e..fa8f32764e8 100644 --- a/tests/e2e/ui/tests/users/searchUsers.spec.ts +++ b/tests/e2e/ui/tests/users/searchUsers.spec.ts @@ -1,91 +1,52 @@ -import { test, expect, Page } from "@playwright/test"; +import { test, expect, Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -test.skip("Internal Users Search", () => { +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +const userRows = (page: PlaywrightPage) => page.getByRole("row").filter({ has: page.getByRole("cell") }); + +async function goToInternalUsers(page: PlaywrightPage) { + await navigateToPage(page, Page.Users); + await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible({ timeout: 30_000 }); + await expect(userRows(page)).not.toHaveCount(0, { timeout: 30_000 }); +} + +test.describe("Internal Users Search", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - async function goToInternalUsers(page: Page) { - await page.goto("/ui"); - - const tab = page.getByRole("menuitem", { name: "Internal User" }); - await expect(tab).toBeVisible(); - await tab.click(); - - await expect(page.locator("tbody tr").first()).toBeVisible(); - await expect(page.locator('[data-slot="skeleton"]')).toHaveCount(0); - } - - test("can search users by email", async ({ page }) => { + test("narrows the table to the matching email, and restores it when cleared", async ({ page }) => { await goToInternalUsers(page); - const rows = page.locator("tbody tr"); - const searchInput = page.getByPlaceholder("Search by email..."); + const search = page.getByPlaceholder("Search by email…"); + await expect(search).toBeVisible(); - await expect(searchInput).toBeVisible(); + await search.fill("noteam@"); + await expect(userRows(page)).toHaveCount(1, { timeout: 30_000 }); + await expect(userRows(page).first()).toContainText("noteam@test.local"); - // Ensure initial data is loaded - const initialCount = await rows.count(); - expect(initialCount).toBeGreaterThan(0); - - // 🔹 Apply filter + wait for backend response - await Promise.all([ - page.waitForResponse( - (res) => - res.url().includes("/user/list") && - res.url().includes("user_email=test%40") && // encoded "test@" - res.status() === 200, - ), - searchInput.fill("test@"), - ]); - await page.waitForTimeout(5000); - const filteredCount = await rows.count(); - await expect(filteredCount).toBeLessThan(initialCount); - - // 🔹 Clear filter + wait for unfiltered request - await Promise.all([ - page.waitForResponse( - (res) => res.url().includes("/user/list") && !res.url().includes("user_email=") && res.status() === 200, - ), - searchInput.clear(), - ]); - - const resetCount = await rows.count(); - await expect(resetCount).toBe(initialCount); + await search.clear(); + await expect(userRows(page).filter({ hasText: "admin@test.local" })).not.toHaveCount(0, { timeout: 30_000 }); }); - test("can filter users by user ID and SSO ID", async ({ page }) => { + test("filters the table down to one user by user ID", async ({ page }) => { await goToInternalUsers(page); - const rows = page.locator("tbody tr"); - // Ensure initial data is loaded - const initialCount = await rows.count(); - expect(initialCount).toBeGreaterThan(0); + await page.getByRole("button", { name: "Filters" }).click(); + await page.getByTestId("users-filter-user-id").fill("e2e-internal-noteam"); + await page.getByTestId("filter-drawer-apply").click(); - const filtersButton = page.getByRole("button", { - name: "Filters", - exact: true, - }); - await filtersButton.click(); + await expect(userRows(page)).toHaveCount(1, { timeout: 30_000 }); + await expect(userRows(page).first()).toContainText("noteam@test.local"); + }); - const userIdInput = page.getByPlaceholder("Filter by User ID"); - const ssoIdInput = page.getByPlaceholder("Filter by SSO ID"); - await Promise.all([ - page.waitForResponse( - (res) => res.url().includes("/user/list") && res.url().includes("user_ids=user") && res.status() === 200, - ), - userIdInput.fill("user"), - ]); + test("shows no users when the SSO ID matches nobody", async ({ page }) => { + await goToInternalUsers(page); - await Promise.all([ - page.waitForResponse( - (res) => - res.url().includes("/user/list") && - res.url().includes("user_ids=user") && - res.url().includes("sso_user_ids=sso") && - res.status() === 200, - ), - ssoIdInput.fill("sso"), - ]); - const combinedFilteredCount = await rows.count(); - await expect(combinedFilteredCount).toBeLessThan(initialCount); + await page.getByRole("button", { name: "Filters" }).click(); + await page.getByTestId("users-filter-sso-id").fill("e2e-sso-id-that-matches-nobody"); + await page.getByTestId("filter-drawer-apply").click(); + + await expect(page.getByText("No users found")).toBeVisible({ timeout: 30_000 }); + await expect(userRows(page).filter({ hasText: "noteam@test.local" })).toHaveCount(0); }); }); diff --git a/tests/e2e/ui/tests/users/viewInternalUsers.spec.ts b/tests/e2e/ui/tests/users/viewInternalUsers.spec.ts index 614191372d0..b46fb4d112a 100644 --- a/tests/e2e/ui/tests/users/viewInternalUsers.spec.ts +++ b/tests/e2e/ui/tests/users/viewInternalUsers.spec.ts @@ -1,54 +1,29 @@ -import { test, expect, Page } from "@playwright/test"; +import { test, expect, Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; -test.skip("Internal Users Page", () => { +async function goToInternalUsers(page: PlaywrightPage) { + await navigateToPage(page, Page.Users); + await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible({ timeout: 30_000 }); + await expect(userRows(page)).not.toHaveCount(0, { timeout: 30_000 }); +} + +const userRows = (page: PlaywrightPage) => page.getByRole("row").filter({ has: page.getByRole("cell") }); + +test.describe("Internal Users Page", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - async function goToInternalUsers(page: Page) { - await page.goto("/ui"); - - const internalUserTab = page.getByRole("menuitem", { name: "Internal User" }); - await expect(internalUserTab).toBeVisible(); - await internalUserTab.click(); - - const firstRow = page.locator("tbody tr").first(); - await expect(firstRow).toBeVisible(); - await expect(page.locator('[data-slot="skeleton"]')).toHaveCount(0); - } - - test("renders internal users table correctly", async ({ page }) => { + test("lists the seeded users under the identifying columns", async ({ page }) => { await goToInternalUsers(page); - const rows = page.locator("tbody tr"); - const rowCount = await rows.count(); - expect(rowCount).toBeGreaterThan(0); - - const userIdHeader = page.getByRole("columnheader", { name: "User ID" }); - await expect(userIdHeader).toBeVisible(); - - const virtualKeysHeader = page.getByRole("columnheader", { name: "Virtual Keys" }); - await expect(virtualKeysHeader).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "User ID" })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "Virtual Keys" })).toBeVisible(); }); - test("pagination controls work correctly", async ({ page }) => { + test("cannot page backwards off the first page", async ({ page }) => { await goToInternalUsers(page); - const paginationInfo = page.locator(".text-sm.text-gray-700"); - const prevButton = page.getByRole("button", { name: "Previous" }); - const nextButton = page.getByRole("button", { name: "Next" }); - - const infoText = (await paginationInfo.textContent()) || ""; - - // On first page, Previous should be disabled - if (infoText.includes("1 -")) { - await expect(prevButton).toBeDisabled(); - } - - await page.waitForTimeout(1000); - // Check if there are more pages - const hasMorePages = infoText.includes("of") && !infoText.endsWith("25 of 25"); - if (hasMorePages) { - await expect(nextButton).toBeEnabled(); - } + await expect(page.getByRole("button", { name: "Go to previous page" })).toBeDisabled(); }); }); diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 498d0cb4723..b3d457707b8 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -681,3 +681,25 @@ class TestSpendLogsPartitionDetectionSchemaScope: def test_only_partitioned_relations_match(self, monkeypatch): query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db") assert "pg_partitioned_table" in query + + +class TestSpendLogsPartitionDetectionMissingPsycopg: + """psycopg ships in the `extra_proxy` install, but a stripped-down image + can still lack it. When it does, detection must fail closed to False + (never crash the migration path) and say so loudly, because a silent + False here is what let a genuinely partitioned LiteLLM_SpendLogs hit the + unfiltered primary-key rewrite in production.""" + + def test_missing_psycopg_returns_false(self, monkeypatch): + monkeypatch.setitem(sys.modules, "psycopg", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db") + assert ProxyExtrasDBManager.spend_logs_is_partitioned() is False + + def test_missing_psycopg_logs_a_warning(self, monkeypatch, caplog): + monkeypatch.setitem(sys.modules, "psycopg", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db") + with caplog.at_level("WARNING", logger="litellm_proxy_extras"): + ProxyExtrasDBManager.spend_logs_is_partitioned() + assert any( + "psycopg is not installed" in record.message for record in caplog.records + ) diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index 1838fb16e91..28912a27501 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 3bde72ccd49..35de9961054 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1317,6 +1317,62 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch): assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test" +@pytest.mark.asyncio +async def test_default_team_settings_newrelic_resolves_traces_and_metrics(): + """Static `default_team_settings` is the config-file twin of POST /team/callback. + + A team pinned to New Relic through `default_team_settings` must reach the + same two loggers the dynamic path does: the per-team metrics logger (cost + and usage) and the trace logger (LLM/agent spans). This proves the static + path resolves both, not just one, so the config-file customer gets the + same per-team routing as the API customer. + """ + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc.config = { + "litellm_settings": { + "default_team_settings": [ + { + "team_id": "team-a", + "success_callback": ["newrelic"], + "newrelic_api_key": "team-a-ingest-key", + "newrelic_region": "eu", + } + ] + } + } + + callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config( + team_id="team-a", + proxy_config=pc, + ) + + assert callback_metadata is not None + assert callback_metadata.success_callback == ["newrelic"] + assert callback_metadata.callback_vars == { + "newrelic_api_key": "team-a-ingest-key", + "newrelic_region": "eu", + } + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="static-nr-1", + function_id="static-nr-1", + ) + logging_obj._trusted_callback_vars = tuple(callback_metadata.callback_vars.items()) + + resolved = logging_obj._resolve_dynamic_callback_string("newrelic") + resolved_names = {type(logger).__name__ for logger in resolved} + assert resolved_names == {"NewRelicMetricsLogger", "NewRelicLogger"} + + def test_proxy_config_state_get_config_state_error(): """ Ensures that get_config_state does not raise an error when the config is not a valid dictionary diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index c824d3e7a0e..54dbe5361d7 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,15 +1,14 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). - -Tests are pure unit tests — no Redis server required. -""" - from unittest.mock import AsyncMock, MagicMock, patch import pytest -import redis.asyncio as async_redis -from litellm._redis import get_redis_async_client, get_redis_connection_pool +from litellm._redis import ( + _coerce_redis_kwargs_types, + _get_redis_client_logic, + _get_redis_env_kwarg_mapping, + get_redis_async_client, + get_redis_connection_pool, +) def test_url_config_uses_passed_pool(): @@ -60,16 +59,14 @@ def test_max_connections_url_config_string_value(monkeypatch): assert pool.max_connections == 25 -def test_max_connections_url_config_invalid_value(): - """Invalid max_connections should be silently ignored, falling back - to the pool default (50 for BlockingConnectionPool).""" - with patch("litellm._redis._get_redis_client_logic") as mock_logic: - mock_logic.return_value = { - "url": "redis://localhost:6379/0", - "max_connections": "not_a_number", - } +def test_max_connections_url_config_invalid_value(monkeypatch): + """Invalid max_connections from an env var should be silently dropped, + falling back to the pool default (50 for BlockingConnectionPool).""" + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.setenv("REDIS_MAX_CONNECTIONS", "not_a_number") - pool = get_redis_connection_pool() + pool = get_redis_connection_pool() # BlockingConnectionPool default is 50 assert pool.max_connections == 50 @@ -128,3 +125,173 @@ async def test_disconnect_idempotent(): await cache.disconnect() await cache.disconnect() # should not raise + + +def test_coerce_redis_kwargs_types_int(): + """String values for int-typed Redis params are coerced to int.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "30", "port": "6380", "db": "1"}) + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + assert result["port"] == 6380 + assert result["db"] == 1 + + +def test_coerce_redis_kwargs_types_bool(): + """String values for bool-typed Redis params are coerced to bool.""" + result = _coerce_redis_kwargs_types({"ssl": "true", "decode_responses": "false"}) + assert result["ssl"] is True + assert result["decode_responses"] is False + + +def test_coerce_redis_kwargs_types_none_default_numeric(): + """String values for known None-default numeric params are coerced.""" + result = _coerce_redis_kwargs_types({"max_connections": "20", "socket_timeout": "5.5"}) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + assert result["socket_timeout"] == 5.5 + assert isinstance(result["socket_timeout"], float) + + +def _redis_signature_pre_8x( + socket_timeout=None, + socket_connect_timeout=None, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py <= 7.x Redis signature, where the timeout defaults are None.""" + + +def _redis_signature_8x( + socket_timeout=5, + socket_connect_timeout=5, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py 8.x Redis signature, where the timeout defaults became int 5.""" + + +@pytest.mark.parametrize( + "client", + [_redis_signature_pre_8x, _redis_signature_8x], + ids=["redis-py<=7.x", "redis-py-8.x"], +) +def test_coerce_fractional_socket_timeout_survives_signature_default_change(client): + """redis-py 8.x changed socket_timeout's default from None to int 5. Deriving the + target type from the signature default made int("5.5") raise, so the key was dropped + and REDIS_SOCKET_TIMEOUT=5.5 silently disappeared on 8.x.""" + result = _coerce_redis_kwargs_types( + {"socket_timeout": "5.5", "socket_connect_timeout": "2.5", "max_connections": "20"}, + client=client, + ) + + assert result["socket_timeout"] == pytest.approx(5.5) + assert isinstance(result["socket_timeout"], float) + assert result["socket_connect_timeout"] == pytest.approx(2.5) + assert isinstance(result["socket_connect_timeout"], float) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + + +def test_coerce_invalid_socket_timeout_is_still_dropped(): + """Garbage must not survive the explicit-type path; Redis falls back to its own default.""" + result = _coerce_redis_kwargs_types({"socket_timeout": "not_a_number"}, client=_redis_signature_8x) + + assert "socket_timeout" not in result + + +def test_coerce_redis_kwargs_types_invalid_drops_key(): + """A string that cannot be coerced to the expected numeric type is dropped.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "not_a_number"}) + assert "health_check_interval" not in result + + +def test_coerce_redis_kwargs_types_non_string_unchanged(): + """Non-string values pass through without modification.""" + result = _coerce_redis_kwargs_types({"health_check_interval": 30, "ssl": True}) + assert result["health_check_interval"] == 30 + assert result["ssl"] is True + + +def test_health_check_interval_from_env_is_int(monkeypatch): + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_HEALTH_CHECK_INTERVAL", "30") + + pool = get_redis_connection_pool() + + assert pool is not None + interval = pool.connection_kwargs.get("health_check_interval") + assert interval == 30 + assert isinstance(interval, int), f"Expected int, got {type(interval)}: {interval!r}" + + +def _signature_without_defaults(testkey): + """Stand-in for a client whose parameter declares no default at all.""" + + +def _signature_with_float_default(myparam=1.0): + """Stand-in for a client whose parameter declares a float default.""" + + +def test_coerce_redis_kwargs_types_empty_default_param_unchanged(): + """String params whose signature entry has no default (inspect.Parameter.empty) are left as-is.""" + result = _coerce_redis_kwargs_types({"testkey": "some_value"}, client=_signature_without_defaults) + + assert result["testkey"] == "some_value" + assert isinstance(result["testkey"], str) + + +def test_coerce_redis_kwargs_types_float_valid(): + """String values for params whose signature default is a float are coerced to float.""" + result = _coerce_redis_kwargs_types({"myparam": "3.14"}, client=_signature_with_float_default) + + assert result["myparam"] == pytest.approx(3.14) + assert isinstance(result["myparam"], float) + + +def test_coerce_redis_kwargs_types_float_invalid_drops_key(): + """An unconvertible string for a float-default param is dropped from the result.""" + result = _coerce_redis_kwargs_types({"myparam": "not_a_float"}, client=_signature_with_float_default) + + assert "myparam" not in result + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("false", False), ("true", True), ("0", False), ("1", True)], +) +def test_coerce_socket_keepalive_string(raw, expected): + """socket_keepalive's signature default is None, so it needs an explicit bool + coercion: a leftover "false" string is truthy and enables keepalive.""" + result = _coerce_redis_kwargs_types({"socket_keepalive": raw}) + + assert result["socket_keepalive"] is expected + + +def test_get_redis_client_logic_coerces_cluster_only_kwargs(monkeypatch): + """Cluster-only kwargs (absent from redis.Redis's signature) must still be + coerced when routing to a cluster, or Helm-stringified values reach + RedisCluster as strings.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + result = _get_redis_client_logic( + startup_nodes='[{"host": "localhost", "port": 7000}]', + cluster_error_retry_attempts="5", + require_full_coverage="false", + health_check_interval="30", + ) + + assert result["cluster_error_retry_attempts"] == 5 + assert isinstance(result["cluster_error_retry_attempts"], int) + assert result["require_full_coverage"] is False + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + + +def test_get_redis_client_logic_raises_without_host_or_url(monkeypatch): + """_get_redis_client_logic raises ValueError when neither host nor url is provided.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + with pytest.raises(ValueError, match="Either 'host' or 'url' must be specified for redis"): + _get_redis_client_logic() diff --git a/tests/test_litellm/endpoints/__init__.py b/tests/test_litellm/endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/endpoints/speech/__init__.py b/tests/test_litellm/endpoints/speech/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/__init__.py b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py new file mode 100644 index 00000000000..c0c720bbaf6 --- /dev/null +++ b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -0,0 +1,62 @@ +from typing import Final +from unittest.mock import MagicMock + +import pytest + +import litellm +from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS +from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( + SpeechToCompletionBridgeTransformationHandler, +) + +GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview" + + +def _bridge_request(response_format: str | None) -> dict: + optional_params: Final = ( + {"temperature": 0.4} if response_format is None else {"temperature": 0.4, "response_format": response_format} + ) + return SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Hello from LiteLLM", + voice="Kore", + optional_params=optional_params, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + +@pytest.mark.parametrize("response_format", ["wav", "mp3", "pcm", None]) +def test_gemini_tts_request_keeps_speech_response_format_out_of_chat_params(response_format: str | None) -> None: + request: Final = _bridge_request(response_format) + + assert "response_format" not in request + assert request["audio"] == {"voice": "Kore", "format": "pcm16"} + assert request["temperature"] == 0.4 + assert request["modalities"] == ["audio"] + + gemini_params: Final = litellm.get_optional_params( + model=GEMINI_TTS_MODEL, + custom_llm_provider="gemini", + **{param: value for param, value in request.items() if param in OPENAI_CHAT_COMPLETION_PARAMS}, + ) + assert gemini_params["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + assert "responseMimeType" not in gemini_params + + +def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> None: + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model="gpt-4o-audio-preview", + input="Hello from LiteLLM", + voice="alloy", + optional_params={"response_format": "wav"}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + + assert "response_format" not in request + assert request["audio"] == {"voice": "alloy", "format": "wav"} diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 115e385eda4..4aa28b5abfd 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" import json +from dataclasses import replace import pytest @@ -215,6 +216,27 @@ def test_genai_mapper_all_request_params(): assert attrs["server.port"] == 443 +def test_genai_mapper_cache_token_attrs(): + cached = replace( + _full_llm_call(), + usage=LLMUsage( + input_tokens=10, + output_tokens=5, + total_tokens=15, + cache_creation_input_tokens=7, + cache_read_input_tokens=3, + ), + ) + attrs = GenAIMapper().map(cached) + assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7 + assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3 + + # No cache usage keeps the span sparse: neither key present. + uncached = GenAIMapper().map(_full_llm_call()) + assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached + assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached + + def test_genai_mapper_stamps_input_output_messages(): data = LLMCallSpanData( operation=GenAIOperation.CHAT, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index baa72b5a7fe..ca628aa3405 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -525,6 +525,28 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): + payload = _sample_payload() + payload["metadata"] = { + **payload["metadata"], + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 5, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 3, + }, + } + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.usage.cache_creation_input_tokens == 7 + assert data.usage.cache_read_input_tokens == 3 + + +def test_llm_call_adapter_cache_tokens_none_without_usage_object(): + data = LLMCallSpanData.from_standard_logging_payload(_sample_payload()) + assert data.usage.cache_creation_input_tokens is None + assert data.usage.cache_read_input_tokens is None + + def test_llm_call_adapter_failure_path(): payload = _sample_payload( status="failure", diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index f9c287fc7b7..1af3dd3f613 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -58,11 +58,12 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock: return prisma -def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock: +def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock: record = MagicMock() for field, value in dict( id=job.id, - api_key_id=api_key_id, + target_type=target_type, + target_id=target_id, router_name=job.router_name, direction=job.direction, baseline_model=job.baseline_model, @@ -123,7 +124,7 @@ def _spend_counter(store=None): return counter, read, write -def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger: +def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger: cache = InMemoryCache(max_size_in_memory=4, default_ttl=60) counter, read, write = _spend_counter(counter_store) funnel_events = [] @@ -137,8 +138,9 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEval ) logger._test_counter = counter logger._test_funnel = funnel_events - if jobs: - cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)}) + seeded = jobs_by_target if jobs_by_target is not None else ({("key", "key-hash"): tuple(jobs)} if jobs else None) + if seeded is not None: + cache.set_cache("shadow_eval:active_jobs", seeded) return logger @@ -837,6 +839,86 @@ class TestSuccessHookSkipChain: prisma.db.litellm_shadowevalattempt.create.assert_not_called() +JWT_IDENTITY = {"user_api_key_hash": None, "user_api_key_team_id": "team-eng", "user_api_key_user_id": "dev-alice"} + + +@pytest.mark.asyncio +class TestTargetMatching: + """A request qualifies for a job through ANY of its resolved identities: key hash, + team id, or user id. Team and user jobs must therefore sample JWT-authenticated + traffic, which carries no key hash at all.""" + + @pytest.mark.parametrize( + "target,sampled", + [ + (("team", "team-eng"), True), + (("user", "dev-alice"), True), + (("key", "some-key"), False), + ], + ids=["team-job-samples-jwt-traffic", "user-job-samples-jwt-traffic", "key-jobs-never-match-keyless-traffic"], + ) + async def test_jwt_shaped_traffic_matches_team_and_user_jobs_but_no_key_job(self, target, sampled): + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, jobs_by_target={target: (_job(),)}) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = dict(JWT_IDENTITY) + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + if sampled: + prisma.db.litellm_shadowevalattempt.create.assert_awaited_once() + assert prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["job_id"] == "job-1" + else: + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + async def test_an_event_with_no_identity_early_returns_without_a_cache_read(self): + prisma = _prisma() + router = _router() + cache = MagicMock(spec=InMemoryCache) + cache.async_get_cache = AsyncMock() + logger = ShadowEvalLogger( + router_provider=lambda: router, + prisma_provider=lambda: prisma, + jobs_cache=cache, + ) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = {} + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + + cache.async_get_cache.assert_not_awaited() + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + async def test_an_event_matching_a_key_job_and_a_team_job_fires_both(self): + """A request's key and its team can each hold a job; the two are separately + budgeted experiments, so both fire and each counts its own start.""" + prisma = _prisma() + logger = _logger( + router=_router(), + prisma=prisma, + jobs_by_target={ + ("key", "key-hash"): (_job(id="key-job"),), + ("team", "team-eng"): (_job(id="team-job"),), + }, + ) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = { + "user_api_key_hash": "key-hash", + "user_api_key_team_id": "team-eng", + } + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list] + assert sorted(row["job_id"] for row in rows) == ["key-job", "team-job"] + assert logger._job_starts == {"key-job": 1, "team-job": 1} + + @pytest.mark.asyncio class TestActiveJobsCache: async def test_cache_miss_reads_db_once_then_serves_from_cache(self): @@ -851,8 +933,8 @@ class TestActiveJobsCache: first = await logger._active_jobs() second = await logger._active_jobs() - assert [job.id for job in first["key-hash"]] == ["job-1"] - assert second["key-hash"][0].attempts == 7 + assert [job.id for job in first[("key", "key-hash")]] == ["job-1"] + assert second[("key", "key-hash")][0].attempts == 7 assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 where = prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["where"] assert where["stopped_at"] is None @@ -899,8 +981,8 @@ class TestActiveJobsCache: jobs = await logger._active_jobs() assert logger._job_starts == {} - assert jobs["key-hash"][0].attempts == 7 - assert jobs["key-hash"][0].spend == 0.05 + assert jobs[("key", "key-hash")][0].attempts == 7 + assert jobs[("key", "key-hash")][0].spend == 0.05 @pytest.mark.asyncio @@ -1249,13 +1331,14 @@ class TestActiveJobsFailClosed: jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), ) - assert [job.id for job in (await logger._active_jobs())["key-hash"]] == ["job-ok"] + assert [job.id for job in (await logger._active_jobs())[("key", "key-hash")]] == ["job-ok"] - async def test_both_of_a_key_s_jobs_survive_the_lookup(self): + async def test_every_targets_jobs_survive_the_lookup_keyed_by_type_and_id(self): records = [ _job_record(_job(id="job-forward")), _job_record(_reverse_job(id="job-reverse")), - _job_record(_job(id="job-other"), api_key_id="other-key"), + _job_record(_job(id="job-other"), target_id="other-key"), + _job_record(_job(id="job-team"), target_type="team", target_id="team-eng"), ] prisma = _prisma(jobs=records, attempt_counts=[("job-reverse", 3)]) logger = ShadowEvalLogger( @@ -1266,9 +1349,11 @@ class TestActiveJobsFailClosed: jobs = await logger._active_jobs() - assert sorted(job.id for job in jobs["key-hash"]) == ["job-forward", "job-reverse"] - assert [job.id for job in jobs["other-key"]] == ["job-other"] - assert {job.id: job.attempts for job in jobs["key-hash"]}["job-reverse"] == 3 + assert sorted(job.id for job in jobs[("key", "key-hash")]) == ["job-forward", "job-reverse"] + assert [job.id for job in jobs[("key", "other-key")]] == ["job-other"] + assert [job.id for job in jobs[("team", "team-eng")]] == ["job-team"] + assert ("team-eng",) not in jobs and "team-eng" not in jobs + assert {job.id: job.attempts for job in jobs[("key", "key-hash")]}["job-reverse"] == 3 def _failing_router(): diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 772fbf98c57..9ab66d55f3f 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1433,3 +1433,50 @@ class TestFlattenTopLevelSchemaCombinators: flatten_top_level_schema_combinators(schema) assert schema == snapshot + + +class TestRequestContainsImageContent: + """One detector for every dialect that reaches pre-routing hooks untranslated.""" + + @pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + }, + ], + ) + def test_detects_every_image_dialect_including_tool_results(self, part): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}, part]}] + assert request_contains_image_content(messages) is True + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "user", "content": "plain string"}], + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "x"}}]}], + [{"role": "user", "content": [{"type": "tool_result", "content": [{"type": "text", "text": "ok"}]}]}], + [{"role": "user", "content": None}], + [], + ], + ) + def test_ignores_text_audio_and_degenerate_shapes(self, messages): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + assert request_contains_image_content(messages) is False + + def test_hostile_nesting_is_depth_bounded(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + nested: dict = {"type": "image", "source": {"type": "base64", "data": "aGk="}} + for _ in range(50): + nested = {"type": "tool_result", "content": [nested]} + assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False diff --git a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py index 6cacd119030..419ca104bb1 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py +++ b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py @@ -184,3 +184,26 @@ class TestTogetherApiBaseResolvesProvider: assert provider == "together_ai" assert api_base == "https://api.together.ai/v1" + + +class TestGigachatApiBaseResolvesProvider: + """ + Regression for the GigaChat api_base branch: the provider-mapping chain + carried an ``endpoint == "https://gigachat.devices.sberbank.ru/api/v1"`` + elif, but the URL was never added to ``openai_compatible_endpoints``, so + the endpoint loop never fired the branch and a caller-supplied GigaChat + api_base raised BadRequestError instead of resolving to ``gigachat``. + """ + + def test_gigachat_api_base_resolves_to_gigachat(self, monkeypatch): + monkeypatch.setenv("GIGACHAT_API_KEY", "gigachat-key-from-env") + + model, provider, dynamic_api_key, returned_api_base = get_llm_provider( + model="GigaChat-2", + api_base="https://gigachat.devices.sberbank.ru/api/v1", + ) + + assert provider == "gigachat" + assert dynamic_api_key == "gigachat-key-from-env" + assert returned_api_base == "https://gigachat.devices.sberbank.ru/api/v1" + assert model == "GigaChat-2" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 947a55410ef..c7328adb0b3 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6101,3 +6101,64 @@ def test_response_timing_metrics_survive_deepcopy(logging_obj): logging_obj.set_response_timing_metrics({"_response_ms": 12.5}) assert copy.deepcopy(logging_obj).response_timing_metrics == {"_response_ms": 12.5} + + +def test_passthrough_embeddings_result_swapped_for_callbacks(): + """ + Regression: for gigachat passthrough /embeddings, normalize_logging_result + produces an EmbeddingResponse, but the result swap only accepted + ModelResponse, so callbacks kept receiving the raw httpx.Response (which + crashes attribute readers like OTEL). The swap must cover + EmbeddingResponse too. + """ + import datetime as dt + + from litellm.types.utils import EmbeddingResponse + + logging_obj = LitellmLogging( + model="EmbeddingsGigaR", + messages=[], + stream=False, + call_type="allm_passthrough_route", + start_time=time.time(), + litellm_call_id="passthrough-embed-call-id", + function_id="passthrough-embed-fn-id", + ) + logging_obj.update_environment_variables( + litellm_params={}, + optional_params={}, + model="EmbeddingsGigaR", + custom_llm_provider="gigachat", + endpoint="/embeddings", + request_data={"model": "EmbeddingsGigaR", "input": ["hello"]}, + input=["hello"], + ) + + httpx_response = httpx.Response( + 200, + json={ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 0, + "usage": {"prompt_tokens": 5}, + } + ], + "model": "EmbeddingsGigaR", + }, + request=httpx.Request( + "POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings" + ), + ) + + _, _, swapped_result = logging_obj._success_handler_helper_fn( + result=httpx_response, + start_time=dt.datetime.now(), + end_time=dt.datetime.now(), + cache_hit=False, + ) + + assert isinstance(swapped_result, EmbeddingResponse) + assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3] diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 572b505e94c..4694fa8fbed 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens(): {"type": "document", "source": source}, ] ) + + +def test_openai_file_block_prices_like_the_equivalent_anthropic_document(): + """An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise. + + Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject` + is in the union this counter accepts, so every local count of a Responses `input_file` raised + `Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens. + """ + prompt = {"type": "text", "text": "Summarize this file."} + inline_file = { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"}, + } + document = { + "type": "document", + "title": "report.pdf", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + } + + assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document]) + assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt]) + + +def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): + """A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens.""" + prompt = {"type": "text", "text": "Summarize this file."} + + by_id = {"type": "file", "file": {"file_id": "file-abc123"}} + assert _count_user_content([prompt, by_id]) == _count_user_content([prompt]) + + named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}} + assert _count_user_content([prompt, named]) == _count_user_content( + [prompt, {"type": "text", "text": "report.pdf"}] + ) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index af3ccd65b11..0fe7730e91e 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -1490,6 +1490,92 @@ class MockCanaryMaskingGuardrail(CustomGuardrail): return inputs +class TestAnthropicMessagesImageSources: + """An Anthropic image block has three source shapes (`AnthropicMessagesImageParam.source`). + + Only the base64 one carries "data", so reading that key alone drops url images + entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"], + not just Bedrock. + """ + + def _data(self, messages): + return {"model": "claude-sonnet-4-5", "messages": messages} + + async def _images_seen(self, content) -> list[str]: + handler = AnthropicMessagesHandler() + + class ImageRecordingGuardrail(MockCanaryMaskingGuardrail): + def __init__(self): + super().__init__() + self.seen_images: list[str] = [] # mutable-ok: accumulator for the assertion + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + self.seen_images.extend(inputs.get("images") or []) + return await super().apply_guardrail(inputs, request_data, input_type, logging_obj) + + guardrail = ImageRecordingGuardrail() + # The text block is what gets the guardrail invoked at all: a message with + # no text gives the handler nothing to scan, so it never reaches the + # guardrail and every source shape would look equally "dropped". + await handler.process_input_messages( + data=self._data([{"role": "user", "content": [{"type": "text", "text": "describe it"}, *content]}]), + guardrail_to_apply=guardrail, + ) + return guardrail.seen_images + + @pytest.mark.asyncio + async def test_url_source_reaches_the_guardrail(self): + """A url source has no "data" key, so it used to yield nothing at all.""" + seen = await self._images_seen( + [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}] + ) + + assert seen == ["https://example.com/a.png"] + + @pytest.mark.asyncio + async def test_base64_source_carries_its_media_type(self): + """Bare base64 leaves the consumer no way to recover the format. + + An API like Bedrock's ApplyGuardrail needs it to build the request, so the + media_type travels with the payload as a data URI. + """ + seen = await self._images_seen( + [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}] + ) + + assert seen == ["data:image/png;base64,AAAA"] + + @pytest.mark.asyncio + async def test_base64_source_without_a_media_type_is_passed_through(self): + """There is no format to attach, so the payload goes through unchanged.""" + seen = await self._images_seen([{"type": "image", "source": {"type": "base64", "data": "AAAA"}}]) + + assert seen == ["AAAA"] + + @pytest.mark.asyncio + async def test_file_source_yields_nothing(self): + """The bytes live behind the Files API and this extractor has no client. + + Documented as a known gap rather than silently handed on as a file_id string, + which a consumer would try to decode as an image. + """ + seen = await self._images_seen([{"type": "image", "source": {"type": "file", "file_id": "file_abc"}}]) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_malformed_source_is_dropped_rather_than_passed_on(self): + seen = await self._images_seen( + [ + {"type": "image", "source": {"type": "base64"}}, + {"type": "image", "source": {"type": "url"}}, + {"type": "image", "source": {"type": "base64", "data": ""}}, + ] + ) + + assert seen == [] + + class TestAnthropicMessagesToolResultScanning: """LIT-5251: tool_result blocks carry whatever a client's local tool fetched, so they are the request-path payload an indirect prompt injection actually arrives in. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py index b8ce11db8d1..11a048edc1f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py @@ -6,9 +6,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.experimental_pass_through.messages import ( - streaming_iterator as streaming_iterator_module, -) +from litellm.llms.anthropic.experimental_pass_through.messages import streaming_iterator as streaming_iterator_module from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( INCOMPLETE_STREAM_ERROR_MESSAGE, AnthropicMessagesStreamHiddenParams, @@ -338,47 +336,6 @@ async def _events_then_hang(events): await asyncio.Event().wait() -@pytest.mark.asyncio -async def test_async_sse_wrapper_logs_partial_chunks_on_client_disconnect(): - """ - Regression test for LIT-5839: a client disconnect tears the generator - down with GeneratorExit at the yield, which used to skip the post-loop - logging dispatch entirely, so the partial output tokens the provider - already generated (and billed) never reached spend tracking. - """ - iterator = _RecordingLoggingIterator( - litellm_logging_obj=_make_logging_obj("test_disconnect_logs_partial_chunks"), - request_body={}, - ) - wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS)) - streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))] - assert iterator.logging_call_count == 0 - - await wrapped.aclose() - - assert iterator.logging_call_count == 1 - assert iterator.logged_chunks == streamed - - -@pytest.mark.asyncio -async def test_async_sse_wrapper_logs_partial_chunks_on_cancellation(): - iterator = _RecordingLoggingIterator( - litellm_logging_obj=_make_logging_obj("test_cancellation_logs_partial_chunks"), - request_body={}, - ) - wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS)) - streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))] - - consume_task = asyncio.ensure_future(wrapped.__anext__()) - await asyncio.sleep(0.01) - consume_task.cancel() - with pytest.raises(asyncio.CancelledError): - await consume_task - - assert iterator.logging_call_count == 1 - assert iterator.logged_chunks == streamed - - @pytest.mark.asyncio async def test_async_sse_wrapper_skips_logging_on_disconnect_before_first_chunk(): iterator = _RecordingLoggingIterator( @@ -408,6 +365,561 @@ def test_incomplete_stream_error_sse_event_is_valid_anthropic_error(): assert event.endswith("\n\n") +_STREAM_PREFIX = ( + {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "The Roman"}}, +) +_STREAM_TAIL = ( + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " Empire ..."}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 64}}, + {"type": "message_stop"}, +) + + +def _output_tokens_from_logged_chunks(chunks: list[bytes]) -> int | None: + """Read the last output_tokens the billing path would see from the SSE bytes.""" + latest: int | None = None + for raw in chunks: + for line in raw.decode().splitlines(): + if not line.startswith("data:"): + continue + data = json.loads(line[len("data:"):].strip()) + usage = data.get("usage") if isinstance(data, dict) else None + if isinstance(usage, dict) and usage.get("output_tokens") is not None: + latest = usage["output_tokens"] + return latest + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_bills_full_stream_after_client_disconnect(): + """ + Regression: on a client disconnect mid-stream the upstream provider keeps + generating (and billing) the full response. The wrapper must keep draining + that upstream to its terminal ``message_delta`` and bill the real + output_tokens (64), not the partial count the client drained before leaving + (the message_start placeholder, 1). + + A ``tail_gated`` event holds back the stream tail until the client has + disconnected, so the tail can only be captured by a drain that survives the + client teardown - exactly the path the previous implementation dropped. + """ + tail_gated = asyncio.Event() + upstream_fully_drained = asyncio.Event() + + async def _gated_stream(): + for event in _STREAM_PREFIX: + yield event + await tail_gated.wait() + for event in _STREAM_TAIL: + yield event + upstream_fully_drained.set() + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_bills_full_stream_after_disconnect"), + request_body={}, + ) + + gen = iterator.async_sse_wrapper(_gated_stream()) + + client_chunks = [] + async for chunk in gen: + client_chunks.append(chunk) + if len(client_chunks) == len(_STREAM_PREFIX): + break + await gen.aclose() + + tail_gated.set() + await asyncio.wait_for(upstream_fully_drained.wait(), timeout=5) + for _ in range(100): + if iterator.logged_chunks: + break + await asyncio.sleep(0.01) + + assert len(client_chunks) == len(_STREAM_PREFIX) + + assert iterator.logged_chunks, "pump never billed after client disconnect" + assert _output_tokens_from_logged_chunks(iterator.logged_chunks) == 64 + assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + assert not any(c.startswith(b"event: error\n") for c in iterator.logged_chunks) + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_bills_full_stream_when_client_reads_all(): + """Happy path: when the client drains the whole stream, billing still sees + the terminal output_tokens (64) and the client gets every chunk.""" + tail_gated = asyncio.Event() + tail_gated.set() # no gating; full stream flows immediately + + async def _full_stream(): + for event in (*_STREAM_PREFIX, *_STREAM_TAIL): + yield event + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_bills_full_stream_happy_path"), + request_body={}, + ) + client_chunks = [chunk async for chunk in iterator.async_sse_wrapper(_full_stream())] + + for _ in range(100): + if iterator.logged_chunks: + break + await asyncio.sleep(0.01) + + assert len(client_chunks) == len(_STREAM_PREFIX) + len(_STREAM_TAIL) + assert _output_tokens_from_logged_chunks(iterator.logged_chunks) == 64 + assert not any(c.startswith(b"event: error\n") for c in iterator.logged_chunks) + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_dispatches_deferred_logging_when_client_disconnects_mid_tail(): + """ + Regression: when the pump finishes draining while the client is still + connected, ``_handle_streaming_logging`` defers billing for the proxy's + post-response hook (``ProxyLogging._fire_deferred_stream_logging``), which + only fires on a normally completed response. If the client then disconnects + before consuming the queued tail, the response generator tears down via + GeneratorExit and that hook never runs. The relay teardown must dispatch + the stored deferred billing itself, or the request logs no spend at all. + """ + dispatched = [] + deferred_fired = asyncio.Event() + + def _deferred_stream_complete(logging_coroutine): + dispatched.append(logging_coroutine) + + async def _consume(): + logging_coroutine.close() + deferred_fired.set() + + return _consume() + + logging_obj = _make_logging_obj("test_deferred_dispatch_on_disconnect_mid_tail") + logging_obj._on_deferred_stream_complete = _deferred_stream_complete + iterator = BaseAnthropicMessagesStreamingIterator(litellm_logging_obj=logging_obj, request_body={}) + + async def _full_stream(): + for event in (*_STREAM_PREFIX, *_STREAM_TAIL): + yield event + + gen = iterator.async_sse_wrapper(_full_stream()) + client_chunks = [] + async for chunk in gen: + client_chunks.append(chunk) + if len(client_chunks) == len(_STREAM_PREFIX): + break + + for _ in range(100): + if getattr(logging_obj, "_deferred_stream_complete_args", None) is not None: + break + await asyncio.sleep(0.01) + assert getattr(logging_obj, "_deferred_stream_complete_args", None) is not None, "pump never deferred billing" + + await gen.aclose() + + assert len(dispatched) == 1, "relay teardown did not dispatch the deferred billing" + assert logging_obj._on_deferred_stream_complete is None + assert logging_obj._deferred_stream_complete_args is None + await asyncio.wait_for(deferred_fired.wait(), timeout=5) + + +class _ProviderStreamError(Exception): + """Stand-in for a provider-specific streaming failure carrying a status code.""" + + def __init__(self, message: str, status_code: int): + super().__init__(message) + self.status_code = status_code + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_reraises_upstream_error_to_connected_client(): + """ + Regression: an upstream failure (Bedrock read / decode / chunk-conversion) + before message_stop must propagate the ORIGINAL provider exception to a + still-connected client, so the proxy's failure handling keeps the + provider-specific status. The pump must not swallow it into a generic + api_error event + normal termination. + """ + + async def _failing_stream(): + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} + raise _ProviderStreamError("bedrock stream blew up", status_code=529) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_reraises_upstream_error"), + request_body={}, + ) + + received = [] + + async def _drain(): + async for chunk in iterator.async_sse_wrapper(_failing_stream()): + received.append(chunk) + + with pytest.raises(_ProviderStreamError) as excinfo: + await _drain() + + assert excinfo.value.status_code == 529 + assert received + assert not any(c.startswith(b"event: error\n") for c in received) + assert iterator.logged_chunks == [] + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_salvages_partial_spend_on_upstream_error_after_disconnect(): + """ + When the upstream errors AFTER the client has already disconnected there is + no live client to re-raise to and no failure hook will run, so the pump + salvages partial spend from what it collected instead of dropping the row. + """ + tail_gated = asyncio.Event() + + async def _gated_failing_stream(): + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} + await tail_gated.wait() + raise _ProviderStreamError("late failure", status_code=500) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_salvage_partial_on_late_error"), + request_body={}, + ) + + gen = iterator.async_sse_wrapper(_gated_failing_stream()) + received = [await gen.__anext__(), await gen.__anext__()] + await gen.aclose() # client disconnects before the upstream error + + tail_gated.set() # let the upstream raise now, after disconnect + for _ in range(100): + if iterator.logged_chunks: + break + await asyncio.sleep(0.01) + + assert len(received) == 2 + assert iterator.logged_chunks == received + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_salvages_spend_when_queued_error_is_never_consumed(): + """ + When the upstream errors while the client is still connected, the pump + forwards the exception through the queue expecting the relay to re-raise it + into the proxy's failure handling. If the client disconnects before + consuming that queued exception, the handoff never happens and no failure + hook runs, so the pump must notice the unconsumed exception at teardown and + salvage partial spend instead of dropping the row entirely. + """ + upstream_errored = asyncio.Event() + + async def _failing_stream(): + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 52, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} + upstream_errored.set() + raise _ProviderStreamError("mid-stream failure", status_code=500) + + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_salvage_on_unconsumed_queued_error"), + request_body={}, + ) + + gen = iterator.async_sse_wrapper(_failing_stream()) + received = [await gen.__anext__(), await gen.__anext__()] + await upstream_errored.wait() # exception is now queued behind the consumed chunks + await gen.aclose() # client disconnects without ever consuming the queued exception + + for _ in range(100): + if iterator.logged_chunks: + break + await asyncio.sleep(0.01) + + assert iterator.logging_call_count == 1 + assert iterator.logged_chunks == received + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_applies_backpressure_to_slow_client(monkeypatch): + """ + Regression: the relay queue is bounded, so a slow client throttles the + upstream read instead of letting the pump buffer the whole response in + memory. With a tiny queue and a client that reads a single chunk, the pump + must stall after producing only a queue's worth of chunks ahead, not race + to the end of a large stream. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2) + + total = 200 + produced = 0 + + async def _fast_stream(): + nonlocal produced + for i in range(total): + produced += 1 + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} + + iterator = _make_iterator("test_backpressure_slow_client") + gen = iterator.async_sse_wrapper(_fast_stream()) + try: + await gen.__anext__() + for _ in range(500): + await asyncio.sleep(0) + assert produced <= 2 + 3, f"pump ran ahead unthrottled: produced {produced} of {total}" + assert produced < total + finally: + await gen.aclose() + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_bills_partial_when_detached_drain_cap_reached(monkeypatch): + """ + Regression: when the concurrent detached-drain cap is already reached, a + pump whose client has disconnected must bill what it collected instead of + continuing to drain (and accumulating) the rest of a large upstream stream, + so slow/abandoned clients can't pin unbounded worker state. + + The cap slot set is pre-occupied so the single slot is unavailable when this + pump reaches its first post-disconnect chunk; that isolates the cap decision + from multi-pump scheduling races. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4) + + async def _hold_slot(): + await asyncio.sleep(3600) + + holder = asyncio.ensure_future(_hold_slot()) + streaming_iterator_module._DETACHED_STREAM_DRAINS.add(holder) + tail_reached = False + + async def _long_stream(): + nonlocal tail_reached + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}} + for i in range(100): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"more{i}"}} + tail_reached = True + yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}} + yield {"type": "message_stop"} + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drain_cap_full"), request_body={}) + try: + gen = iterator.async_sse_wrapper(_long_stream()) + await gen.__anext__() # message_start + await gen.__anext__() # first delta + await gen.aclose() # client disconnects; 100+ chunks remain upstream + + for _ in range(200): + if iterator.logged_chunks: + break + await asyncio.sleep(0) + + assert iterator.logged_chunks, "capped pump never billed" + assert len(iterator.logged_chunks) <= 2 + streaming_iterator_module.ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE + assert len(iterator.logged_chunks) < 100 + assert not any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + assert tail_reached is False, "pump kept draining past the cap instead of stopping" + finally: + holder.cancel() + streaming_iterator_module._DETACHED_STREAM_DRAINS.discard(holder) + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_bills_partial_when_detached_drains_disabled(monkeypatch): + """ + Regression: ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS=0 must disable + detached draining entirely, not just shrink the cap. With no slots ever + available, the very first post-disconnect chunk must fall back to partial + spend logging instead of hanging on a cap that's unreachable. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 0) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4) + + tail_reached = False + + async def _long_stream(): + nonlocal tail_reached + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}} + for i in range(100): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"more{i}"}} + tail_reached = True + yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}} + yield {"type": "message_stop"} + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drains_disabled"), request_body={}) + gen = iterator.async_sse_wrapper(_long_stream()) + await gen.__anext__() # message_start + await gen.__anext__() # first delta + await gen.aclose() # client disconnects; 100+ chunks remain upstream + + for _ in range(200): + if iterator.logged_chunks: + break + await asyncio.sleep(0) + + assert iterator.logged_chunks, "pump never billed with detached drains disabled" + assert len(iterator.logged_chunks) <= 2 + streaming_iterator_module.ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE + assert len(iterator.logged_chunks) < 100 + assert not any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + assert tail_reached is False, "pump kept draining despite detached drains being disabled" + assert len(streaming_iterator_module._DETACHED_STREAM_DRAINS) == 0 + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_aborts_upstream_when_detached_drain_cap_reached(monkeypatch): + """ + Regression: when the cap is full and a disconnected pump bails, it must call + aclose on the upstream stream so the provider stops generating and billing, + not continue running the stream while we record only the partial prefix. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4) + + async def _hold_slot(): + await asyncio.sleep(3600) + + holder = asyncio.ensure_future(_hold_slot()) + streaming_iterator_module._DETACHED_STREAM_DRAINS.add(holder) + + class _AbortableStream: + def __init__(self): + self.aclose_called = False + self._remaining = iter( + ( + {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}}, + ) + + tuple( + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"t{i}"}} + for i in range(50) + ) + ) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._remaining) + except StopIteration: + raise StopAsyncIteration + + async def aclose(self): + self.aclose_called = True + + stream = _AbortableStream() + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("abort_upstream_at_cap"), request_body={}) + try: + gen = iterator.async_sse_wrapper(stream) + await gen.__anext__() + await gen.__anext__() + await gen.aclose() + + for _ in range(200): + if iterator.logged_chunks: + break + await asyncio.sleep(0) + + assert iterator.logged_chunks, "capped pump never billed" + assert stream.aclose_called, "upstream aclose was not called when the detached-drain cap was reached" + finally: + holder.cancel() + streaming_iterator_module._DETACHED_STREAM_DRAINS.discard(holder) + + +@pytest.mark.asyncio +async def test_abort_upstream_logs_warning_when_aclose_raises(caplog): + """_abort_upstream must swallow and log any exception from aclose().""" + import logging + + class _ExplodingStream: + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + async def aclose(self): + raise RuntimeError("aclose exploded") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await BaseAnthropicMessagesStreamingIterator._abort_upstream(_ExplodingStream()) + + assert any("abort" in r.message and "RuntimeError" in r.message for r in caplog.records) + + +@pytest.mark.asyncio +async def test_enqueue_for_client_returns_false_when_already_detached(): + """_enqueue_for_client must return False immediately (without touching the queue) + when client_detached is already set before the call.""" + from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + BaseAnthropicMessagesStreamingIterator, + ) + + queue: asyncio.Queue[bytes | None | BaseException] = asyncio.Queue(maxsize=1) + client_detached = asyncio.Event() + client_detached.set() + + result = await BaseAnthropicMessagesStreamingIterator._enqueue_for_client(queue, client_detached, b"chunk") + assert result is False + assert queue.empty() + + +@pytest.mark.asyncio +async def test_enqueue_for_client_returns_false_when_client_detaches_while_queue_full(): + """_enqueue_for_client must return False (and cancel the put) when the queue + is full and client_detached fires before space becomes available.""" + from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + BaseAnthropicMessagesStreamingIterator, + ) + + queue: asyncio.Queue[bytes | None | BaseException] = asyncio.Queue(maxsize=1) + queue.put_nowait(b"already-full") + + client_detached = asyncio.Event() + + async def _set_detached_soon(): + await asyncio.sleep(0.01) + client_detached.set() + + asyncio.create_task(_set_detached_soon()) + result = await BaseAnthropicMessagesStreamingIterator._enqueue_for_client(queue, client_detached, b"new-chunk") + assert result is False + assert queue.qsize() == 1 + assert queue.get_nowait() == b"already-full" + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_drains_detached_when_cap_available(monkeypatch): + """Complement to the cap test: with a slot free, a disconnected pump drains + the full upstream and bills the terminal usage, and releases its slot after.""" + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 1) + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 4) + + async def _stream(): + yield {"type": "message_start", "message": {"id": "m", "usage": {"input_tokens": 5, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "x"}} + for i in range(20): + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": f"m{i}"}} + yield {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 42}} + yield {"type": "message_stop"} + + iterator = _RecordingLoggingIterator(litellm_logging_obj=_make_logging_obj("drain_cap_free"), request_body={}) + gen = iterator.async_sse_wrapper(_stream()) + await gen.__anext__() + await gen.__anext__() + await gen.aclose() + + for _ in range(300): + if iterator.logged_chunks: + break + await asyncio.sleep(0.01) + + assert any(c.startswith(b"event: message_stop\n") for c in iterator.logged_chunks) + assert len(streaming_iterator_module._DETACHED_STREAM_DRAINS) == 0 + + def _decode_sse_events(events: tuple[bytes, ...]) -> list[tuple[str, dict]]: decoded = [] for event in events: @@ -599,20 +1111,35 @@ async def test_normal_end_with_deferred_dispatch_armed_parks_logging_coroutine(m @pytest.mark.asyncio async def test_client_disconnect_enqueues_immediately_even_when_deferred_dispatch_armed(monkeypatch): """ - On client disconnect the guardrail end-of-stream scan never runs, so - deferral would strand the spend log; the teardown path must keep - enqueueing immediately (LIT-5839) even when the deferred callback is armed. + Regression: on client disconnect the guardrail end-of-stream scan never + runs, so deferral would strand the spend log. The detached pump's + post-disconnect bill must bypass the deferred-dispatch park and enqueue + immediately (LIT-5839) even when the deferred callback is armed (LIT-6409). """ worker = _RecordingLoggingWorker() monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker) iterator = _make_iterator("test_disconnect_enqueues_when_armed") iterator.litellm_logging_obj._on_deferred_stream_complete = _noop_deferred_dispatch - wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS)) + tail_gated = asyncio.Event() + + async def _gated_stream(): + for event in TRUNCATED_TOOL_USE_EVENTS: + yield event + await tail_gated.wait() + yield {"type": "message_stop"} + + wrapped = iterator.async_sse_wrapper(_gated_stream()) for _ in range(len(TRUNCATED_TOOL_USE_EVENTS)): await wrapped.__anext__() await wrapped.aclose() + tail_gated.set() + for _ in range(100): + if worker.enqueued: + break + await asyncio.sleep(0.01) + assert len(worker.enqueued) == 1 assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None worker.close_enqueued() @@ -629,3 +1156,123 @@ async def test_normal_end_without_deferred_dispatch_enqueues_immediately(monkeyp assert len(worker.enqueued) == 1 assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None worker.close_enqueued() + + +def _backpressured_wrapper(iterator, upstream_exhausted: asyncio.Event): + async def _stream(): + try: + for event in COMPLETE_STREAM_EVENTS: + yield event + finally: + upstream_exhausted.set() + + return iterator.async_sse_wrapper(_stream()) + + +async def _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted: asyncio.Event) -> list: + received = [] + while not upstream_exhausted.is_set(): + received.append(await gen.__anext__()) + for _ in range(25): + await asyncio.sleep(0) + assert len(received) <= len(COMPLETE_STREAM_EVENTS) + return received + + +@pytest.mark.asyncio +async def test_normal_end_parks_deferred_logging_even_when_sentinel_enqueue_backpressured(monkeypatch): + """ + Regression: with a full relay queue at end of stream, the pump suspends + while enqueueing the end-of-stream sentinel, and a client that then drains + the whole tail tears the relay down (setting ``client_detached``) before + the pump resumes. That teardown is a normally completed response, not a + disconnect: billing must still park for the proxy's post-response hook + (preserving post_call decoration such as guardrail_information) instead of + enqueueing immediately through the teardown path. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2) + worker = _RecordingLoggingWorker() + monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker) + + dispatched = [] + + async def _deferred_stream_complete(logging_coroutine): + dispatched.append(logging_coroutine) + logging_coroutine.close() + + iterator = _make_iterator("test_sentinel_backpressure_normal_end") + iterator.litellm_logging_obj._on_deferred_stream_complete = _deferred_stream_complete + + upstream_exhausted = asyncio.Event() + gen = _backpressured_wrapper(iterator, upstream_exhausted) + received = await _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted) + + while True: + try: + received.append(await gen.__anext__()) + except StopAsyncIteration: + break + + for _ in range(100): + if worker.enqueued or getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None): + break + await asyncio.sleep(0.01) + + assert len(received) == len(COMPLETE_STREAM_EVENTS) + assert worker.enqueued == [], "fully delivered stream billed through the teardown path" + assert dispatched == [] + parked = getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) + assert parked is not None, "pump never parked deferred billing" + parked[0].close() + + +@pytest.mark.asyncio +async def test_relay_teardown_dispatches_deferred_billing_when_sentinel_never_consumed(monkeypatch): + """ + Regression: when the pump has parked deferred billing but its end-of-stream + sentinel never fits in the full relay queue (the client disconnects without + draining the tail), the proxy's post-response hook never fires. Exactly one + of the relay teardown or the pump's fallback must dispatch the parked + billing, or the request logs no spend at all. + """ + monkeypatch.setattr(streaming_iterator_module, "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 2) + worker = _RecordingLoggingWorker() + monkeypatch.setattr(streaming_iterator_module, "GLOBAL_LOGGING_WORKER", worker) + + dispatched = [] + deferred_fired = asyncio.Event() + + def _deferred_stream_complete(logging_coroutine): + dispatched.append(logging_coroutine) + + async def _consume(): + logging_coroutine.close() + deferred_fired.set() + + return _consume() + + iterator = _make_iterator("test_sentinel_never_consumed_dispatch") + iterator.litellm_logging_obj._on_deferred_stream_complete = _deferred_stream_complete + + upstream_exhausted = asyncio.Event() + gen = _backpressured_wrapper(iterator, upstream_exhausted) + await _drain_with_pauses_until_upstream_exhausted(gen, upstream_exhausted) + + for _ in range(100): + if getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is not None: + break + await asyncio.sleep(0.01) + + await gen.aclose() + + for _ in range(100): + if dispatched: + break + await asyncio.sleep(0.01) + + assert len(dispatched) == 1, "parked billing was never dispatched" + assert getattr(iterator.litellm_logging_obj, "_deferred_stream_complete_args", None) is None + assert getattr(iterator.litellm_logging_obj, "_on_deferred_stream_complete", None) is None + assert len(worker.enqueued) == 1, "teardown billing enqueued alongside the deferred dispatch" + await worker.enqueued[0] + assert deferred_fired.is_set() diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 226bba6826a..63f895e1819 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -957,6 +957,28 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert fields["tools"][0]["type"] == "computer_20250124" +def test_config_blocks_do_not_leak_into_inference_config(): + """Regression: inferenceConfig was built before the config blocks were popped, so a dead + nested copy of each block (guardrailConfig, performanceConfig, serviceTier) rode inside + inferenceConfig alongside the real top-level one.""" + data = AmazonConverseConfig()._transform_request_helper( + model="anthropic.claude-haiku-4-5-20251001-v1:0", + system_content_blocks=[], + optional_params={ + "maxTokens": 100, + "guardrailConfig": {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"}, + "performanceConfig": {"latency": "optimized"}, + "serviceTier": {"type": "priority"}, + }, + messages=[{"role": "user", "content": "hi"}], + ) + + assert data["inferenceConfig"] == {"maxTokens": 100} + assert data["guardrailConfig"] == {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"} + assert data["performanceConfig"] == {"latency": "optimized"} + assert data["serviceTier"] == {"type": "priority"} + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost @@ -2853,17 +2875,11 @@ def test_guarded_text_guardrail_config_preserved(): headers={}, ) - # GuardrailConfig should be present at top level assert "guardrailConfig" in result assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123" - # GuardrailConfig should also be in inferenceConfig assert "inferenceConfig" in result - assert "guardrailConfig" in result["inferenceConfig"] - assert ( - result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"] - == "gr-abc123" - ) + assert "guardrailConfig" not in result["inferenceConfig"] def test_auto_convert_last_user_message_to_guarded_text(): diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 1e09afd6919..8d07d38b1b6 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -3066,17 +3066,21 @@ def test_bedrock_invoke_messages_allows_converted_websearch_function_tool(): async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect(): """ Regression test for LIT-5839: closing the outer bedrock_sse_wrapper - mid-stream (what the proxy does on a client disconnect) must close the - inner async_sse_wrapper deterministically so the partial-stream logging - fires. `completion_start_time` is only stamped on the logging object by - that dispatch, so it observing a value proves the whole chain ran. + mid-stream (what the proxy does on a client disconnect) must not lose the + stream's spend logging. Since the detached-pump relay, the upstream read + survives the disconnect and billing fires once the provider stream ends, + so the dispatch is awaited after releasing the upstream instead of being + observed synchronously at aclose(). `completion_start_time` is only + stamped on the logging object by that dispatch, so it observing a value + proves the whole chain ran. """ cfg = AmazonAnthropicClaudeMessagesConfig() + release_upstream = asyncio.Event() - async def _hanging_stream(): + async def _gated_stream(): yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 25, "output_tokens": 1}}} yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} - await asyncio.Event().wait() + await release_upstream.wait() logging_obj = LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", @@ -3087,11 +3091,16 @@ async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect(): litellm_call_id="test_bedrock_sse_wrapper_disconnect_logging", function_id="test_bedrock_sse_wrapper_disconnect_logging", ) - wrapped = cfg.bedrock_sse_wrapper(_hanging_stream(), litellm_logging_obj=logging_obj, request_body={}) + wrapped = cfg.bedrock_sse_wrapper(_gated_stream(), litellm_logging_obj=logging_obj, request_body={}) await wrapped.__anext__() await wrapped.__anext__() assert logging_obj.completion_start_time is None await wrapped.aclose() + release_upstream.set() + for _ in range(500): + if logging_obj.completion_start_time is not None: + break + await asyncio.sleep(0.01) assert logging_obj.completion_start_time is not None diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9efcee192b1..0ea5b7ad4a1 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pytest - +import litellm from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig @@ -104,12 +104,24 @@ class RealtimeClientWS: self.closed = True -class ImmediatelyEndingBedrockStream: - def __init__(self): +class ScriptedBedrockReceiver: + def __init__(self, payloads): + self._payloads = list(payloads) + + async def receive(self): + if not self._payloads: + return None + payload = self._payloads.pop(0) + return SimpleNamespace(value=SimpleNamespace(bytes_=payload.encode("utf-8"))) + + +class ScriptedBedrockStream: + def __init__(self, payloads): self.input_stream = FakeInputStream() + self._receiver = ScriptedBedrockReceiver(payloads) async def await_output(self): - return (None, EndedBedrockReceiver()) + return (None, self._receiver) class FakeStaticCredentialsResolver: @@ -151,7 +163,7 @@ def stub_aws_sdk_client(monkeypatch): async def invoke_model_with_bidirectional_stream(self, operation_input): captured["operation_input"] = operation_input - return ImmediatelyEndingBedrockStream() + return ScriptedBedrockStream(captured.get("scripted_payloads", [])) package = types.ModuleType("aws_sdk_bedrock_runtime") client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") @@ -271,19 +283,132 @@ class TestBedrockRealtimeHandler: assert "sessionEnd" in event_names assert stream.input_stream.closed + @pytest.mark.asyncio + async def test_forwarded_events_are_filtered_to_logged_types_for_spend_logging(self): + handler = BedrockRealtime() + stream = ScriptedBedrockStream( + [ + json.dumps({"event": {"userSpeechStart": {}}}), + json.dumps({"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}), + json.dumps({"event": {"textOutput": {"content": "Hi"}}}), + json.dumps({"event": {"contentEnd": {"stopReason": "END_TURN"}}}), + ] + ) + client_ws = RealtimeClientWS() + + logged_events = [ + event + async for event in handler._forward_bedrock_to_client( + stream, + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + FakeLogging(), + {}, + ) + ] + + assert [event["type"] for event in logged_events] == ["response.done"] + sent_types = [json.loads(message)["type"] for message in client_ws.sent_to_client] + assert "input_audio_buffer.speech_started" in sent_types + assert "response.text.delta" in sent_types + assert "response.done" in sent_types + assert client_ws.closed + + @pytest.mark.asyncio + async def test_logged_event_types_star_collects_every_forwarded_event(self, monkeypatch): + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + handler = BedrockRealtime() + stream = ScriptedBedrockStream( + [ + json.dumps({"event": {"userSpeechStart": {}}}), + json.dumps({"event": {"userSpeechEnd": {}}}), + ] + ) + client_ws = RealtimeClientWS() + + logged_events = [ + event + async for event in handler._forward_bedrock_to_client( + stream, + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + FakeLogging(), + {}, + ) + ] + + assert [event["type"] for event in logged_events] == [ + "input_audio_buffer.speech_started", + "input_audio_buffer.speech_stopped", + ] + + @pytest.mark.asyncio + async def test_trailing_usage_after_last_done_is_dispatched_for_spend(self, stub_aws_sdk_client, monkeypatch): + import litellm.llms.bedrock.realtime.handler as handler_module + + dispatched = {} + + class RecordingLogging(FakeLogging): + async def dispatch_success_handlers(self, result=None, prefer_async_handlers=False, **kwargs): + dispatched["events"] = result + + class RecordingLoggingWorker: + def ensure_initialized_and_enqueue(self, coro): + dispatched["coro"] = coro + + monkeypatch.setattr(handler_module, "GLOBAL_LOGGING_WORKER", RecordingLoggingWorker()) + stub_aws_sdk_client["scripted_payloads"] = [ + json.dumps( + { + "event": { + "usageEvent": { + "totalInputTokens": 3, + "totalOutputTokens": 6, + "totalTokens": 9, + "details": { + "total": { + "input": {"speechTokens": 3, "textTokens": 0}, + "output": {"speechTokens": 0, "textTokens": 6}, + } + }, + } + } + } + ) + ] + + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=RealtimeClientWS(), + logging_obj=RecordingLogging(), + aws_region_name="us-east-1", + aws_access_key_id="k", + aws_secret_access_key="s", + ) + await dispatched["coro"] + + assert [event["type"] for event in dispatched["events"]] == ["response.done"] + usage = dispatched["events"][0]["response"]["usage"] + assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (3, 6, 9) + assert usage["input_token_details"] == {"audio_tokens": 3, "text_tokens": 0, "cached_tokens": 0} + assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6} + @pytest.mark.asyncio async def test_bedrock_stream_end_closes_client_websocket(self): handler = BedrockRealtime() client_ws = ClosableClientWS() - await handler._forward_bedrock_to_client( + async for _ in handler._forward_bedrock_to_client( EndedBedrockStream(), client_ws, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", MagicMock(), {}, - ) + ): + pass assert client_ws.closed @@ -320,9 +445,7 @@ class TestBedrockRealtimeSessionLifecycle: [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] ) - await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging() - ) + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) acked = [json.loads(message) for message in client_ws.sent_to_client] updated = [event for event in acked if event["type"] == "session.updated"] @@ -334,9 +457,7 @@ class TestBedrockRealtimeSessionLifecycle: handler = BedrockRealtime() config = BedrockRealtimeConfig() stream = FakeBedrockStream() - client_ws = DisconnectingClientWS( - [json.dumps({"type": "session.update", "session": {"instructions": "hi"}})] - ) + client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index ae6b1febd6b..a74f03449a1 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -827,5 +827,310 @@ class TestBedrockRealtimeSessionEvents: assert event["session"]["modalities"] == ["text", "audio"] +class TestBedrockRealtimeUserEventsAndUsage: + """Regression tests for #38346: USER ASR transcripts, speech boundary events, + usage propagation, and duplicate response.created""" + + @staticmethod + def _run(config, messages): + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + state = { + "session_configuration_request": json.dumps({"configured": True}), + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + } + all_events = [] + for msg in messages: + result = config.transform_realtime_response( + json.dumps(msg), + "amazon.nova-2-sonic-v1:0", + logging_obj, + realtime_response_transform_input=dict(state), + ) + all_events.extend(result["response"]) + state.update( + { + "current_output_item_id": result["current_output_item_id"], + "current_response_id": result["current_response_id"], + "current_conversation_id": result["current_conversation_id"], + "current_delta_chunks": result["current_delta_chunks"], + "current_delta_type": result["current_delta_type"], + } + ) + return all_events + + def test_user_speech_start_and_stop_events(self): + events = self._run( + BedrockRealtimeConfig(), + [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}], + ) + assert [e["type"] for e in events] == [ + "input_audio_buffer.speech_started", + "input_audio_buffer.speech_stopped", + ] + assert all(e["event_id"] and e["item_id"] for e in events) + assert events[0]["item_id"] == events[1]["item_id"] + + def test_utterance_lifecycle_shares_one_item_id(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"userSpeechStart": {}}}, + {"event": {"userSpeechEnd": {}}}, + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "ready"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + item_ids = {e["item_id"] for e in events if "item_id" in e} + assert len(item_ids) == 1 + + def test_new_utterance_gets_new_item_id(self): + config = BedrockRealtimeConfig() + first = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}]) + second = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}]) + assert first[0]["item_id"] == first[1]["item_id"] + assert second[0]["item_id"] == second[1]["item_id"] + assert first[0]["item_id"] != second[0]["item_id"] + + def test_user_transcript_emits_input_audio_transcription_events(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "ready"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"] + completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"] + assert len(deltas) == 1 and deltas[0]["delta"] == "ready" + assert len(completed) == 1 and completed[0]["transcript"] == "ready" + assert deltas[0]["item_id"] == completed[0]["item_id"] + assert not any(e["type"] == "response.text.delta" for e in events) + + def test_speculative_user_transcript_emits_delta_only(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}), + } + } + }, + {"event": {"textOutput": {"content": "rea"}}}, + ], + ) + assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"] + + def test_user_transcript_state_resets_on_content_end(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "USER", "type": "TEXT"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi there"}}}, + ], + ) + text_deltas = [e for e in events if e["type"] == "response.text.delta"] + assert len(text_deltas) == 1 and text_deltas[0]["delta"] == "Hi there" + assert not any(e["type"].startswith("conversation.item.input_audio_transcription") for e in events) + + def test_response_created_emitted_once_per_response(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}}, + ], + ) + assert sum(1 for e in events if e["type"] == "response.created") == 1 + + def test_usage_event_propagates_to_response_done(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "usageEvent": { + "totalInputTokens": 25, + "totalOutputTokens": 40, + "totalTokens": 65, + "details": { + "total": { + "input": {"speechTokens": 20, "textTokens": 5}, + "output": {"speechTokens": 30, "textTokens": 10}, + } + }, + } + } + }, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ], + ) + done_events = [e for e in events if e["type"] == "response.done"] + assert len(done_events) == 1 + usage = done_events[0]["response"]["usage"] + assert usage["input_tokens"] == 25 + assert usage["output_tokens"] == 40 + assert usage["total_tokens"] == 65 + assert usage["input_token_details"]["audio_tokens"] == 20 + assert usage["input_token_details"]["text_tokens"] == 5 + assert usage["output_token_details"]["audio_tokens"] == 30 + assert usage["output_token_details"]["text_tokens"] == 10 + + def test_response_done_without_usage_event_reports_zero_usage(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ], + ) + done_events = [e for e in events if e["type"] == "response.done"] + assert len(done_events) == 1 + usage = done_events[0]["response"]["usage"] + assert usage["input_tokens"] == 0 + assert usage["output_tokens"] == 0 + assert usage["total_tokens"] == 0 + + @staticmethod + def _usage_event(total_input, total_output, in_speech, in_text, out_speech, out_text): + return { + "event": { + "usageEvent": { + "totalInputTokens": total_input, + "totalOutputTokens": total_output, + "totalTokens": total_input + total_output, + "details": { + "total": { + "input": {"speechTokens": in_speech, "textTokens": in_text}, + "output": {"speechTokens": out_speech, "textTokens": out_text}, + } + }, + } + } + } + + _ASSISTANT_TURN = ( + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ) + + def test_multi_turn_usage_reports_per_response_deltas_not_cumulative_totals(self): + events = self._run( + BedrockRealtimeConfig(), + [ + self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10), + *self._ASSISTANT_TURN, + self._usage_event(40, 100, in_speech=30, in_text=10, out_speech=75, out_text=25), + *self._ASSISTANT_TURN, + ], + ) + usages = [e["response"]["usage"] for e in events if e["type"] == "response.done"] + assert len(usages) == 2 + assert (usages[0]["input_tokens"], usages[0]["output_tokens"], usages[0]["total_tokens"]) == (25, 40, 65) + assert (usages[1]["input_tokens"], usages[1]["output_tokens"], usages[1]["total_tokens"]) == (15, 60, 75) + assert usages[1]["input_token_details"] == {"audio_tokens": 10, "text_tokens": 5, "cached_tokens": 0} + assert usages[1]["output_token_details"] == {"audio_tokens": 45, "text_tokens": 15} + assert sum(u["total_tokens"] for u in usages) == 140 + + def test_usage_reported_after_last_response_done_flushes_as_logged_only_done(self): + config = BedrockRealtimeConfig() + self._run( + config, + [ + self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10), + *self._ASSISTANT_TURN, + ], + ) + assert config.leftover_usage_done_events() == () + + self._run(config, [self._usage_event(25, 46, in_speech=20, in_text=5, out_speech=30, out_text=16)]) + leftover = config.leftover_usage_done_events() + assert len(leftover) == 1 + assert leftover[0]["type"] == "response.done" + usage = leftover[0]["response"]["usage"] + assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (0, 6, 6) + assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6} + assert config.leftover_usage_done_events() == () + + def test_final_transcript_fragments_emit_one_completed_with_full_transcript(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "What is the "}}}, + {"event": {"textOutput": {"content": "capital of France?"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"] + completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"] + assert [d["delta"] for d in deltas] == ["What is the ", "capital of France?"] + assert len(completed) == 1 + assert completed[0]["transcript"] == "What is the capital of France?" + assert {e["item_id"] for e in deltas + completed} == {completed[0]["item_id"]} + + def test_speculative_transcript_block_end_emits_no_completed(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}), + } + } + }, + {"event": {"textOutput": {"content": "rea"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"] + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/gigachat/__init__.py b/tests/test_litellm/llms/gigachat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_streaming.py b/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_streaming.py new file mode 100644 index 00000000000..35ca93319f5 --- /dev/null +++ b/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_streaming.py @@ -0,0 +1,87 @@ +""" +Tests for litellm.llms.gigachat.chat.streaming +""" + +from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator + + +def _parse(chunk: dict) -> dict: + iterator = GigaChatModelResponseIterator(streaming_response=None, sync_stream=True) + return dict(iterator.chunk_parser(chunk=chunk)) + + +class TestChunkParserUsage: + def test_usage_on_stop_chunk(self): + parsed = _parse( + { + "choices": [{"delta": {"content": ""}, "index": 0, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 25, "completion_tokens": 7, "total_tokens": 32}, + } + ) + + assert parsed["finish_reason"] == "stop" + assert parsed["usage"] is not None + assert parsed["usage"]["prompt_tokens"] == 25 + assert parsed["usage"]["completion_tokens"] == 7 + assert parsed["usage"]["total_tokens"] == 32 + + def test_usage_on_function_call_chunk(self): + """Regression: a final chunk ending in function_call still carries usage; it must not be dropped.""" + parsed = _parse( + { + "choices": [ + { + "delta": {"function_call": {"name": "get_weather", "arguments": {"city": "Moscow"}}}, + "index": 0, + "finish_reason": "function_call", + } + ], + "usage": {"prompt_tokens": 40, "completion_tokens": 12, "total_tokens": 52}, + } + ) + + assert parsed["finish_reason"] == "tool_calls" + assert parsed["tool_use"] is not None + assert parsed["usage"] is not None + assert parsed["usage"]["prompt_tokens"] == 40 + assert parsed["usage"]["completion_tokens"] == 12 + assert parsed["usage"]["total_tokens"] == 52 + + def test_usage_on_length_chunk(self): + parsed = _parse( + { + "choices": [{"delta": {"content": "truncated"}, "index": 0, "finish_reason": "length"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 128, "total_tokens": 138}, + } + ) + + assert parsed["usage"] is not None + assert parsed["usage"]["total_tokens"] == 138 + + def test_no_usage_on_interim_chunk(self): + parsed = _parse({"choices": [{"delta": {"content": "hello"}, "index": 0, "finish_reason": None}]}) + + assert parsed["text"] == "hello" + assert parsed["is_finished"] is False + assert parsed["usage"] is None + + def test_cache_hit_usage_folds_cached_tokens_back_in(self): + """GigaChat reports prompt_tokens and total_tokens after subtracting cached tokens + (docs example: prompt_tokens=1, precached_prompt_tokens=37, total_tokens=5), so the + OpenAI-convention usage must add them back and surface them as cached_tokens.""" + parsed = _parse( + { + "choices": [{"delta": {"content": ""}, "index": 0, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 7, + "total_tokens": 32, + "precached_prompt_tokens": 20, + }, + } + ) + + assert parsed["usage"] is not None + assert parsed["usage"]["prompt_tokens"] == 45 + assert parsed["usage"]["total_tokens"] == 52 + assert parsed["usage"]["prompt_tokens_details"]["cached_tokens"] == 20 diff --git a/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_transformation.py b/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_transformation.py new file mode 100644 index 00000000000..2f9511e642c --- /dev/null +++ b/tests/test_litellm/llms/gigachat/chat/test_gigachat_chat_transformation.py @@ -0,0 +1,883 @@ +""" +Unit tests for GigaChat chat transformation. + +Tests GigaChatConfig covering get_complete_url, validate_environment, +get_supported_openai_params, map_openai_params, _convert_tools_to_functions, +_map_tool_choice, _transform_messages, transform_request, transform_response, +get_model_response_iterator, and get_error_class. +""" + +import json +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.llms.gigachat.chat.transformation import ( + GigaChatConfig, + GigaChatError, + is_valid_json, +) +from litellm.types.utils import ModelResponse, Usage + +TRANSFORM_MODULE = "litellm.llms.gigachat.chat.transformation" + + +def _make_httpx_response( + body: dict, status_code: int = 200 +) -> httpx.Response: + return httpx.Response( + status_code=status_code, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request( + "POST", + "https://gigachat.devices.sberbank.ru/api/v1/chat/completions", + ), + ) + + +# --------------------------------------------------------------------------- +# is_valid_json +# --------------------------------------------------------------------------- + + +class TestIsValidJson: + def test_valid_json_object(self): + assert is_valid_json('{"key": "value"}') is True + + def test_valid_json_array(self): + assert is_valid_json("[1, 2, 3]") is True + + def test_valid_json_string(self): + assert is_valid_json('"hello"') is True + + def test_invalid_json(self): + assert is_valid_json("{invalid}") is False + + def test_empty_string(self): + assert is_valid_json("") is False + + +# --------------------------------------------------------------------------- +# GigaChatConfig +# --------------------------------------------------------------------------- + + +class TestGetCompleteUrl: + def setup_method(self): + self.config = GigaChatConfig() + + def test_uses_api_base_from_param(self): + url = self.config.get_complete_url( + api_base="https://custom.example.com", + api_key=None, + model="GigaChat", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url == "https://custom.example.com/chat/completions" + + def test_uses_api_base_with_trailing_slash(self): + url = self.config.get_complete_url( + api_base="https://custom.example.com/", + api_key=None, + model="GigaChat", + optional_params={}, + litellm_params={}, + stream=False, + ) + # get_api_base passes the value through without stripping the slash + assert url == "https://custom.example.com//chat/completions" + + def test_uses_api_base_from_get_api_base_when_none(self): + url = self.config.get_complete_url( + api_base=None, + api_key=None, + model="GigaChat", + optional_params={}, + litellm_params={}, + stream=False, + ) + assert url.endswith("/chat/completions") + + +class TestValidateEnvironment: + def setup_method(self): + self.config = GigaChatConfig() + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="test-token") + @patch(f"{TRANSFORM_MODULE}.get_secret_str", return_value=None) + def test_sets_auth_headers(self, mock_get_secret, mock_get_token): + headers: dict = {} + result = self.config.validate_environment( + headers=headers, + model="GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key="creds", + api_base="https://api.example.com", + ) + assert result["Authorization"] == "Bearer test-token" + assert result["Content-Type"] == "application/json" + assert result["Accept"] == "application/json" + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token") + @patch(f"{TRANSFORM_MODULE}.get_secret_str", return_value=None) + def test_stores_credentials_and_api_base_for_image_uploads( + self, mock_get_secret, mock_get_token + ): + self.config.validate_environment( + headers={}, + model="GigaChat", + messages=[], + optional_params={}, + litellm_params={}, + api_key="my-creds", + api_base="https://my-api.example.com", + ) + assert self.config._current_credentials == "my-creds" + assert self.config._current_api_base == "https://my-api.example.com" + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token") + @patch(f"{TRANSFORM_MODULE}.get_secret_str") + def test_falls_back_to_env_for_credentials( # test-quality-ok: mock-echo of internal wiring + self, mock_get_secret, mock_get_token + ): + mock_get_secret.return_value = "env-creds" + self.config.validate_environment( + headers={}, + model="GigaChat", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + mock_get_secret.assert_any_call("GIGACHAT_CREDENTIALS") # test-quality-ok: mock-echo of internal wiring + + +class TestGetSupportedOpenAiParams: + def setup_method(self): + self.config = GigaChatConfig() + + def test_returns_expected_params(self): + params = self.config.get_supported_openai_params("GigaChat") + expected = [ + "stream", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "stop", + "tools", + "tool_choice", + "functions", + "function_call", + "response_format", + ] + assert params == expected + + +class TestMapOpenAiParams: + def setup_method(self): + self.config = GigaChatConfig() + + def test_stream(self): + result = self.config.map_openai_params( + non_default_params={"stream": True}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["stream"] is True + + def test_temperature_zero_maps_to_top_p_zero(self): + result = self.config.map_openai_params( + non_default_params={"temperature": 0}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["top_p"] == 0 + assert "temperature" not in result + + def test_temperature_non_zero(self): + result = self.config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["temperature"] == 0.7 + + def test_top_p(self): + result = self.config.map_openai_params( + non_default_params={"top_p": 0.5}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["top_p"] == 0.5 + + def test_max_tokens(self): + result = self.config.map_openai_params( + non_default_params={"max_tokens": 100}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["max_tokens"] == 100 + + def test_max_completion_tokens(self): + result = self.config.map_openai_params( + non_default_params={"max_completion_tokens": 200}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["max_tokens"] == 200 + + def test_stop_is_dropped(self): + result = self.config.map_openai_params( + non_default_params={"stop": ["\n\n"]}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert "stop" not in result + + def test_tools_converted_to_functions(self): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": {"type": "object"}, + }, + } + ] + result = self.config.map_openai_params( + non_default_params={"tools": tools}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert "functions" in result + assert result["functions"] == [ + {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object"}} + ] + + def test_tool_choice_auto(self): + result = self.config.map_openai_params( + non_default_params={"tool_choice": "auto"}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result.get("function_call") == "auto" + + def test_tool_choice_none(self): + result = self.config.map_openai_params( + non_default_params={"tool_choice": "none"}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result.get("function_call") == "none" + + def test_tool_choice_required(self): + result = self.config.map_openai_params( + non_default_params={"tool_choice": "required"}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result.get("function_call") == "auto" + + def test_tool_choice_dict(self): + result = self.config.map_openai_params( + non_default_params={ + "tool_choice": { + "type": "function", + "function": {"name": "get_weather"}, + } + }, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result.get("function_call") == {"name": "get_weather"} + + def test_functions(self): + funcs = [{"name": "my_func", "description": "desc", "parameters": {}}] + result = self.config.map_openai_params( + non_default_params={"functions": funcs}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["functions"] == funcs + + def test_function_call(self): + result = self.config.map_openai_params( + non_default_params={"function_call": {"name": "my_func"}}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result["function_call"] == {"name": "my_func"} + + def test_response_format_json_schema(self): + response_format = { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + } + result = self.config.map_openai_params( + non_default_params={"response_format": response_format}, + optional_params={"functions": []}, + model="GigaChat", + drop_params=False, + ) + # Should add a function for the schema + assert len(result["functions"]) == 1 + assert result["functions"][0]["name"] == "test_schema" + assert result["function_call"] == {"name": "test_schema"} + assert result["_structured_output"] is True + + +class TestConvertToolsToFunctions: + def setup_method(self): + self.config = GigaChatConfig() + + def test_converts_function_tools_only(self): + tools = [ + {"type": "function", "function": {"name": "a", "description": "d", "parameters": {}}}, + {"type": "code_interpreter"}, # should be ignored + ] + result = self.config._convert_tools_to_functions(tools) + assert len(result) == 1 + assert result[0]["name"] == "a" + + def test_empty_tools(self): + assert self.config._convert_tools_to_functions([]) == [] + + +class TestMapToolChoice: + def setup_method(self): + self.config = GigaChatConfig() + + def test_none(self): + assert self.config._map_tool_choice("none") == "none" + + def test_auto(self): + assert self.config._map_tool_choice("auto") == "auto" + + def test_required(self): + assert self.config._map_tool_choice("required") == "auto" + + def test_dict_with_function(self): + result = self.config._map_tool_choice( + {"type": "function", "function": {"name": "get_weather"}} + ) + assert result == {"name": "get_weather"} + + def test_dict_without_name(self): + result = self.config._map_tool_choice( + {"type": "function", "function": {}} + ) + assert result is None + + def test_unknown_value(self): + assert self.config._map_tool_choice("unknown") is None + + +class TestTransformMessages: + def setup_method(self): + self.config = GigaChatConfig() + + def test_developer_role_to_system(self): + result = self.config._transform_messages( + [{"role": "developer", "content": "be helpful"}] + ) + assert result[0]["role"] == "system" + assert result[0]["content"] == "be helpful" + + def test_system_message_not_first_becomes_user(self): + result = self.config._transform_messages([ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "instruction"}, + ]) + assert result[0]["role"] == "user" + assert result[1]["role"] == "user" + assert result[1]["content"] == "instruction" + + def test_tool_role_to_function(self): + result = self.config._transform_messages([ + {"role": "tool", "content": '{"result": "ok"}'} + ]) + assert result[0]["role"] == "function" + + def test_tool_role_content_wraps_non_json(self): + result = self.config._transform_messages([ + {"role": "tool", "content": "plain text"} + ]) + assert result[0]["role"] == "function" + assert is_valid_json(result[0]["content"]) + + def test_none_content_becomes_empty_string(self): + result = self.config._transform_messages([ + {"role": "user", "content": None} + ]) + assert result[0]["content"] == "" + + def test_name_field_removed(self): + result = self.config._transform_messages([ + {"role": "user", "content": "hi", "name": "John"} + ]) + assert "name" not in result[0] + + def test_tool_calls_converted_to_function_call(self): + result = self.config._transform_messages([ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "London"}', + }, + } + ], + } + ]) + assert "tool_calls" not in result[0] + assert result[0]["function_call"]["name"] == "get_weather" + assert result[0]["function_call"]["arguments"] == {"city": "London"} + + def test_tool_calls_with_dict_arguments(self): + result = self.config._transform_messages([ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_xyz", + "type": "function", + "function": { + "name": "search", + "arguments": {"query": "test"}, + }, + } + ], + } + ]) + assert result[0]["function_call"]["arguments"] == {"query": "test"} + + def test_list_content_multimodal(self): + content = [ + {"type": "text", "text": "describe this"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.jpg"}, + }, + ] + with patch.object(self.config, "_upload_image", return_value="file-123"): + result = self.config._transform_messages([ + {"role": "user", "content": content} + ]) + assert result[0]["content"] == "describe this" + assert result[0]["attachments"] == ["file-123"] + + def test_list_content_with_image_url_string(self): + content = [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": "https://example.com/img.jpg"}, + ] + with patch.object(self.config, "_upload_image", return_value="file-456"): + result = self.config._transform_messages([ + {"role": "user", "content": content} + ]) + assert result[0]["content"] == "look" + assert "file-456" in result[0]["attachments"] + + +class TestTransformRequest: + def setup_method(self): + self.config = GigaChatConfig() + + def test_builds_basic_request(self): + body = self.config.transform_request( + model="gigachat/GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["model"] == "GigaChat" + assert len(body["messages"]) == 1 + assert body["messages"][0]["content"] == "hi" + + def test_model_prefix_stripped(self): + body = self.config.transform_request( + model="gigachat/GigaChat-Pro", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["model"] == "GigaChat-Pro" + + def test_includes_optional_params(self): + body = self.config.transform_request( + model="gigachat/GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "temperature": 0.5, + "max_tokens": 100, + "stream": True, + }, + litellm_params={}, + headers={}, + ) + assert body["temperature"] == 0.5 + assert body["max_tokens"] == 100 + assert body["stream"] is True + + def test_includes_functions(self): + body = self.config.transform_request( + model="gigachat/GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={ + "functions": [{"name": "my_func"}], + "function_call": {"name": "my_func"}, + }, + litellm_params={}, + headers={}, + ) + assert body["functions"] == [{"name": "my_func"}] + assert body["function_call"] == {"name": "my_func"} + + def test_skips_unsupported_params(self): + body = self.config.transform_request( + model="gigachat/GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={"n": 2, "user": "abc"}, + litellm_params={}, + headers={}, + ) + assert "n" not in body + assert "user" not in body + + +class TestTransformResponse: + def setup_method(self): + self.config = GigaChatConfig() + + def test_basic_response(self): + raw = _make_httpx_response({ + "id": "chatcmpl-123", + "created": 1700000000, + "model": "GigaChat", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.choices[0].message.content == "Hello!" + assert result.choices[0].finish_reason == "stop" + assert result.usage.prompt_tokens == 5 + assert result.usage.total_tokens == 8 + + def test_function_call_into_tool_calls(self): + raw = _make_httpx_response({ + "id": "chatcmpl-456", + "created": 1700000000, + "model": "GigaChat", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "function_call": { + "name": "get_weather", + "arguments": {"city": "Moscow"}, + }, + }, + "finish_reason": "function_call", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.choices[0].finish_reason == "tool_calls" + tool_calls = result.choices[0].message.tool_calls + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].function.name == "get_weather" + assert '{"city": "Moscow"}' in tool_calls[0].function.arguments + + def test_function_call_structured_output(self): + raw = _make_httpx_response({ + "id": "chatcmpl-789", + "created": 1700000000, + "model": "GigaChat", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "function_call": { + "name": "test_schema", + "arguments": {"name": "John"}, + }, + }, + "finish_reason": "function_call", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={"_structured_output": True}, + litellm_params={}, + encoding=None, + ) + # Structured output: function_call -> content + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].message.content is not None + assert '"name": "John"' in result.choices[0].message.content + + def test_function_call_string_arguments(self): + raw = _make_httpx_response({ + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "function_call": { + "name": "get_weather", + "arguments": '{"city": "Moscow"}', + }, + }, + "finish_reason": "function_call", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + tc = result.choices[0].message.tool_calls[0] + assert '{"city": "Moscow"}' in tc.function.arguments + + def test_cleans_up_gigachat_specific_fields(self): + raw = _make_httpx_response({ + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "done", + "functions_state_id": "some-state", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + # functions_state_id should have been removed from the message data + assert result.choices[0].message.content == "done" + + def test_raises_on_invalid_json(self): + raw = httpx.Response( + status_code=500, + headers={"content-type": "text/plain"}, + content=b"not json", + request=httpx.Request("POST", "https://example.com"), + ) + model_response = ModelResponse() + with pytest.raises(GigaChatError) as exc_info: + self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert "Invalid JSON response" in str(exc_info.value.message) + + def test_empty_choices(self): + raw = _make_httpx_response({ + "choices": [], + "usage": {}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert result.choices == [] + + def test_function_call_with_non_dict_arguments(self): + raw = _make_httpx_response({ + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "function_call": { + "name": "say_hello", + "arguments": "hello", + }, + }, + "finish_reason": "function_call", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }) + model_response = ModelResponse() + result = self.config.transform_response( + model="gigachat/GigaChat", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + tc = result.choices[0].message.tool_calls[0] + assert tc.function.arguments == "hello" + + +class TestGetModelResponseIterator: + def setup_method(self): + self.config = GigaChatConfig() + + def test_returns_gigachat_iterator_sync(self): + from litellm.llms.gigachat.chat.streaming import ( + GigaChatModelResponseIterator, + ) + + result = self.config.get_model_response_iterator( + streaming_response=iter(["data"]), + sync_stream=True, + json_mode=False, + ) + assert isinstance(result, GigaChatModelResponseIterator) + + +class TestGetErrorClass: + def setup_method(self): + self.config = GigaChatConfig() + + def test_returns_gigachat_error(self): + error = self.config.get_error_class( + error_message="something went wrong", + status_code=400, + headers={"x-request-id": "abc"}, + ) + assert isinstance(error, GigaChatError) + assert error.status_code == 400 + assert error.message == "something went wrong" + assert error.headers == {"x-request-id": "abc"} + + +class TestUploadImage: + def setup_method(self): + self.config = GigaChatConfig() + + @patch(f"{TRANSFORM_MODULE}.upload_file_sync", return_value="file-uploaded") + def test_upload_image_success(self, mock_upload): + self.config._current_credentials = "creds" + self.config._current_api_base = "https://api.example.com" + result = self.config._upload_image("https://example.com/img.jpg") + assert result == "file-uploaded" + mock_upload.assert_called_once_with( + image_url="https://example.com/img.jpg", + credentials="creds", + api_base="https://api.example.com", + ) + + @patch(f"{TRANSFORM_MODULE}.upload_file_sync", side_effect=Exception("fail")) + def test_upload_image_failure_returns_none(self, mock_upload): + result = self.config._upload_image("https://example.com/img.jpg") + assert result is None \ No newline at end of file diff --git a/tests/test_litellm/llms/gigachat/embedding/__init__.py b/tests/test_litellm/llms/gigachat/embedding/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/gigachat/embedding/test_gigachat_embedding_transformation.py b/tests/test_litellm/llms/gigachat/embedding/test_gigachat_embedding_transformation.py new file mode 100644 index 00000000000..8537793ea72 --- /dev/null +++ b/tests/test_litellm/llms/gigachat/embedding/test_gigachat_embedding_transformation.py @@ -0,0 +1,372 @@ +""" +Unit tests for GigaChat embedding transformation. + +Tests GigaChatEmbeddingConfig covering get_config, get_supported_openai_params, +map_openai_params, _get_openai_compatible_provider_info, get_complete_url, +transform_embedding_request, transform_embedding_response, validate_environment, +and get_error_class. +""" + +import json +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm import LlmProviders +from litellm.llms.gigachat.embedding.transformation import ( + GigaChatEmbeddingConfig, + GigaChatEmbeddingError, +) +from litellm.types.utils import EmbeddingResponse + +TRANSFORM_MODULE = "litellm.llms.gigachat.embedding.transformation" + + +def _make_httpx_response(body: dict, status_code: int = 200) -> httpx.Response: + return httpx.Response( + status_code=status_code, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request("POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"), + ) + + +# --------------------------------------------------------------------------- +# GigaChatEmbeddingConfig +# --------------------------------------------------------------------------- + + +class TestGetConfig: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_contains_only_abc_impl(self): + """get_config returns ABC internal data due to inheritance.""" + result = self.config.get_config() + # The only key should be _abc_impl from ABC base class + assert set(result.keys()) == {"_abc_impl"} + + +class TestGetSupportedOpenAiParams: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_returns_empty_list(self): + params = self.config.get_supported_openai_params("GigaChat") + assert params == [] + + +class TestMapOpenAiParams: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_returns_optional_params_unchanged(self): + result = self.config.map_openai_params( + non_default_params={"model": "test"}, + optional_params={"temperature": 0.5}, + model="GigaChat", + drop_params=False, + ) + assert result == {"temperature": 0.5} + + def test_returns_empty_dict_when_no_optional_params(self): + result = self.config.map_openai_params( + non_default_params={}, + optional_params={}, + model="GigaChat", + drop_params=False, + ) + assert result == {} + + +class TestGetOpenaiCompatibleProviderInfo: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_returns_gigachat_provider(self): + provider, api_base, api_key = self.config._get_openai_compatible_provider_info( + api_base="https://api.example.com", api_key="test-key" + ) + assert provider == LlmProviders.GIGACHAT.value + assert api_base == "https://api.example.com" + assert api_key == "test-key" + + def test_resolves_api_base_when_none(self, monkeypatch): + monkeypatch.delenv("GIGACHAT_API_BASE", raising=False) + provider, api_base, api_key = self.config._get_openai_compatible_provider_info( + api_base=None, api_key="key" + ) + assert api_base is not None + assert api_base.endswith("/api/v1") + + def test_returns_none_api_key(self): + _, _, api_key = self.config._get_openai_compatible_provider_info( + api_base="https://example.com", api_key=None + ) + assert api_key is None + + +class TestGetCompleteUrl: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_default_url(self): + url = self.config.get_complete_url( + api_base=None, api_key=None, model="GigaChat", + optional_params={}, litellm_params={}, + ) + assert url.endswith("/embeddings") + + def test_custom_api_base(self): + url = self.config.get_complete_url( + api_base="https://custom.example.com", api_key=None, model="GigaChat", + optional_params={}, litellm_params={}, + ) + assert url == "https://custom.example.com/embeddings" + + def test_trailing_slash_api_base(self): + url = self.config.get_complete_url( + api_base="https://custom.example.com/", api_key=None, model="GigaChat", + optional_params={}, litellm_params={}, + ) + # get_api_base doesn't strip slash, so we get double slash + assert url == "https://custom.example.com//embeddings" + + +class TestTransformEmbeddingRequest: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_string_input(self): + result = self.config.transform_embedding_request( + model="gigachat/Embeddings", + input="hello world", + optional_params={}, + headers={}, + ) + assert result == {"model": "Embeddings", "input": ["hello world"]} + + def test_list_input(self): + result = self.config.transform_embedding_request( + model="gigachat/Embeddings", + input=["text1", "text2"], + optional_params={}, + headers={}, + ) + assert result == {"model": "Embeddings", "input": ["text1", "text2"]} + + def test_strips_gigachat_prefix(self): + result = self.config.transform_embedding_request( + model="gigachat/GigaChat-Pro", + input="test", + optional_params={}, + headers={}, + ) + assert result["model"] == "GigaChat-Pro" + + def test_model_without_prefix(self): + result = self.config.transform_embedding_request( + model="Embeddings", + input="test", + optional_params={}, + headers={}, + ) + assert result["model"] == "Embeddings" + + +class TestTransformEmbeddingResponse: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + self.logging_obj = MagicMock() + + def _make_gigachat_response(self, data: list[dict]) -> httpx.Response: + return _make_httpx_response({ + "object": "list", + "data": data, + "model": "Embeddings", + }) + + def test_basic_response(self): + raw = self._make_gigachat_response([ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 0, + } + ]) + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="gigachat/Embeddings", + raw_response=raw, + model_response=model_response, + logging_obj=self.logging_obj, + api_key="test-key", + request_data={"input": ["text"]}, + optional_params={}, + litellm_params={}, + ) + assert result.object == "list" + assert len(result.data) == 1 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[0]["index"] == 0 + assert result.usage.prompt_tokens == 0 + assert result.usage.total_tokens == 0 + + def test_aggregates_per_embedding_usage(self): + raw = self._make_gigachat_response([ + { + "object": "embedding", + "embedding": [0.1, 0.2], + "index": 0, + "usage": {"prompt_tokens": 5}, + }, + { + "object": "embedding", + "embedding": [0.3, 0.4], + "index": 1, + "usage": {"prompt_tokens": 7}, + }, + ]) + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="gigachat/Embeddings", + raw_response=raw, + model_response=model_response, + logging_obj=self.logging_obj, + api_key="test-key", + request_data={"input": ["a", "b"]}, + optional_params={}, + litellm_params={}, + ) + # Total should be sum of per-embedding prompt_tokens + assert result.usage.prompt_tokens == 12 + assert result.usage.total_tokens == 12 + # Usage should be removed from individual embedding data + assert "usage" not in result.data[0] + assert "usage" not in result.data[1] + + def test_usage_removed_from_individual_embeddings(self): + raw = self._make_gigachat_response([ + { + "object": "embedding", + "embedding": [0.5], + "index": 0, + "usage": {"prompt_tokens": 3}, + } + ]) + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="gigachat/Embeddings", + raw_response=raw, + model_response=model_response, + logging_obj=self.logging_obj, + api_key="key", + request_data={"input": ["x"]}, + optional_params={}, + litellm_params={}, + ) + # usage should NOT be in the final EmbeddingResponse data items + for emb in result.data: + assert "usage" not in emb + + def test_passes_model_from_response(self): + raw = self._make_gigachat_response([ + {"object": "embedding", "embedding": [0.1], "index": 0}, + ]) + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="gigachat/Embeddings", + raw_response=raw, + model_response=model_response, + logging_obj=self.logging_obj, + api_key="key", + request_data={"input": ["x"]}, + optional_params={}, + litellm_params={}, + ) + assert result.model == "Embeddings" + + def test_calls_logging_post_call(self): + raw = self._make_gigachat_response([ + {"object": "embedding", "embedding": [0.1], "index": 0}, + ]) + model_response = EmbeddingResponse() + self.config.transform_embedding_response( + model="gigachat/Embeddings", + raw_response=raw, + model_response=model_response, + logging_obj=self.logging_obj, + api_key="test-api-key", + request_data={"input": ["hello"]}, + optional_params={}, + litellm_params={}, + ) + self.logging_obj.post_call.assert_called_once() + args = self.logging_obj.post_call.call_args.kwargs + assert args["api_key"] == "test-api-key" + assert args["input"] == ["hello"] + + +class TestValidateEnvironment: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="test-token") + def test_sets_oauth_headers(self, mock_get_token): + headers = self.config.validate_environment( + headers={}, + model="GigaChat", + messages=[], + optional_params={}, + litellm_params={}, + api_key="creds", + api_base="https://api.example.com", + ) + assert headers["Authorization"] == "Bearer test-token" + assert headers["Content-Type"] == "application/json" + mock_get_token.assert_called_once_with(credentials="creds", litellm_params={}) + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token") + def test_merges_custom_headers(self, mock_get_token): + headers = self.config.validate_environment( + headers={"X-Custom": "value"}, + model="GigaChat", + messages=[], + optional_params={}, + litellm_params={}, + api_key="creds", + api_base="https://api.example.com", + ) + assert headers["Authorization"] == "Bearer token" + assert headers["Content-Type"] == "application/json" + assert headers["X-Custom"] == "value" + + @patch(f"{TRANSFORM_MODULE}.get_access_token", return_value="token") + def test_custom_header_overwrites_default(self, mock_get_token): + headers = self.config.validate_environment( + headers={"Authorization": "Bearer custom"}, + model="GigaChat", + messages=[], + optional_params={}, + litellm_params={}, + api_key="creds", + api_base="https://api.example.com", + ) + # Merge: default headers first, then custom headers on top + assert headers["Authorization"] == "Bearer custom" + + +class TestGetErrorClass: + def setup_method(self): + self.config = GigaChatEmbeddingConfig() + + def test_returns_gigachat_embedding_error(self): + error = self.config.get_error_class( + error_message="embedding failed", + status_code=400, + headers={"x-request-id": "abc"}, + ) + assert isinstance(error, GigaChatEmbeddingError) + assert error.status_code == 400 + assert error.message == "embedding failed" \ No newline at end of file diff --git a/tests/test_litellm/llms/gigachat/passthrough/__init__.py b/tests/test_litellm/llms/gigachat/passthrough/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/gigachat/passthrough/test_gigachat_passthrough_transformation.py b/tests/test_litellm/llms/gigachat/passthrough/test_gigachat_passthrough_transformation.py new file mode 100644 index 00000000000..0a6ef364954 --- /dev/null +++ b/tests/test_litellm/llms/gigachat/passthrough/test_gigachat_passthrough_transformation.py @@ -0,0 +1,607 @@ +""" +Unit tests for GigaChatPassthroughConfig transformation. + +Tests the GigaChat-specific passthrough configuration including URL construction, +streaming detection, authentication handling, and logging response transformations. +""" + +import json +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.llms.gigachat.passthrough.transformation import GigaChatPassthroughConfig +from litellm.types.utils import EmbeddingResponse, ModelResponse + + +def _gigachat_chat_completion_body(): + return { + "id": "chatcmpl-test123", + "object": "chat.completion", + "created": 1700000000, + "model": "GigaChat", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello from GigaChat", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 3, + "total_tokens": 8, + }, + } + + +def _gigachat_embedding_body(): + return { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 0, + "usage": {"prompt_tokens": 4}, + } + ], + "model": "Embeddings", + } + + +def _make_httpx_response(body: dict) -> httpx.Response: + return httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request( + "POST", "https://gigachat.devices.sberbank.ru/api/v1/chat/completions" + ), + ) + + +class TestGigaChatPassthroughConfig: + """Tests for GigaChatPassthroughConfig class.""" + + def test_is_streaming_request_true(self): + """Test streaming is detected when stream=True.""" + config = GigaChatPassthroughConfig() + assert ( + config.is_streaming_request("chat/completions", {"stream": True}) is True + ) + + def test_is_streaming_request_false(self): + """Test streaming is not detected when stream=False.""" + config = GigaChatPassthroughConfig() + assert ( + config.is_streaming_request("chat/completions", {"stream": False}) + is False + ) + + def test_is_streaming_request_missing_stream_key(self): + """Test streaming defaults to False when stream key is missing.""" + config = GigaChatPassthroughConfig() + assert ( + config.is_streaming_request("chat/completions", {"model": "GigaChat"}) + is False + ) + + def test_get_complete_url_with_api_base(self): + """Test URL construction with explicit api_base.""" + config = GigaChatPassthroughConfig() + api_base = "https://custom.gigachat.ru/api/v1" + endpoint = "chat/completions" + + complete_url, base_target_url = config.get_complete_url( + api_base=api_base, + api_key=None, + model="GigaChat", + endpoint=endpoint, + request_query_params=None, + litellm_params={}, + ) + + assert isinstance(complete_url, httpx.URL) + assert str(complete_url) == f"{api_base}/{endpoint}" + assert base_target_url == api_base + + def test_get_complete_url_with_leading_slash_endpoint(self): + """Test URL construction with endpoint having leading slash.""" + config = GigaChatPassthroughConfig() + api_base = "https://custom.gigachat.ru/api/v1" + endpoint = "/chat/completions" + + complete_url, base_target_url = config.get_complete_url( + api_base=api_base, + api_key=None, + model="GigaChat", + endpoint=endpoint, + request_query_params=None, + litellm_params={}, + ) + + assert str(complete_url) == "https://custom.gigachat.ru/api/v1/chat/completions" + assert base_target_url == api_base + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_complete_url_with_env_api_base(self, mock_get_secret): + """Test URL construction with api_base from environment.""" + config = GigaChatPassthroughConfig() + env_api_base = "https://env.gigachat.ru/api/v1" + mock_get_secret.return_value = env_api_base + + complete_url, base_target_url = config.get_complete_url( + api_base=None, + api_key=None, + model="GigaChat", + endpoint="embeddings", + request_query_params=None, + litellm_params={}, + ) + + assert isinstance(complete_url, httpx.URL) + assert str(complete_url).startswith(env_api_base) + assert base_target_url == env_api_base + mock_get_secret.assert_called_once_with("GIGACHAT_API_BASE") + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_complete_url_fallback_to_default(self, mock_get_secret): + """Test URL construction falls back to default GIGACHAT_BASE_URL.""" + config = GigaChatPassthroughConfig() + mock_get_secret.return_value = None + + complete_url, base_target_url = config.get_complete_url( + api_base=None, + api_key=None, + model="GigaChat", + endpoint="models", + request_query_params=None, + litellm_params={}, + ) + + assert isinstance(complete_url, httpx.URL) + assert "gigachat.devices.sberbank.ru" in str(complete_url) + assert base_target_url == "https://gigachat.devices.sberbank.ru/api/v1" + + def test_get_complete_url_no_api_base_raises(self): + """Test that exception is raised when no api_base can be resolved.""" + config = GigaChatPassthroughConfig() + with patch( + "litellm.llms.gigachat.passthrough.transformation.get_secret_str", # test-quality-ok: patching litellm internal for unit test isolation + return_value=None, + ): + with patch( + "litellm.llms.gigachat.passthrough.transformation.GIGACHAT_BASE_URL", # test-quality-ok: patching litellm internal for unit test isolation + None, + ): + with pytest.raises(Exception, match="GigaChat api base not found"): + config.get_complete_url( + api_base=None, + api_key=None, + model="GigaChat", + endpoint="chat/completions", + request_query_params=None, + litellm_params={}, + ) + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_access_token" + ) + def test_validate_environment(self, mock_get_access_token): + """Test headers are set correctly with OAuth token.""" + config = GigaChatPassthroughConfig() + mock_get_access_token.return_value = "test-token-123" + + headers = config.validate_environment( + headers={}, + model="GigaChat", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key="test-credentials", + api_base="https://custom.gigachat.ru", + ) + + assert headers["Authorization"] == "Bearer test-token-123" + assert headers["Content-Type"] == "application/json" + assert headers["Accept"] == "application/json" + mock_get_access_token.assert_called_once_with( + credentials="test-credentials", + litellm_params={}, + ) + + def test_logging_non_streaming_response_chat_completions(self): + """Test chat completions endpoint returns ModelResponse.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + result = config.logging_non_streaming_response( + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + httpx_response=_make_httpx_response(_gigachat_chat_completion_body()), + request_data={ + "model": "gigachat/GigaChat", + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=logging_obj, + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hello from GigaChat" + assert result.usage.prompt_tokens == 5 + assert result.usage.completion_tokens == 3 + assert result.usage.total_tokens == 8 + + def test_logging_non_streaming_response_embeddings(self): + """Test embeddings endpoint returns EmbeddingResponse.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + result = config.logging_non_streaming_response( + model="gigachat/Embeddings", + custom_llm_provider="gigachat", + httpx_response=_make_httpx_response(_gigachat_embedding_body()), + request_data={"input": ["hello"], "model": "gigachat/Embeddings"}, + logging_obj=logging_obj, + endpoint="embeddings", + ) + + assert isinstance(result, EmbeddingResponse) + assert len(result.data) == 1 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + + def test_logging_non_streaming_response_unknown_endpoint_returns_none(self): + """Test unknown endpoint returns None.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + result = config.logging_non_streaming_response( + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + httpx_response=_make_httpx_response(_gigachat_chat_completion_body()), + request_data={}, + logging_obj=logging_obj, + endpoint="images/generations", + ) + + assert result is None + + def test_handle_logging_collected_chunks_with_string_chunks(self): + """Test converting string chunks to model response.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + '{"choices": [{"delta": {"content": "Hello"}, "index": 0}]}', + '{"choices": [{"delta": {"content": " world"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}}', + ] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hello world" + + def test_handle_logging_collected_chunks_with_bytes_chunks(self): + """Test converting string chunks to model response (bytes pre-decoded upstream).""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + '{"choices": [{"delta": {"content": "Hi"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}', + ] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hi" + + def test_handle_logging_collected_chunks_with_done_and_empty(self): + """Test that [DONE] and empty chunks are skipped.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + "", + "[DONE]", + '{"choices": [{"delta": {"content": "test"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}', + ] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "test" + + def test_handle_logging_collected_chunks_with_dict_chunks(self): + """Test converting string-serialized dict chunks (dicts pre-serialized upstream).""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + '{"choices": [{"delta": {"content": "direct"}, "index": 0}]}', + json.dumps( + { + "choices": [ + { + "delta": {}, + "finish_reason": "stop", + "index": 0, + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + } + ), + ] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "direct" + + def test_handle_logging_collected_chunks_empty_list_returns_none(self): + """Test empty chunks list returns None.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert result is None + + def test_handle_logging_collected_chunks_invalid_json_skipped(self): + """Test invalid JSON chunks are skipped gracefully.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + "not-valid-json", + '{"choices": [{"delta": {"content": "valid"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}', + ] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "valid" + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_api_base_with_explicit_value(self, mock_get_secret): + """Test get_api_base returns explicit value when provided.""" + explicit_base = "https://custom.gigachat.ru/api/v1" + result = GigaChatPassthroughConfig.get_api_base(api_base=explicit_base) + assert result == explicit_base + mock_get_secret.assert_not_called() + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_api_base_from_environment(self, mock_get_secret): + """Test get_api_base retrieves from environment when not provided.""" + env_base = "https://env.gigachat.ru/api/v1" + mock_get_secret.return_value = env_base + result = GigaChatPassthroughConfig.get_api_base(api_base=None) + assert result == env_base + mock_get_secret.assert_called_once_with("GIGACHAT_API_BASE") + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_api_base_fallback_to_default(self, mock_get_secret): + """Test get_api_base falls back to GIGACHAT_BASE_URL.""" + mock_get_secret.return_value = None + result = GigaChatPassthroughConfig.get_api_base(api_base=None) + assert result == "https://gigachat.devices.sberbank.ru/api/v1" + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_api_key_with_explicit_value(self, mock_get_secret): + """Test get_api_key returns explicit value when provided.""" + explicit_key = "test-api-key" + result = GigaChatPassthroughConfig.get_api_key(api_key=explicit_key) + assert result == explicit_key + mock_get_secret.assert_not_called() + + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.passthrough.transformation.get_secret_str" + ) + def test_get_api_key_from_environment(self, mock_get_secret): + """Test get_api_key retrieves from environment when not provided.""" + env_key = "env-api-key" + mock_get_secret.return_value = env_key + result = GigaChatPassthroughConfig.get_api_key(api_key=None) + assert result == env_key + mock_get_secret.assert_called_once_with("GIGACHAT_API_KEY") + + def test_get_base_model_returns_model(self): + """Test get_base_model returns the model as-is.""" + model = "gigachat/GigaChat" + result = GigaChatPassthroughConfig.get_base_model(model) + assert result == model + + def test_get_models(self): + """Test get_models delegates to base class.""" + config = GigaChatPassthroughConfig() + result = config.get_models() + assert result == [] + + def test_logging_non_streaming_chat_raises_when_no_config(self): + """Test raise when ProviderConfigManager returns None for chat.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_chat_config", # test-quality-ok: patching litellm internal for unit test isolation + return_value=None, + ): + with pytest.raises(ValueError, match="No provider config found for model"): + config.logging_non_streaming_response( + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + httpx_response=_make_httpx_response(_gigachat_chat_completion_body()), + request_data={ + "model": "gigachat/GigaChat", + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=logging_obj, + endpoint="chat/completions", + ) + + def test_logging_non_streaming_embedding_raises_when_no_config(self): + """Test raise when ProviderConfigManager returns None for embeddings.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_embedding_config", # test-quality-ok: patching litellm internal for unit test isolation + return_value=None, + ): + with pytest.raises(ValueError, match="No provider config found for model"): + config.logging_non_streaming_response( + model="gigachat/Embeddings", + custom_llm_provider="gigachat", + httpx_response=_make_httpx_response(_gigachat_embedding_body()), + request_data={ + "input": ["hello"], + "model": "gigachat/Embeddings", + }, + logging_obj=logging_obj, + endpoint="embeddings", + ) + + def test_handle_logging_collected_chunks_with_model_response_stream_chunk(self): + """Test that a chunk returning ModelResponseStream from chunk_parser is handled. + + Requires patching GigaChatModelResponseIterator.chunk_parser to return + a ModelResponseStream so the elif branch is exercised. + """ + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + from litellm.types.utils import ModelResponseStream + + stream_chunk = ModelResponseStream( + choices=[ + { + "index": 0, + "delta": {"content": "streamed"}, + "finish_reason": None, + } + ] + ) + + chunks = [ + '{"choices": [{"delta": {"content": "streamed"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}', + ] + + with patch( + "litellm.llms.gigachat.passthrough.transformation.GigaChatModelResponseIterator.chunk_parser", # test-quality-ok: patching litellm internal for unit test isolation + return_value=stream_chunk, + ): + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "streamedstreamed" + + def test_handle_logging_collected_chunks_skips_unknown_chunk_type(self): + """Test that chunk_parser returning an unknown type is skipped.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + chunks = [ + '{"choices": [{"delta": {"content": "good"}, "index": 0}]}', + '{"choices": [{"delta": {}, "finish_reason": "stop", "index": 0}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}', + ] + + with patch( + "litellm.llms.gigachat.passthrough.transformation.GigaChatModelResponseIterator.chunk_parser", # test-quality-ok: patching litellm internal for unit test isolation + return_value=12345, # not dict and not ModelResponseStream + ): + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + # All chunks skipped, returns None + assert result is None + + def test_handle_logging_collected_chunks_skips_unsupported_chunk_type(self): + """Test that unsupported chunk types (non-JSON str) are skipped.""" + config = GigaChatPassthroughConfig() + logging_obj = MagicMock() + + # Both are valid str chunks; "not-a-valid-json" fails json.loads, int is not a str + chunks: list[str] = ["not-a-valid-json"] + + result = config.handle_logging_collected_chunks( + all_chunks=chunks, + litellm_logging_obj=logging_obj, + model="gigachat/GigaChat", + custom_llm_provider="gigachat", + endpoint="chat/completions", + ) + + assert result is None diff --git a/tests/test_litellm/llms/gigachat/test_authenticator.py b/tests/test_litellm/llms/gigachat/test_authenticator.py new file mode 100644 index 00000000000..0a2695dc21e --- /dev/null +++ b/tests/test_litellm/llms/gigachat/test_authenticator.py @@ -0,0 +1,494 @@ +""" +Unit tests for GigaChat OAuth authenticator. + +Tests get_access_token and get_access_token_async covering token resolution +from litellm_params/env, credential validation, caching, and error handling. +""" + +import time +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from litellm.llms.gigachat import authenticator +from litellm.llms.gigachat.authenticator import ( + GigaChatAuthError, + TOKEN_EXPIRY_BUFFER_MS, + get_access_token, + get_access_token_async, +) + + +AUTH_MODULE = "litellm.llms.gigachat.authenticator" + + +def _future_expires_at_ms(offset_seconds: float = 3600) -> int: + return int(time.time() * 1000 + offset_seconds * 1000) + + +def _past_expires_at_ms(offset_seconds: float = 3600) -> int: + return int(time.time() * 1000 - offset_seconds * 1000) + + +@pytest.fixture(autouse=True) +def _isolate_token_cache(): + """Each test gets a fresh module-level token cache to avoid cross-test leakage.""" + with patch(f"{AUTH_MODULE}._token_cache", new=MagicMock()): + authenticator._token_cache.get_cache.return_value = None + authenticator._token_cache.set_cache = MagicMock() + yield + + +class TestGetAccessTokenSync: + def test_returns_token_from_litellm_params(self): + token = get_access_token(litellm_params={"gigachat_access_token": "param-token"}) + assert token == "param-token" + authenticator._token_cache.get_cache.assert_not_called() + + @patch(f"{AUTH_MODULE}.get_secret_str") + def test_returns_token_from_env(self, mock_get_secret): + mock_get_secret.return_value = "env-access-token" + token = get_access_token() + assert token == "env-access-token" + + @patch(f"{AUTH_MODULE}._get_credentials", return_value=None) + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_raises_when_no_credentials(self, mock_get_secret, mock_get_creds): + with pytest.raises(GigaChatAuthError) as exc_info: + get_access_token() + assert exc_info.value.status_code == 401 + assert "credentials not provided" in exc_info.value.message + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value=None) + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_raises_when_no_credentials_even_with_other_resolvers( + self, mock_get_secret, mock_get_creds, mock_scope, mock_auth_url, mock_request + ): + with pytest.raises(GigaChatAuthError) as exc_info: + get_access_token() + assert exc_info.value.status_code == 401 + mock_request.assert_not_called() + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds-from-env") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_requests_new_token_and_caches(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + token = "fresh-token" + expires_at = _future_expires_at_ms() + mock_request.return_value = (token, expires_at) + + result = get_access_token() + + assert result == token + mock_request.assert_called_once_with("creds-from-env", "GIGACHAT_API_PERS", "https://auth.example.com") + authenticator._token_cache.set_cache.assert_called_once() + call_args = authenticator._token_cache.set_cache.call_args + assert call_args.args[1] == (token, expires_at) + assert call_args.kwargs["ttl"] > 0 + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_does_not_cache_when_no_expiry(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + mock_request.return_value = ("token-no-exp", 0) + + result = get_access_token() + + assert result == "token-no-exp" + authenticator._token_cache.set_cache.assert_not_called() + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_does_not_cache_when_ttl_non_positive(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + expires_at = int(time.time() * 1000) + TOKEN_EXPIRY_BUFFER_MS - 1000 + mock_request.return_value = ("token", expires_at) + + result = get_access_token() + + assert result == "token" + authenticator._token_cache.set_cache.assert_not_called() + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_returns_cached_valid_token(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + cached_token = "cached-token" + cached_expires_at = _future_expires_at_ms(offset_seconds=7200) + authenticator._token_cache.get_cache.return_value = (cached_token, cached_expires_at) + + result = get_access_token(credentials="creds") + + assert result == cached_token + mock_request.assert_not_called() + authenticator._token_cache.set_cache.assert_not_called() + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_requests_new_token_when_cache_expired(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + cached_token = "stale-token" + cached_expires_at = _past_expires_at_ms(offset_seconds=10) + authenticator._token_cache.get_cache.return_value = (cached_token, cached_expires_at) + + new_token = "refreshed-token" + mock_request.return_value = (new_token, _future_expires_at_ms()) + + result = get_access_token(credentials="creds") + + assert result == new_token + mock_request.assert_called_once() + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://default-auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="env-creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_litellm_params_override_scope_and_auth_url(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): # test-quality-ok: mock-echo of internal wiring + mock_request.return_value = ("token", _future_expires_at_ms()) + + get_access_token( + litellm_params={ + "gigachat_scope": "GIGACHAT_API_CORP", + "gigachat_auth_url": "https://params-auth.example.com", + } + ) + + mock_request.assert_called_once_with( # test-quality-ok: mock-echo of internal wiring + "env-creds", "GIGACHAT_API_CORP", "https://params-auth.example.com" + ) + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://default-auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="env-creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_explicit_args_override_everything(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): # test-quality-ok: mock-echo of internal wiring + mock_request.return_value = ("token", _future_expires_at_ms()) + + get_access_token( + credentials="explicit-creds", + scope="EXPLICIT_SCOPE", + auth_url="https://explicit.example.com", + litellm_params={ + "gigachat_scope": "PARAM_SCOPE", + "gigachat_auth_url": "https://params.example.com", + }, + ) + + mock_request.assert_called_once_with( # test-quality-ok: mock-echo of internal wiring + "explicit-creds", "EXPLICIT_SCOPE", "https://explicit.example.com" + ) + + @patch(f"{AUTH_MODULE}._request_token_sync") + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + def test_propagates_auth_error_from_request(self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request): + mock_request.side_effect = GigaChatAuthError(status_code=403, message="forbidden") + + with pytest.raises(GigaChatAuthError) as exc_info: + get_access_token() + assert exc_info.value.status_code == 403 + assert exc_info.value.message == "forbidden" + + +class TestGetAccessTokenAsync: + @pytest.mark.asyncio + async def test_returns_token_from_litellm_params(self): + token = await get_access_token_async( + litellm_params={"gigachat_access_token": "param-token"} + ) + assert token == "param-token" + authenticator._token_cache.get_cache.assert_not_called() + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}.get_secret_str") + async def test_returns_token_from_env(self, mock_get_secret): + mock_get_secret.return_value = "env-access-token" + token = await get_access_token_async() + assert token == "env-access-token" + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._get_credentials", return_value=None) + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_raises_when_no_credentials(self, mock_get_secret, mock_get_creds): + with pytest.raises(GigaChatAuthError) as exc_info: + await get_access_token_async() + assert exc_info.value.status_code == 401 + assert "credentials not provided" in exc_info.value.message + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds-from-env") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_requests_new_token_and_caches( + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + token = "fresh-token-async" + expires_at = _future_expires_at_ms() + mock_request.return_value = (token, expires_at) + + result = await get_access_token_async() + + assert result == token + mock_request.assert_called_once_with( + "creds-from-env", "GIGACHAT_API_PERS", "https://auth.example.com" + ) + authenticator._token_cache.set_cache.assert_called_once() + call_args = authenticator._token_cache.set_cache.call_args + assert call_args.args[1] == (token, expires_at) + assert call_args.kwargs["ttl"] > 0 + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_does_not_cache_when_no_expiry( + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + mock_request.return_value = ("token-no-exp", 0) + + result = await get_access_token_async() + + assert result == "token-no-exp" + authenticator._token_cache.set_cache.assert_not_called() + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_returns_cached_valid_token( + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + cached_token = "cached-token-async" + cached_expires_at = _future_expires_at_ms(offset_seconds=7200) + authenticator._token_cache.get_cache.return_value = (cached_token, cached_expires_at) + + result = await get_access_token_async(credentials="creds") + + assert result == cached_token + mock_request.assert_not_called() + authenticator._token_cache.set_cache.assert_not_called() + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_requests_new_token_when_cache_expired( + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + cached_expires_at = _past_expires_at_ms(offset_seconds=10) + authenticator._token_cache.get_cache.return_value = ("stale", cached_expires_at) + + new_token = "refreshed-token-async" + mock_request.return_value = (new_token, _future_expires_at_ms()) + + result = await get_access_token_async(credentials="creds") + + assert result == new_token + mock_request.assert_called_once() + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://default-auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="env-creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_litellm_params_override_scope_and_auth_url( # test-quality-ok: mock-echo of internal wiring + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + mock_request.return_value = ("token", _future_expires_at_ms()) + + await get_access_token_async( + litellm_params={ + "gigachat_scope": "GIGACHAT_API_CORP", + "gigachat_auth_url": "https://params-auth.example.com", + } + ) + + mock_request.assert_called_once_with( # test-quality-ok: mock-echo of internal wiring + "env-creds", "GIGACHAT_API_CORP", "https://params-auth.example.com" + ) + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://default-auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="env-creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_explicit_args_override_everything( # test-quality-ok: mock-echo of internal wiring + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + mock_request.return_value = ("token", _future_expires_at_ms()) + + await get_access_token_async( + credentials="explicit-creds", + scope="EXPLICIT_SCOPE", + auth_url="https://explicit.example.com", + litellm_params={ + "gigachat_scope": "PARAM_SCOPE", + "gigachat_auth_url": "https://params.example.com", + }, + ) + + mock_request.assert_called_once_with( # test-quality-ok: mock-echo of internal wiring + "explicit-creds", "EXPLICIT_SCOPE", "https://explicit.example.com" + ) + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}._request_token_async", new_callable=AsyncMock) + @patch(f"{AUTH_MODULE}._get_auth_url", return_value="https://auth.example.com") + @patch(f"{AUTH_MODULE}._get_scope", return_value="GIGACHAT_API_PERS") + @patch(f"{AUTH_MODULE}._get_credentials", return_value="creds") + @patch(f"{AUTH_MODULE}.get_secret_str", return_value=None) + async def test_propagates_auth_error_from_request( + self, mock_get_secret, mock_creds, mock_scope, mock_auth_url, mock_request + ): + mock_request.side_effect = GigaChatAuthError(status_code=403, message="forbidden") + + with pytest.raises(GigaChatAuthError) as exc_info: + await get_access_token_async() + assert exc_info.value.status_code == 403 + assert exc_info.value.message == "forbidden" + + +class TestRequestTokenSyncErrorMapping: + @patch(f"{AUTH_MODULE}._get_http_client") + def test_http_status_error_maps_to_auth_error(self, mock_get_client): + client = MagicMock() + request = httpx.Request("POST", "https://auth.example.com") + response = httpx.Response(status_code=401, content=b"bad creds", request=request) + http_error = httpx.HTTPStatusError("unauthorized", request=request, response=response) + client.post.side_effect = http_error + mock_get_client.return_value = client + + from litellm.llms.gigachat.authenticator import _request_token_sync + + with pytest.raises(GigaChatAuthError) as exc_info: + _request_token_sync("creds", "GIGACHAT_API_PERS", "https://auth.example.com") + assert exc_info.value.status_code == 401 + assert "bad creds" in exc_info.value.message + + @patch(f"{AUTH_MODULE}._get_http_client") + def test_request_error_maps_to_auth_error(self, mock_get_client): + client = MagicMock() + client.post.side_effect = httpx.ConnectError("connection refused") + mock_get_client.return_value = client + + from litellm.llms.gigachat.authenticator import _request_token_sync + + with pytest.raises(GigaChatAuthError) as exc_info: + _request_token_sync("creds", "GIGACHAT_API_PERS", "https://auth.example.com") + assert exc_info.value.status_code == 500 + assert "connection refused" in exc_info.value.message + + +class TestRequestTokenAsyncErrorMapping: + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}.get_async_httpx_client") + async def test_http_status_error_maps_to_auth_error(self, mock_get_client): + client = MagicMock() + request = httpx.Request("POST", "https://auth.example.com") + response = httpx.Response(status_code=401, content=b"bad creds", request=request) + http_error = httpx.HTTPStatusError("unauthorized", request=request, response=response) + client.post = AsyncMock(side_effect=http_error) + mock_get_client.return_value = client + + from litellm.llms.gigachat.authenticator import _request_token_async + + with pytest.raises(GigaChatAuthError) as exc_info: + await _request_token_async("creds", "GIGACHAT_API_PERS", "https://auth.example.com") + assert exc_info.value.status_code == 401 + assert "bad creds" in exc_info.value.message + + @pytest.mark.asyncio + @patch(f"{AUTH_MODULE}.get_async_httpx_client") + async def test_request_error_maps_to_auth_error(self, mock_get_client): + client = MagicMock() + client.post = AsyncMock(side_effect=httpx.ConnectError("connection refused")) + mock_get_client.return_value = client + + from litellm.llms.gigachat.authenticator import _request_token_async + + with pytest.raises(GigaChatAuthError) as exc_info: + await _request_token_async("creds", "GIGACHAT_API_PERS", "https://auth.example.com") + assert exc_info.value.status_code == 500 + assert "connection refused" in exc_info.value.message + + +class TestParseTokenResponse: + def _make_response(self, body: dict) -> httpx.Response: + import json + + return httpx.Response( + status_code=200, + content=json.dumps(body).encode("utf-8"), + request=httpx.Request("POST", "https://auth.example.com"), + ) + + def test_parses_tok_exp_fields(self): + from litellm.llms.gigachat.authenticator import _parse_token_response + + token, expires_at = _parse_token_response( + self._make_response({"tok": "abc", "exp": 1700000000000}) + ) + assert token == "abc" + assert expires_at == 1700000000000 + + def test_parses_access_token_expires_at_fields(self): + from litellm.llms.gigachat.authenticator import _parse_token_response + + token, expires_at = _parse_token_response( + self._make_response({"access_token": "xyz", "expires_at": 1700000000000}) + ) + assert token == "xyz" + assert expires_at == 1700000000000 + + def test_parses_string_expires_at(self): + from litellm.llms.gigachat.authenticator import _parse_token_response + + token, expires_at = _parse_token_response( + self._make_response({"tok": "abc", "exp": "1700000000000"}) + ) + assert token == "abc" + assert expires_at == 1700000000000 + assert isinstance(expires_at, int) + + def test_raises_when_no_access_token(self): + from litellm.llms.gigachat.authenticator import _parse_token_response + + with pytest.raises(GigaChatAuthError) as exc_info: + _parse_token_response(self._make_response({"exp": 1700000000000})) + assert exc_info.value.status_code == 500 + assert "Invalid token response" in exc_info.value.message + + +class TestGetHttpClient: + def test_reuses_cached_client_across_calls(self): + """Regression: the sync OAuth path must use the shared cached httpx client, + not construct a fresh HTTPHandler per token request.""" + assert authenticator._get_http_client() is authenticator._get_http_client() diff --git a/tests/test_litellm/llms/gigachat/test_file_handler.py b/tests/test_litellm/llms/gigachat/test_file_handler.py new file mode 100644 index 00000000000..ce9505f11f2 --- /dev/null +++ b/tests/test_litellm/llms/gigachat/test_file_handler.py @@ -0,0 +1,504 @@ +""" +Unit tests for GigaChat file handler. + +Tests _get_url_hash, _parse_data_url, _download_image_sync, _download_image_async, +upload_file_sync, and upload_file_async covering caching, base64 data URL decoding, +network errors, and the full upload flow. +""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from litellm.llms.gigachat import file_handler +from litellm.llms.gigachat.file_handler import ( + _file_cache, + _get_url_hash, + _parse_data_url, + upload_file_async, + upload_file_sync, +) + +FILE_MODULE = "litellm.llms.gigachat.file_handler" + +# A valid 1x1 red PNG as base64 +_RED_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAA" + "DUlEQVQI12NgYPgPAAEDAQAR3X3ZAAAASUVORK5CYII=" +) +_RED_PNG_DATA_URL = f"data:image/png;base64,{_RED_PNG_B64}" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _isolate_file_cache(): + """Each test gets a fresh module-level file cache to avoid cross-test leakage.""" + _file_cache.clear() + yield + _file_cache.clear() + + +# --------------------------------------------------------------------------- +# _get_url_hash +# --------------------------------------------------------------------------- + + +class TestGetUrlHash: + def test_returns_hex_string(self): + h = _get_url_hash("https://example.com/image.png") + assert isinstance(h, str) + assert len(h) == 64 # SHA-256 + + def test_different_urls_different_hashes(self): + h1 = _get_url_hash("https://example.com/a.png") + h2 = _get_url_hash("https://example.com/b.png") + assert h1 != h2 + + def test_same_url_same_hash(self): + h1 = _get_url_hash("https://example.com/image.png") + h2 = _get_url_hash("https://example.com/image.png") + assert h1 == h2 + + +# --------------------------------------------------------------------------- +# _parse_data_url +# --------------------------------------------------------------------------- + + +class TestParseDataUrl: + def test_valid_base64_png(self): + result = _parse_data_url(_RED_PNG_DATA_URL) + assert result is not None + content_bytes, content_type, ext = result + assert content_type == "image/png" + assert ext == "png" + assert len(content_bytes) > 0 + + def test_valid_base64_jpeg(self): + # Simple valid base64 (24 chars, properly padded, no + or / chars) + valid_b64 = "aGVsbG8gd29ybGQhISEhIQ==" + data_url = f"data:image/jpeg;base64,{valid_b64}" + result = _parse_data_url(data_url) + assert result is not None + _, content_type, ext = result + assert content_type == "image/jpeg" + assert ext == "jpeg" + + def test_valid_base64_with_semicolon_in_type(self): + """Data URLs with charset before base64 segment do not match the regex.""" + # The regex `data:([^;]+);base64,(.+)` requires the pattern to be + # `data:;base64,`. If `;charset=utf-8` appears before + # `;base64,`, the regex sees `data:image/png` as group 1 but then + # looks for `;base64,` immediately after — which isn't there because + # `;charset=utf-8;base64,` has extra text before `;base64,` + data_url = "data:image/png;charset=utf-8;base64," + _RED_PNG_B64 + result = _parse_data_url(data_url) + assert result is None + + def test_invalid_data_url_returns_none(self): + assert _parse_data_url("not-a-data-url") is None + + def test_empty_base64_returns_none(self): + """Empty base64 data (nothing after comma) does not match regex `(.+)`.""" + assert _parse_data_url("data:image/png;base64,") is None + + def test_missing_base64_segment(self): + assert _parse_data_url("data:image/png;base64") is None + + def test_unknown_extension_falls_back_to_jpg(self): + data_url = "data:application/octet-stream;base64," + _RED_PNG_B64 + result = _parse_data_url(data_url) + assert result is not None + _, content_type, ext = result + assert content_type == "application/octet-stream" + # The extension is derived from content_type.split("/")[-1].split(";")[0] + # which gives "octet-stream", not "jpg" + assert ext == "octet-stream" + + +# --------------------------------------------------------------------------- +# _download_image_sync +# --------------------------------------------------------------------------- + + +class TestDownloadImageSync: + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_downloads_image_successfully(self, mock_http_handler_cls): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.content = b"fake-image-bytes" + mock_response.headers = {"content-type": "image/jpeg"} + mock_client.get.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + content_bytes, content_type, ext = file_handler._download_image_sync("https://example.com/img.jpg") + + assert content_bytes == b"fake-image-bytes" + assert content_type == "image/jpeg" + assert ext == "jpeg" + mock_client.get.assert_called_once_with("https://example.com/img.jpg") + + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_raises_on_http_error(self, mock_http_handler_cls): + mock_client = MagicMock() + mock_client.get.side_effect = httpx.HTTPStatusError( + "Not Found", + request=httpx.Request("GET", "https://example.com/404"), + response=httpx.Response(status_code=404, request=httpx.Request("GET", "https://example.com/404")), + ) + mock_http_handler_cls.return_value = mock_client + + with pytest.raises(httpx.HTTPStatusError): + file_handler._download_image_sync("https://example.com/404") + + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_parse_content_type_fallback(self, mock_http_handler_cls): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.content = b"data" + mock_response.headers = {} + mock_client.get.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + _, content_type, ext = file_handler._download_image_sync("https://example.com/img") + + assert content_type == "image/jpeg" + assert ext == "jpeg" + + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_extracts_extension_from_parametrized_type(self, mock_http_handler_cls): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.content = b"data" + mock_response.headers = {"content-type": "image/png; charset=utf-8"} + mock_client.get.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + _, _, ext = file_handler._download_image_sync("https://example.com/img.png") + + assert ext == "png" + + +# --------------------------------------------------------------------------- +# _download_image_async +# --------------------------------------------------------------------------- + + +class TestDownloadImageAsync: + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_async_httpx_client") + async def test_downloads_image_successfully(self, mock_get_client): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.content = b"fake-image-bytes" + mock_response.headers = {"content-type": "image/webp"} + mock_client.get = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + content_bytes, content_type, ext = await file_handler._download_image_async( + "https://example.com/img.webp" + ) + + assert content_bytes == b"fake-image-bytes" + assert content_type == "image/webp" + assert ext == "webp" + mock_client.get.assert_called_once_with("https://example.com/img.webp") + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_async_httpx_client") + async def test_raises_on_http_error(self, mock_get_client): + mock_client = MagicMock() + mock_client.get = AsyncMock( + side_effect=httpx.HTTPStatusError( + "Forbidden", + request=httpx.Request("GET", "https://example.com/403"), + response=httpx.Response(status_code=403, request=httpx.Request("GET", "https://example.com/403")), + ) + ) + mock_get_client.return_value = mock_client + + with pytest.raises(httpx.HTTPStatusError): + await file_handler._download_image_async("https://example.com/403") + + +# --------------------------------------------------------------------------- +# upload_file_sync +# --------------------------------------------------------------------------- + + +class TestUploadFileSync: + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_uploads_base64_image_and_caches( + self, mock_http_handler_cls, mock_get_token, mock_get_api_base + ): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json.return_value = {"id": "file-12345"} + mock_response.raise_for_status = MagicMock() + mock_client.post.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + result = upload_file_sync( + image_url=_RED_PNG_DATA_URL, + credentials="creds", + api_base="https://custom.example.com", + ) + + assert result == "file-12345" + # Verify it was cached + url_hash = _get_url_hash(_RED_PNG_DATA_URL) + assert _file_cache[url_hash] == "file-12345" + + # Check the upload request — url is passed as first positional arg + call_args = mock_client.post.call_args + assert call_args.args[0] == "https://api.example.com/files" + assert call_args.kwargs["headers"]["Authorization"] == "Bearer test-token" + # Verify purpose + assert call_args.kwargs["data"] == {"purpose": "general"} + # Verify a file was attached + assert "file" in call_args.kwargs["files"] + + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_returns_cached_file_id( + self, mock_http_handler_cls, mock_get_token, mock_get_api_base + ): + # Pre-populate the cache + url_hash = _get_url_hash(_RED_PNG_DATA_URL) + _file_cache[url_hash] = "cached-file-id" + + result = upload_file_sync(image_url=_RED_PNG_DATA_URL, credentials="creds") + + assert result == "cached-file-id" + # No upload call was made + mock_http_handler_cls.return_value.post.assert_not_called() + + @patch(f"{FILE_MODULE}._get_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}._download_image_sync") + def test_downloads_and_uploads_url_image( + self, mock_download, mock_get_api_base, mock_get_token, mock_http_handler_cls + ): + mock_download.return_value = (b"remote-bytes", "image/png", "png") + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json.return_value = {"id": "file-remote"} + mock_response.raise_for_status = MagicMock() + mock_client.post.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + result = upload_file_sync( + image_url="https://example.com/remote.png", credentials="creds" + ) + + assert result == "file-remote" + mock_download.assert_called_once_with("https://example.com/remote.png") + + @patch(f"{FILE_MODULE}._get_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + def test_returns_none_on_upload_failure( + self, mock_get_api_base, mock_get_token, mock_http_handler_cls + ): + mock_client = MagicMock() + mock_client.post.side_effect = httpx.HTTPStatusError( + "Bad Request", + request=httpx.Request("POST", "https://api.example.com/files"), + response=httpx.Response(status_code=400, request=httpx.Request("POST", "https://api.example.com/files")), + ) + mock_http_handler_cls.return_value = mock_client + + # upload_file_sync catches all exceptions and returns None + result = upload_file_sync( + image_url=_RED_PNG_DATA_URL, credentials="creds" + ) + + assert result is None + + @patch(f"{FILE_MODULE}._get_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + def test_returns_none_when_response_missing_id( + self, mock_get_api_base, mock_get_token, mock_http_handler_cls + ): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json.return_value = {"status": "ok"} # no "id" key + mock_response.raise_for_status = MagicMock() + mock_client.post.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + result = upload_file_sync( + image_url=_RED_PNG_DATA_URL, credentials="creds" + ) + + assert result is None + + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token", return_value="test-token") + @patch(f"{FILE_MODULE}._get_httpx_client") + def test_uploads_without_optional_args( + self, mock_http_handler_cls, mock_get_token, mock_get_api_base + ): + """Verify that credentials, api_base, and litellm_params are optional.""" + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json.return_value = {"id": "file-no-args"} + mock_response.raise_for_status = MagicMock() + mock_client.post.return_value = mock_response + mock_http_handler_cls.return_value = mock_client + + result = upload_file_sync(image_url=_RED_PNG_DATA_URL) + + assert result == "file-no-args" + # Should still have called get_access_token without args + mock_get_token.assert_called_once_with(credentials=None, litellm_params=None) + + +# --------------------------------------------------------------------------- +# upload_file_async +# --------------------------------------------------------------------------- + + +class TestUploadFileAsync: + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_async_httpx_client") + async def test_uploads_base64_image_and_caches( + self, mock_get_client, mock_get_token, mock_get_api_base + ): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json = MagicMock(return_value={"id": "async-file-1"}) + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + result = await upload_file_async( + image_url=_RED_PNG_DATA_URL, + credentials="creds", + api_base="https://custom.example.com", + ) + + assert result == "async-file-1" + # Verify cache + url_hash = _get_url_hash(_RED_PNG_DATA_URL) + assert _file_cache[url_hash] == "async-file-1" + + # Check upload request details — url is first positional arg + call_args = mock_client.post.call_args + assert call_args.args[0] == "https://api.example.com/files" + assert call_args.kwargs["headers"]["Authorization"] == "Bearer test-token-async" + assert "purpose" in str(call_args.kwargs["data"]) + assert "file" in call_args.kwargs["files"] + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_async_httpx_client") + async def test_returns_cached_file_id( + self, mock_get_client, mock_get_token, mock_get_api_base + ): + url_hash = _get_url_hash(_RED_PNG_DATA_URL) + _file_cache[url_hash] = "cached-async-id" + + result = await upload_file_async(image_url=_RED_PNG_DATA_URL, credentials="creds") + + assert result == "cached-async-id" + mock_get_client.return_value.post.assert_not_called() + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_async_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}._download_image_async") + async def test_downloads_and_uploads_url_image( + self, mock_download, mock_get_api_base, mock_get_token, mock_get_client + ): + mock_download.return_value = (b"remote-bytes-async", "image/png", "png") + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json = MagicMock(return_value={"id": "async-file-remote"}) + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + result = await upload_file_async( + image_url="https://example.com/remote.png", credentials="creds" + ) + + assert result == "async-file-remote" + mock_download.assert_called_once_with("https://example.com/remote.png") + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_async_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + async def test_returns_none_on_upload_failure( + self, mock_get_api_base, mock_get_token, mock_get_client + ): + mock_client = MagicMock() + mock_client.post = AsyncMock( + side_effect=httpx.HTTPStatusError( + "Bad Request", + request=httpx.Request("POST", "https://api.example.com/files"), + response=httpx.Response(status_code=400, request=httpx.Request("POST", "https://api.example.com/files")), + ) + ) + mock_get_client.return_value = mock_client + + result = await upload_file_async( + image_url=_RED_PNG_DATA_URL, credentials="creds" + ) + + assert result is None + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_async_httpx_client") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + async def test_returns_none_when_response_missing_id( + self, mock_get_api_base, mock_get_token, mock_get_client + ): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json = MagicMock(return_value={"status": "ok"}) + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + result = await upload_file_async( + image_url=_RED_PNG_DATA_URL, credentials="creds" + ) + + assert result is None + + @pytest.mark.asyncio + @patch(f"{FILE_MODULE}.get_api_base", return_value="https://api.example.com") + @patch(f"{FILE_MODULE}.get_access_token_async", return_value="test-token-async") + @patch(f"{FILE_MODULE}.get_async_httpx_client") + async def test_uploads_without_optional_args( + self, mock_get_client, mock_get_token, mock_get_api_base + ): + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.json = MagicMock(return_value={"id": "async-no-args"}) + mock_response.raise_for_status = MagicMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + result = await upload_file_async(image_url=_RED_PNG_DATA_URL) + + assert result == "async-no-args" + mock_get_token.assert_called_once_with(credentials=None, litellm_params=None) \ No newline at end of file diff --git a/tests/test_litellm/llms/gigachat/test_utils.py b/tests/test_litellm/llms/gigachat/test_utils.py new file mode 100644 index 00000000000..71a193d7b29 --- /dev/null +++ b/tests/test_litellm/llms/gigachat/test_utils.py @@ -0,0 +1,79 @@ +""" +Tests for litellm.llms.gigachat.utils +""" + +import pytest +from litellm.llms.gigachat.utils import convert_usage +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + +class TestConvertUsage: + def test_basic_usage_without_precached(self): + """Test convert_usage with standard tokens, no precached prompt tokens.""" + result = convert_usage( + { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + } + ) + + assert result == Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=None, + ) + + def test_usage_with_precached_prompt_tokens(self): + """GigaChat's prompt_tokens and total_tokens exclude cached tokens (docs example: + prompt_tokens=1, precached_prompt_tokens=37, total_tokens=5), so OpenAI-convention + usage adds precached back in and surfaces it as cached_tokens.""" + result = convert_usage( + { + "prompt_tokens": 10, + "completion_tokens": 5, + "precached_prompt_tokens": 3, + "total_tokens": 15, + } + ) + + assert result == Usage( + prompt_tokens=13, + completion_tokens=5, + total_tokens=18, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=3), + ) + + def test_zero_precached_prompt_tokens(self): + """Test convert_usage with zero precached_prompt_tokens does not create details wrapper.""" + result = convert_usage( + { + "prompt_tokens": 10, + "completion_tokens": 5, + "precached_prompt_tokens": 0, + "total_tokens": 15, + } + ) + + assert result == Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=None, + ) + + def test_missing_optional_fields(self): + """Test convert_usage with missing optional fields defaults to zero.""" + result = convert_usage( + { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + } + ) + + assert result.prompt_tokens == 10 + assert result.completion_tokens == 5 + assert result.total_tokens == 15 + assert result.prompt_tokens_details is None \ No newline at end of file diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py index e1cc6a92927..c2efc1acdb9 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool(): } +def test_messages_to_responses_input_preserves_images(): + """An image block must survive the round trip, or OpenAI counts only the text. + + A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it + turned a 268-token request into a 13-token one. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"}, + }, + ], + } + ] + + input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert instructions is None + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + ), + } + ] + + +def test_messages_to_responses_input_image_without_detail_defaults_to_auto(): + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_bare_string_image_url_is_preserved(): + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string(): + """Text-only content must keep collapsing to a string so existing counts do not shift.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "first\nsecond"}] + + +def test_messages_to_responses_input_drops_unmappable_blocks(): + """A block with no Responses API equivalent is skipped, never forwarded verbatim.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[0]["content"] == ( + {"type": "input_text", "text": "hi"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ) + + +def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string(): + """An assistant turn must never forward chat `text` blocks. + + The Responses API only accepts output_text and refusal inside an assistant turn, so + forwarding them 400s the whole request and silently drops the count back to the local + tokenizer, which is exactly what defeats the image fix above. + """ + messages = [ + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Paris."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "Paris."}, + ] + + +def test_messages_to_responses_input_assistant_image_block_is_dropped(): + """An image part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + +def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + }, + {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"}, + ), + }, + {"role": "assistant", "content": "A cat."}, + ] + + +def test_messages_to_responses_input_preserves_inline_files(): + """An inline file must survive the round trip, or the count silently drops the file. + + A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same + request counting 13, the text-only total. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [ + { + "role": "user", + "content": ( + {"type": "input_text", "text": "Summarize this file."}, + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + }, + ), + } + ] + + +def test_messages_to_responses_input_drops_a_file_with_no_inline_data(): + """OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file."}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "user", "content": "Summarize this file."}] + + +def test_messages_to_responses_input_assistant_file_block_is_dropped(): + """A file part is illegal inside an assistant turn, so it must not reach the provider.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here it is"}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="}, + }, + ], + } + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items == [{"role": "assistant", "content": "Here it is"}] + + def test_validate_request_valid(): """Test that valid requests pass validation.""" config = OpenAICountTokensConfig() diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py new file mode 100644 index 00000000000..d8d9936e9a1 --- /dev/null +++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py @@ -0,0 +1,238 @@ +import json +import sys +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig +from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError +from litellm.llms.openai.openai import OpenAIChatCompletion +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.openai.workload_identity import ( + OpenAIWorkloadIdentityConfig, + _workload_identity_auth, + get_workload_identity_bearer_token, + resolve_openai_workload_identity_config, +) +from litellm.types.router import GenericLiteLLMParams + +TOKEN_EXCHANGE_URL: Final = "https://auth.openai.com/oauth/token" + + +@pytest.fixture +def wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> OpenAIWorkloadIdentityConfig: + token_file: Final = tmp_path / "subject_token.jwt" + token_file.write_text("subject-token-from-file") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123") + monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456") + monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file)) + _workload_identity_auth.cache_clear() + litellm.in_memory_llm_clients_cache.flush_cache() + return OpenAIWorkloadIdentityConfig( + identity_provider_id="idp_test123", + service_account_id="user-test456", + token_file=str(token_file), + ) + + +def mock_token_exchange(access_token: str = "exchanged-bearer-token") -> respx.Route: + return respx.post(TOKEN_EXCHANGE_URL).mock( + return_value=httpx.Response(200, json={"access_token": access_token, "expires_in": 3600}) + ) + + +class TestResolveConfig: + def test_resolves_from_env(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_static_api_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key="sk-static", api_base=None) is None + + def test_env_openai_api_key_wins( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + @pytest.mark.parametrize("empty_key", ["", " "]) + def test_empty_api_key_arg_does_not_disable_wif( + self, wif_env: OpenAIWorkloadIdentityConfig, empty_key: str + ) -> None: + assert resolve_openai_workload_identity_config(api_key=empty_key, api_base=None) == wif_env + + @pytest.mark.parametrize("empty_key", ["", " "]) + def test_empty_env_openai_api_key_does_not_disable_wif( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, empty_key: str + ) -> None: + monkeypatch.setenv("OPENAI_API_KEY", empty_key) + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None + + def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env + + def test_plaintext_http_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + assert resolve_openai_workload_identity_config(api_key=None, api_base="http://api.openai.com/v1") is None + + def test_foreign_env_base_url_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://my-vllm.internal/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + def test_openai_env_base_url_allows( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env + + def test_foreign_litellm_api_base_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "api_base", "https://my-vllm.internal/v1") + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + @pytest.mark.parametrize( + "missing_var", + ["OPENAI_IDENTITY_PROVIDER_ID", "OPENAI_SERVICE_ACCOUNT_ID", "OPENAI_IDENTITY_TOKEN_FILE"], + ) + def test_partial_env_disables( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, missing_var: str + ) -> None: + monkeypatch.delenv(missing_var) + assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None + + +class TestTokenExchange: + @respx.mock + def test_exchanges_subject_token_for_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + route: Final = mock_token_exchange() + assert get_workload_identity_bearer_token(wif_env) == "exchanged-bearer-token" + request_body: Final = json.loads(route.calls.last.request.content) + assert request_body["grant_type"] == "urn:ietf:params:oauth:grant-type:token-exchange" + assert request_body["subject_token"] == "subject-token-from-file" + assert request_body["subject_token_type"] == "urn:ietf:params:oauth:token-type:jwt" + assert request_body["identity_provider_id"] == "idp_test123" + assert request_body["service_account_id"] == "user-test456" + + @respx.mock + def test_token_cached_across_mints(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + route: Final = mock_token_exchange() + first: Final = get_workload_identity_bearer_token(wif_env) + second: Final = get_workload_identity_bearer_token(wif_env) + assert first == second == "exchanged-bearer-token" + assert route.call_count == 1 + + def test_old_sdk_raises_upgrade_error( + self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + import openai as openai_module + + monkeypatch.delattr(openai_module, "auth", raising=False) + monkeypatch.setitem(sys.modules, "openai.auth", None) + with pytest.raises(OpenAIError, match=r"openai>=2\.32\.0"): + wif_env.to_sdk_workload_identity() + + +class TestClientConstruction: + def test_sync_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None) + assert isinstance(client, OpenAI) + assert client.api_key == "workload-identity-auth" + assert client._workload_identity_auth is not None + + def test_async_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=True, api_key=None, api_base=None) + assert isinstance(client, AsyncOpenAI) + assert client.api_key == "workload-identity-auth" + assert client._workload_identity_auth is not None + + def test_static_key_client_unaffected(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key="sk-static", api_base=None) + assert isinstance(client, OpenAI) + assert client.api_key == "sk-static" + assert client._workload_identity_auth is None + + def test_cache_key_separates_wif_identities(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + other_config: Final = OpenAIWorkloadIdentityConfig( + identity_provider_id="idp_other", + service_account_id="user-other", + token_file=wif_env.token_file, + ) + keys: Final = tuple( + BaseOpenAILLM.get_openai_client_cache_key( + client_initialization_params={"api_key": None, "is_async": False, "workload_identity_config": config}, + client_type="openai", + ) + for config in (wif_env, other_config, None) + ) + assert len(set(keys)) == 3 + + @respx.mock + def test_request_carries_exchanged_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + mock_token_exchange() + completion_route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-wif", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + client = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None) + assert isinstance(client, OpenAI) + client.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}]) + auth_header: Final = completion_route.calls.last.request.headers["Authorization"] + assert auth_header == "Bearer exchanged-bearer-token" + + +class TestResponsesValidateEnvironment: + @respx.mock + def test_mints_bearer_when_wif_configured(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + mock_token_exchange() + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer exchanged-bearer-token" + + def test_static_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams(api_key="sk-responses") + ) + assert headers["Authorization"] == "Bearer sk-responses" + + def test_foreign_api_base_skips_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = OpenAIResponsesAPIConfig().validate_environment( + headers={}, + model="gpt-4o-mini", + litellm_params=GenericLiteLLMParams(api_base="https://my-vllm.internal/v1"), + ) + assert headers["Authorization"] == "Bearer None" + + def test_litellm_proxy_subclass_never_mints_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None: + headers: Final = LiteLLMProxyResponsesAPIConfig().validate_environment( + headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams() + ) + assert headers["Authorization"] == "Bearer None" diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py index 29d22e844a5..a4d67606698 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py @@ -982,6 +982,116 @@ class TestVertexBase: assert result_url == f"{gateway_api_base}:embedContent" + def test_check_custom_proxy_vertex_api_base_with_version_path_grafts_default_path(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://aiplatform.googleapis.com/v1beta1", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://aiplatform.googleapis.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_trailing_slash_grafts_default_path(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1/", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_grafts_before_query(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1beta1?key=abc", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent?key=abc" + ) + + def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_streaming_appends_alt_sse(self): + vertex_base = VertexBase() + + _, result_url = vertex_base._check_custom_proxy( + api_base="https://internal-gateway.example.com/v1beta1?key=abc", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="streamGenerateContent", + stream=True, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent", + model="gemini-3.5-flash-lite", + ) + + assert ( + result_url + == "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent?key=abc&alt=sse" + ) + + def test_check_custom_proxy_vertex_api_base_with_non_version_path_keeps_endpoint_append(self): + vertex_base = VertexBase() + gateway_api_base = "https://gateway.example.com/vertex-proxy" + + _, result_url = vertex_base._check_custom_proxy( + api_base=gateway_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent", + model="gemini-3.5-flash-lite", + ) + + assert result_url == f"{gateway_api_base}:generateContent" + + def test_check_custom_proxy_vertex_api_base_without_projects_in_default_url_keeps_endpoint_append(self): + vertex_base = VertexBase() + gemma_api_base = "https://example.com/custom/gemma-deployment" + + _, result_url = vertex_base._check_custom_proxy( + api_base=gemma_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header=None, + url=gemma_api_base, + model="gemma-3-27b-it", + ) + + assert result_url == f"{gemma_api_base}:predict" + def test_check_custom_proxy_vertex_bare_host_streaming_keeps_single_alt_sse(self): vertex_base = VertexBase() diff --git a/tests/test_litellm/models/test_models.py b/tests/test_litellm/models/test_models.py index 669dba8e466..9ae9b732066 100644 --- a/tests/test_litellm/models/test_models.py +++ b/tests/test_litellm/models/test_models.py @@ -5,6 +5,7 @@ Tests for backend domain models. from datetime import datetime import pytest +from pydantic import BaseModel, TypeAdapter from litellm.models.access_group import LiteLLM_AccessGroupTable from litellm.models.budget import ( @@ -130,6 +131,33 @@ class TestModel: assert model.litellm_params == {"model": "gpt-4"} assert model.model_info == {"team_id": "t1"} + def test_response_type_adapter_accepts_pydantic_row(self): + class PrismaModelRow(BaseModel): + model_id: str + model_name: str + litellm_params: dict[str, str] + model_info: dict[str, str] | None = None + blocked: bool = False + + row = PrismaModelRow( + model_id="m1", + model_name="gpt-4", + litellm_params={"model": "gpt-4"}, + model_info={"team_id": "t1"}, + blocked=True, + ) + + model = TypeAdapter(LiteLLM_ProxyModelTable | None).validate_python( + row, + from_attributes=True, + ) + + assert model is not None + assert model.model_id == "m1" + assert model.litellm_params == {"model": "gpt-4"} + assert model.model_info == {"team_id": "t1"} + assert model.blocked is True + def test_team_helpers_none_when_no_model_info(self): model = LiteLLM_ProxyModelTable( model_id="m1", model_name="gpt-4", litellm_params={}, model_info=None diff --git a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py index faf4ea46c43..9f2b436d2d8 100644 --- a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py +++ b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py @@ -1,12 +1,12 @@ """ -Tests for error propagation in _async_streaming passthrough routes. +Tests for error propagation in async passthrough streaming routes. -Verifies that HTTP 4xx/5xx errors from upstream (e.g. Azure 429 rate limits) -raise exceptions instead of being silently forwarded as raw bytes under HTTP 200. - -See: litellm/passthrough/main.py _async_streaming() +Verifies that streaming passthrough wrappers preserve the previous guarantees: +HTTP 4xx/5xx failures must raise instead of being silently forwarded as bytes, +and successful streaming responses should still yield chunks normally. """ +import asyncio import json from unittest.mock import AsyncMock, MagicMock @@ -54,19 +54,19 @@ def _make_mock_logging_obj(): @pytest.mark.asyncio async def test_async_streaming_429_raises(): """429 from upstream should raise HTTPStatusError, not yield error bytes.""" - from litellm.passthrough.main import _async_streaming - + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + error_body = json.dumps( {"error": {"code": "429", "message": "Rate limit exceeded."}} ).encode() mock_response = _make_mock_response(429, error_body) - + async def response_coro(): return mock_response - + chunks = [] async def _drain(): - async for chunk in _async_streaming( + async for chunk in AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=_make_mock_logging_obj(), provider_config=MagicMock(), @@ -83,45 +83,78 @@ async def test_async_streaming_429_raises(): @pytest.mark.asyncio async def test_async_streaming_500_raises(): """500 from upstream should also raise, not yield error bytes.""" - from litellm.passthrough.main import _async_streaming - + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + error_body = json.dumps( {"error": {"code": "500", "message": "Internal server error"}} ).encode() mock_response = _make_mock_response(500, error_body) - + async def response_coro(): return mock_response - + with pytest.raises(httpx.HTTPStatusError) as exc_info: - async for _ in _async_streaming( + async for _ in AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=_make_mock_logging_obj(), provider_config=MagicMock(), ): pass - + assert exc_info.value.response.status_code == 500 @pytest.mark.asyncio -async def test_async_streaming_200_yields_chunks(): +async def test_async_passthrough_wrapper_200_yields_chunks(): """Successful 200 streaming responses should continue to work normally.""" - from litellm.passthrough.main import _async_streaming + from litellm.passthrough.main import AsyncPassthroughStreamingResponse sse_data = b'data: {"type":"response.created"}\n\ndata: [DONE]\n\n' mock_response = _make_mock_response(200, sse_data) + mock_logging_obj = _make_mock_logging_obj() async def response_coro(): return mock_response - chunks = [] - async for chunk in _async_streaming( + async_stream = AsyncPassthroughStreamingResponse( response=response_coro(), - litellm_logging_obj=_make_mock_logging_obj(), + litellm_logging_obj=mock_logging_obj, provider_config=MagicMock(), - ): + ) + + chunks = [] + async for chunk in async_stream: chunks.append(chunk) + await asyncio.sleep(0) + assert len(chunks) == 1 assert b"response.created" in chunks[0] + mock_logging_obj.async_flush_passthrough_collected_chunks.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_error_body_readable_after_failed_await(): + """The upstream error body must stay readable so the proxy can map the real status and message.""" + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + + error_body = b'{"message":"model not found"}' + + async def byte_stream(): + yield error_body + + request = httpx.Request("POST", "https://bedrock.example.com/model/x/converse-stream") + response = httpx.Response(400, content=byte_stream(), request=request) + + async def response_coro(): + return response + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await AsyncPassthroughStreamingResponse( + response=response_coro(), + litellm_logging_obj=_make_mock_logging_obj(), + provider_config=MagicMock(), + ) + + assert exc_info.value.response.status_code == 400 + assert await exc_info.value.response.aread() == error_body diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index b8f265ad7ea..1950c37a12e 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -645,13 +645,14 @@ async def test_allm_passthrough_route_429_streaming_raises(): Regression test: Azure 429 during streaming must raise HTTPStatusError, not be silently forwarded as raw bytes under HTTP 200. - Before the fix, _async_streaming() would yield the 429 error JSON as - chunks and allm_passthrough_route returned an async generator. The - caller (azure_proxy_route) wrapped it in StreamingResponse(status_code=200), + Before the fix, the async passthrough streaming path would yield the 429 + error JSON as chunks and allm_passthrough_route returned a streaming + iterator. The caller (azure_proxy_route) wrapped it in + StreamingResponse(status_code=200), so the client saw HTTP 200 + unparseable SSE body → silent task_complete(null). - After the fix, raise_for_status() fires inside _async_streaming() before - any chunks are yielded, so the exception propagates all the way up. + After the fix, raise_for_status() fires before the streaming wrapper is + returned, so the exception propagates all the way up. """ mock_provider_config = MagicMock() mock_provider_config.get_complete_url.return_value = ( @@ -679,6 +680,7 @@ async def test_allm_passthrough_route_429_streaming_raises(): mock_logging_obj = MagicMock() mock_logging_obj.update_environment_variables = MagicMock() mock_logging_obj.async_flush_passthrough_collected_chunks = AsyncMock() + mock_logging_obj.async_failure_handler = AsyncMock() with ( patch( @@ -701,29 +703,101 @@ async def test_allm_passthrough_route_429_streaming_raises(): patch.object(async_client.client, "send", mock_send), patch.object(async_client.client, "build_request", mock_build_request), ): - result = await allm_passthrough_route( - model="azure/gpt-4", - endpoint="openai/deployments/gpt-4/responses", - method="POST", - custom_llm_provider="azure", - api_base="https://my-azure.openai.azure.com", - api_key="fake-azure-key", - json={"model": "gpt-4", "input": "hello", "stream": True}, - client=async_client, - litellm_logging_obj=mock_logging_obj, - ) - - # result is an async generator — consuming it must raise, not silently yield error bytes - chunks = [] - async def _drain(): - async for chunk in result: # type: ignore[union-attr] - chunks.append(chunk) - with pytest.raises(httpx.HTTPStatusError) as exc_info: - await _drain() + await allm_passthrough_route( + model="azure/gpt-4", + endpoint="openai/deployments/gpt-4/responses", + method="POST", + custom_llm_provider="azure", + api_base="https://my-azure.openai.azure.com", + api_key="fake-azure-key", + json={"model": "gpt-4", "input": "hello", "stream": True}, + client=async_client, + litellm_logging_obj=mock_logging_obj, + ) assert exc_info.value.response.status_code == 429 - assert len(chunks) == 0, "No chunks should be yielded before the 429 raises" + + +def test_llm_passthrough_route_sync_streaming_error_maps_upstream_status(): + """ + Regression test: a sync streaming passthrough whose upstream answers an + error status must surface the mapped provider error, not + httpx.ResponseNotRead. + + Before the fix, raise_for_status() raised on the still-unread streamed + response, and _handle_error then touched e.response.text, which raises + ResponseNotRead on a streamed-but-unread body, masking the real upstream + error entirely. + """ + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + error_body = json.dumps( + { + "error": { + "code": "429", + "message": "Rate limit exceeded. Retry after 10 seconds.", + } + } + ).encode() + + class _UnreadErrorStream(httpx.SyncByteStream): + def __iter__(self): + yield error_body + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 429, + stream=_UnreadErrorStream(), + headers={"content-type": "application/json"}, + ) + + sync_client = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_handler)) + ) + + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL("https://gigachat.devices.sberbank.ru/api/v1/chat/completions"), + "https://gigachat.devices.sberbank.ru/api/v1", + ) + mock_provider_config.get_api_key.return_value = "fake-key" + mock_provider_config.validate_environment.return_value = { + "Authorization": "Bearer fake-key" + } + mock_provider_config.sign_request.return_value = ( + {"Authorization": "Bearer fake-key"}, + None, + ) + mock_provider_config.is_streaming_request.return_value = True + mock_provider_config.get_error_class.side_effect = ( + lambda error_message, status_code, headers: BaseLLMException( + status_code=status_code, message=error_message, headers=headers + ) + ) + + mock_logging_obj = MagicMock() + + with pytest.raises(BaseLLMException) as exc_info: + llm_passthrough_route( + model="gigachat/GigaChat-2", + endpoint="chat/completions", + method="POST", + custom_llm_provider="gigachat", + api_base="https://gigachat.devices.sberbank.ru/api/v1", + api_key="fake-key", + json={ + "model": "GigaChat-2", + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + client=sync_client, + litellm_logging_obj=mock_logging_obj, + provider_config=mock_provider_config, + ) + + assert exc_info.value.status_code == 429 + assert "Rate limit exceeded" in str(exc_info.value) def test_llm_passthrough_route_propagates_allm_passthrough_route_to_logging_obj(): diff --git a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py index 3783e218e4e..a88b0ef0c4b 100644 --- a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py +++ b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py @@ -35,11 +35,14 @@ class _ImmediateExecutor: @pytest.mark.asyncio -async def test_async_streaming_flushes_on_normal_completion(): - from litellm.passthrough.main import _async_streaming +async def test_asyncpassthroughstreamingresponse_flushes_on_normal_completion(): + from litellm.passthrough.main import AsyncPassthroughStreamingResponse chunks = [b"chunk-1", b"chunk-2", b"chunk-3"] mock_response = _make_streaming_response(chunks) + mock_response.headers = httpx.Headers( + {"content-type": "application/octet-stream", "x-request-id": "req-123"} + ) async def response_coro(): return mock_response @@ -48,14 +51,19 @@ async def test_async_streaming_flushes_on_normal_completion(): provider_config = MagicMock() received = [] - async for chunk in _async_streaming( + received_response = AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=mock_logging_obj, provider_config=provider_config, - ): + ) + + async for chunk in received_response: received.append(chunk) assert received == chunks + + assert received_response.headers["content-type"] == "application/octet-stream" + assert received_response.headers["x-request-id"] == "req-123" await asyncio.sleep(0) @@ -68,8 +76,8 @@ async def test_async_streaming_flushes_on_normal_completion(): @pytest.mark.asyncio -async def test_async_streaming_flushes_on_client_disconnect(): - from litellm.passthrough.main import _async_streaming +async def test_asyncpassthroughstreamingresponse_flushes_on_client_disconnect(): + from litellm.passthrough.main import AsyncPassthroughStreamingResponse chunks = [ b'{"chunk": 1, "outputTokens": 10}', @@ -77,6 +85,9 @@ async def test_async_streaming_flushes_on_client_disconnect(): b'{"chunk": 3, "outputTokens": 8}', ] mock_response = _make_streaming_response(chunks) + mock_response.headers = httpx.Headers( + {"content-type": "application/octet-stream", "x-request-id": "req-123"} + ) async def response_coro(): return mock_response @@ -84,7 +95,7 @@ async def test_async_streaming_flushes_on_client_disconnect(): mock_logging_obj = _make_logging_obj() provider_config = MagicMock() - gen = _async_streaming( + gen = AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=mock_logging_obj, provider_config=provider_config, @@ -105,11 +116,14 @@ async def test_async_streaming_flushes_on_client_disconnect(): @pytest.mark.asyncio -async def test_async_streaming_does_not_flush_on_4xx(): - from litellm.passthrough.main import _async_streaming +async def test_asyncpassthroughstreamingresponse_does_not_flush_on_4xx(): + from litellm.passthrough.main import AsyncPassthroughStreamingResponse err_response = MagicMock(spec=httpx.Response) err_response.status_code = 429 + err_response.headers = httpx.Headers( + {"content-type": "application/octet-stream"} + ) def _raise(): raise httpx.HTTPStatusError( @@ -129,7 +143,7 @@ async def test_async_streaming_does_not_flush_on_4xx(): mock_logging_obj = _make_logging_obj() with pytest.raises(httpx.HTTPStatusError): - async for _ in _async_streaming( + async for _ in AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=mock_logging_obj, provider_config=MagicMock(), @@ -140,8 +154,8 @@ async def test_async_streaming_does_not_flush_on_4xx(): @pytest.mark.asyncio -async def test_async_streaming_flushes_on_upstream_exception_with_partial_data(): - from litellm.passthrough.main import _async_streaming +async def test_asyncpassthroughstreamingresponse_flushes_on_upstream_exception_with_partial_data(): + from litellm.passthrough.main import AsyncPassthroughStreamingResponse partial_chunks = [b"partial-chunk-1", b"partial-chunk-2"] @@ -149,6 +163,9 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() mock_response.status_code = 200 mock_response.raise_for_status = MagicMock(return_value=None) mock_response.aclose = AsyncMock() + mock_response.headers = httpx.Headers( + {"content-type": "application/octet-stream", "x-request-id": "req-123"} + ) async def _aiter_bytes_then_raise(): for c in partial_chunks: @@ -165,7 +182,7 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() received = [] async def _drain(): - async for chunk in _async_streaming( + async for chunk in AsyncPassthroughStreamingResponse( response=response_coro(), litellm_logging_obj=mock_logging_obj, provider_config=provider_config, @@ -186,12 +203,16 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() assert call_kwargs["raw_bytes"] == partial_chunks -def test_sync_streaming_flushes_on_normal_completion(): - from litellm.passthrough.main import _sync_streaming +def test_passthroughstreamingresponse_flushes_on_normal_completion(): + from litellm.passthrough.main import PassthroughStreamingResponse chunks = [b"a", b"b", b"c"] mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = httpx.Headers( + {"content-type": "application/octet-stream", "x-request-id": "req-123"} + ) def _iter_bytes(): yield from chunks @@ -202,25 +223,33 @@ def test_sync_streaming_flushes_on_normal_completion(): mock_logging_obj.flush_passthrough_collected_chunks = MagicMock() provider_config = MagicMock() + received_responce = PassthroughStreamingResponse( + response=mock_response, + litellm_logging_obj=mock_logging_obj, + provider_config=provider_config, + ) + with patch("litellm.utils.executor", _ImmediateExecutor()): - received = list( - _sync_streaming( - response=mock_response, - litellm_logging_obj=mock_logging_obj, - provider_config=provider_config, - ) - ) + received = list(received_responce) assert received == chunks + + assert received_responce.headers["content-type"] == "application/octet-stream" + assert received_responce.headers["x-request-id"] == "req-123" + mock_logging_obj.flush_passthrough_collected_chunks.assert_called_once() -def test_sync_streaming_flushes_on_early_close(): - from litellm.passthrough.main import _sync_streaming +def test_passthroughstreamingresponse_flushes_on_early_close(): + from litellm.passthrough.main import PassthroughStreamingResponse chunks = [b"first", b"second", b"third"] mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = httpx.Headers( + {"content-type": "application/octet-stream", "x-request-id": "req-123"} + ) def _iter_bytes(): yield from chunks @@ -232,7 +261,7 @@ def test_sync_streaming_flushes_on_early_close(): provider_config = MagicMock() with patch("litellm.utils.executor", _ImmediateExecutor()): - gen = _sync_streaming( + gen = PassthroughStreamingResponse( response=mock_response, litellm_logging_obj=mock_logging_obj, provider_config=provider_config, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 3279c59acd4..598e9276423 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -35,6 +35,22 @@ def mock_mcp_client_ip(): yield +@pytest.fixture(autouse=True) +def isolate_global_mcp_registry(): + """Restore the module-global MCP server registry after each test. + + Tests here register servers on ``global_mcp_server_manager`` directly; without a + restore, entries leak into other test modules sharing the same worker and break + assertions over the full registry contents. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + + snapshot = dict(global_mcp_server_manager.registry) + yield + global_mcp_server_manager.registry.clear() + global_mcp_server_manager.registry.update(snapshot) + + def _mock_callback_request(base_url: str = "http://localhost:3000/"): """Return a MagicMock Request for callback/authorize same-origin tests. diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 90b3b29d919..90be51cfa5b 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -26,6 +26,7 @@ from prisma.errors import ( from litellm._logging import verbose_proxy_logger +from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler @@ -703,23 +704,43 @@ async def test_auth_failure_ip_stamp_does_not_mutate_callers_request_data(): assert request_data == {"model": "gpt-4o"} +def _marked_malformed_key_error() -> HTTPException: + """Build the malformed-key 401 as its raise site does: marker stamped on it.""" + error = HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test") + setattr(error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return error + + @pytest.mark.asyncio @pytest.mark.parametrize( - "auth_error,expect_traceback", + "auth_error,expect_traceback,expect_level", [ pytest.param( ProxyException( message="Authentication Error", type=ProxyErrorTypes.auth_error, param=None, code=401 ), False, + "ERROR", id="expected_401_no_traceback", ), - pytest.param(ValueError("unexpected internal error"), True, id="unexpected_error_keeps_traceback"), + pytest.param(ValueError("unexpected internal error"), True, "ERROR", id="unexpected_error_keeps_traceback"), + pytest.param( + _marked_malformed_key_error(), + False, + "WARNING", + id="malformed_virtual_key_warning_no_traceback", + ), + pytest.param( + HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test"), + False, + "ERROR", + id="phrase_without_marker_stays_loud", + ), ], ) -async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, caplog): +async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, expect_level, caplog): """Regression for LIT-6043: expected 4xx auth rejections must not format a - traceback via logger.exception; unexpected errors must keep it.""" + traceback via logger.exception; malformed virtual keys log at WARNING.""" handler = UserAPIKeyAuthExceptionHandler() with ( @@ -740,8 +761,8 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( try: try: raise auth_error - except (ProxyException, ValueError) as caught: - with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException): + except (ProxyException, ValueError, HTTPException) as caught: + with caplog.at_level(expect_level, logger="LiteLLM Proxy"), pytest.raises((ProxyException, HTTPException)): await handler._handle_authentication_error( caught, MagicMock(), @@ -756,3 +777,6 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( records = [r for r in caplog.records if "user_api_key_auth(): Exception occured" in r.getMessage()] assert len(records) == 1 assert (records[0].exc_info is not None) is expect_traceback + assert records[0].levelname == expect_level + expected_logger_name = "LiteLLM Proxy.stdout" if expect_level == "WARNING" else "LiteLLM Proxy" + assert records[0].name == expected_logger_name diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 504654e103a..8f3508fc4e9 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -203,7 +203,7 @@ async def test_get_all_transactions_from_redis_buffer_pipeline(redis_update_buff "window_duration": "30d", "window_start": "2026-08-01T00:00:00.000000", "spend": 3.0, - "request_ids": ["req-1"], + "started_at": None, } ] ) @@ -233,13 +233,11 @@ async def test_get_all_transactions_from_redis_buffer_pipeline(redis_update_buff window_spend, ) = result - # Budget window spend from two pods is summed per window, not overwritten, - # and both pods' request ids reach the seed exclusion. + # Budget window spend from two pods is summed per window, not overwritten. assert window_spend is not None assert len(window_spend) == 1 assert window_spend[0]["spend"] == 6.0 assert window_spend[0]["entity_id"] == "hashed-token" - assert window_spend[0]["request_ids"] == ("req-1",) # Verify db spend was parsed correctly assert db_spend is not None @@ -326,7 +324,6 @@ async def test_restored_window_spend_transactions_drain_back_unchanged(redis_upd window_duration="30d", window_start=datetime(2026, 8, 1, tzinfo=timezone.utc), spend=3.0, - request_id="req-1", started_at=datetime(2026, 8, 10, 12, 0, tzinfo=timezone.utc), ), ) @@ -500,7 +497,6 @@ async def test_store_in_memory_spend_updates_pushes_budget_window_spend(redis_up window_duration="30d", window_start=datetime(2026, 8, 1, tzinfo=timezone.utc), spend=1.25, - request_id="req-1", started_at=datetime(2026, 8, 10, 12, 0, 0, tzinfo=timezone.utc), ) ) @@ -526,12 +522,45 @@ async def test_store_in_memory_spend_updates_pushes_budget_window_spend(redis_up "window_duration": "30d", "window_start": "2026-08-01T00:00:00.000000", "spend": 1.25, - "request_ids": ["req-1"], "started_at": "2026-08-10T12:00:00.000000", + "request_ids": [], } ] +@pytest.mark.asyncio +async def test_budget_window_payloads_keep_request_ids_for_older_workers(redis_update_buffer, mock_redis_cache): + """A leader from before the field was dropped indexes request_ids while + merging what it popped, and the pop is destructive, so a payload without + the key would cost a rolling deploy those increments.""" + from datetime import datetime, timezone + + from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( + WindowSpendUpdateQueue, + build_window_spend_transaction, + ) + + mock_redis_cache.async_rpush_pipeline = AsyncMock(return_value=[1]) + window_queue = WindowSpendUpdateQueue() + await window_queue.add_update( + build_window_spend_transaction( + entity_type="key", + entity_id="hashed-token", + window_duration="30d", + window_start=datetime(2026, 8, 1, tzinfo=timezone.utc), + spend=1.25, + ) + ) + + await redis_update_buffer.restore_transactions_to_redis( + window_spend_update_transactions=await window_queue.flush_and_get_aggregated_window_spend_transactions(), + ) + + rpush_list = mock_redis_cache.async_rpush_pipeline.call_args.kwargs["rpush_list"] + restored = json.loads(rpush_list[0]["values"][0]) + assert [payload["request_ids"] for payload in restored] == [[]] + + @pytest.mark.asyncio async def test_store_in_memory_spend_updates_restores_budget_window_spend_on_rpush_failure( redis_update_buffer, mock_redis_cache diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_window_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_window_spend_update_queue.py index b1ecda57afa..6632b1c8e35 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_window_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_window_spend_update_queue.py @@ -19,7 +19,6 @@ def _txn( spend: float, duration: str = "30d", entity_type: str = "key", - request_id: str | None = None, started_at: datetime | None = None, ): return build_window_spend_transaction( @@ -28,7 +27,6 @@ def _txn( window_duration=duration, window_start=window_start, spend=spend, - request_id=request_id, started_at=started_at, ) @@ -38,13 +36,12 @@ def test_build_window_spend_transaction_stores_naive_utc_iso(): TIMESTAMP(3) column, so a non-UTC input must be converted, not truncated.""" non_utc = datetime(2026, 8, 1, 20, 0, tzinfo=timezone(timedelta(hours=-4))) - assert _txn("k1", non_utc, 1.0, request_id="req-1") == { + assert _txn("k1", non_utc, 1.0) == { "entity_type": "key", "entity_id": "k1", "window_duration": "30d", "window_start": "2026-08-02T00:00:00.000000", "spend": 1.0, - "request_ids": ("req-1",), "started_at": None, } @@ -59,19 +56,19 @@ def test_build_window_spend_transaction_stores_started_at_as_naive_utc_iso(): @pytest.mark.asyncio async def test_aggregation_keeps_the_earliest_started_at_of_the_batch(): - """The seed bounds its request-id exclusion at the batch's earliest start, - so a later start must never win the merge.""" + """The seed stops at the batch's earliest start, so a later start must never + win the merge: it would push the cutoff forward and count a request the + increments already cover.""" queue = WindowSpendUpdateQueue() earliest = datetime(2026, 8, 10, 12, 0, 0, tzinfo=timezone.utc) - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-2", started_at=earliest + timedelta(seconds=5))) - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-1", started_at=earliest)) - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-3")) + await queue.add_update(_txn("k1", WINDOW_A, 1.0, started_at=earliest + timedelta(seconds=5))) + await queue.add_update(_txn("k1", WINDOW_A, 1.0, started_at=earliest)) + await queue.add_update(_txn("k1", WINDOW_A, 1.0)) aggregated = await queue.flush_and_get_aggregated_window_spend_transactions() assert len(aggregated) == 1 assert aggregated[0]["started_at"] == "2026-08-10T12:00:00.000000" - assert aggregated[0]["request_ids"] == ("req-1", "req-2", "req-3") def test_to_naive_utc_leaves_naive_values_alone(): @@ -208,62 +205,12 @@ def test_aggregation_survives_the_redis_json_round_trip(): assert reloaded == aggregated -@pytest.mark.asyncio -async def test_aggregation_unions_the_request_ids_of_merged_increments(): - """The seed excludes exactly the requests its batch already covers, so every - merged increment's id has to survive aggregation.""" - queue = WindowSpendUpdateQueue() - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-1")) - await queue.add_update(_txn("k1", WINDOW_A, 2.0, request_id="req-2")) - - aggregated = await queue.flush_and_get_aggregated_window_spend_transactions() - - assert len(aggregated) == 1 - assert aggregated[0]["request_ids"] == ("req-1", "req-2") - - -@pytest.mark.asyncio -async def test_request_ids_stay_with_their_own_window(): - queue = WindowSpendUpdateQueue() - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-a")) - await queue.add_update(_txn("k1", WINDOW_B, 2.0, request_id="req-b")) - - aggregated = await queue.flush_and_get_aggregated_window_spend_transactions() - - assert {payload["window_start"]: payload["request_ids"] for payload in aggregated} == { - "2026-08-01T00:00:00.000000": ("req-a",), - "2026-08-31T00:00:00.000000": ("req-b",), - } - - -@pytest.mark.asyncio -async def test_request_ids_are_deduplicated_and_ordered(): - queue = WindowSpendUpdateQueue() - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-b")) - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-a")) - await queue.add_update(_txn("k1", WINDOW_A, 1.0, request_id="req-a")) - - aggregated = await queue.flush_and_get_aggregated_window_spend_transactions() - - assert aggregated[0]["request_ids"] == ("req-a", "req-b") - - -@pytest.mark.asyncio -async def test_increment_without_a_request_id_carries_no_exclusion(): - queue = WindowSpendUpdateQueue() - await queue.add_update(_txn("k1", WINDOW_A, 1.0)) - - aggregated = await queue.flush_and_get_aggregated_window_spend_transactions() - - assert aggregated[0]["request_ids"] == () - - -def test_request_ids_survive_the_redis_json_round_trip(): +def test_started_at_survives_the_redis_json_round_trip(): aggregated = WindowSpendUpdateQueue.get_aggregated_window_spend_transactions( - [(_txn("k1", WINDOW_A, 1.0, request_id="req-1"),)] + [(_txn("k1", WINDOW_A, 1.0, started_at=datetime(2026, 8, 10, 12, 0, tzinfo=timezone.utc)),)] ) reloaded = WindowSpendUpdateQueue.get_aggregated_window_spend_transactions([json.loads(json.dumps(aggregated))]) - assert reloaded[0]["request_ids"] == ("req-1",) + assert reloaded[0]["started_at"] == "2026-08-10T12:00:00.000000" assert reloaded[0]["spend"] == 1.0 diff --git a/tests/test_litellm/proxy/db/test_budget_window_spend_writer.py b/tests/test_litellm/proxy/db/test_budget_window_spend_writer.py index a849317c930..130f0c56ccf 100644 --- a/tests/test_litellm/proxy/db/test_budget_window_spend_writer.py +++ b/tests/test_litellm/proxy/db/test_budget_window_spend_writer.py @@ -6,9 +6,10 @@ from typing import Any import pytest from litellm.proxy.db.budget_window_spend_writer import ( + WindowSeedTotals, commit_window_spend_updates, roll_window_spend_row, - spend_logs_total_excluding, + spend_logs_seed_totals, ) from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, @@ -70,10 +71,15 @@ class _FakePrismaClient: class _RecordingAggregate: - """Stands in for the LiteLLM_SpendLogs seed aggregate.""" + """Stands in for the LiteLLM_SpendLogs seed aggregate. before_batch + defaults to the full total, the state where none of this batch's own log + rows have been persisted yet.""" - def __init__(self, value: float = 5.0) -> None: - self.value = value + def __init__(self, total: float = 5.0, before_batch: float | None = None) -> None: + self.totals = WindowSeedTotals( + total=total, + before_batch=total if before_batch is None else before_batch, + ) self.calls: list[dict[str, Any]] = [] async def __call__( @@ -82,25 +88,23 @@ class _RecordingAggregate: entity_type: str, entity_id: str, window_start: datetime, - exclude_request_ids: Any, - exclude_started_at: datetime | None, - ) -> float | None: + batch_started_at: datetime | None, + ) -> WindowSeedTotals | None: self.calls.append( { "entity_type": entity_type, "entity_id": entity_id, "window_start": window_start, - "exclude_request_ids": tuple(exclude_request_ids), - "exclude_started_at": exclude_started_at, + "batch_started_at": batch_started_at, } ) - return self.value + return self.totals class _SpendLogsFake: """Sums the LiteLLM_SpendLogs rows (request_id, spend, startTime) it holds, - honouring the exclusion exactly as the real aggregate's - NOT (request_id = ANY(...) AND startTime >= bound) does.""" + splitting them at the batch start exactly as the real aggregate's + SUM(...) FILTER (WHERE startTime < bound) does.""" def __init__(self, rows: tuple[tuple[str, float, datetime], ...]) -> None: self.rows = rows @@ -111,25 +115,25 @@ class _SpendLogsFake: entity_type: str, entity_id: str, window_start: datetime, - exclude_request_ids: Any, - exclude_started_at: datetime | None, - ) -> float | None: - excluded = frozenset(exclude_request_ids) if exclude_started_at is not None else frozenset() - return math.fsum( - spend - for request_id, spend, started_at in self.rows - if not (request_id in excluded and started_at >= exclude_started_at) + batch_started_at: datetime | None, + ) -> WindowSeedTotals | None: + return WindowSeedTotals( + total=math.fsum(spend for _request_id, spend, _started_at in self.rows), + before_batch=math.fsum( + spend + for _request_id, spend, started_at in self.rows + if batch_started_at is None or started_at < batch_started_at + ), ) -def _batch(request_ids: tuple[str, ...], spend: float, started_at: datetime | None = BATCH_STARTED_AT) -> dict: +def _batch(spend: float, started_at: datetime | None = BATCH_STARTED_AT) -> dict: return { "entity_type": "key", "entity_id": "k1", "window_duration": "30d", "window_start": "2026-08-01T00:00:00.000000", "spend": spend, - "request_ids": request_ids, "started_at": None if started_at is None else started_at.replace(tzinfo=None).isoformat(timespec="microseconds"), @@ -156,7 +160,7 @@ async def test_missing_row_is_seeded_from_spend_logs_once(): existed, so a brand new primary key inserts the SpendLogs total plus this increment.""" db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=5.0) + aggregate = _RecordingAggregate(total=5.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -183,7 +187,7 @@ async def test_existing_row_is_never_reseeded(): """The seed is a full LiteLLM_SpendLogs scan; running it for a row that is already maintained would both cost a scan and double count.""" db = _FakeDB(existing_rows=[_existing("key", "k1", "30d")]) - aggregate = _RecordingAggregate(value=5.0) + aggregate = _RecordingAggregate(total=5.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -200,7 +204,7 @@ async def test_existing_row_is_never_reseeded(): @pytest.mark.asyncio async def test_seed_runs_only_for_the_primary_keys_that_are_missing(): db = _FakeDB(existing_rows=[_existing("key", "k1", "30d")]) - aggregate = _RecordingAggregate(value=5.0) + aggregate = _RecordingAggregate(total=5.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -223,7 +227,7 @@ async def test_insert_spend_and_increment_differ_only_when_a_row_is_seeded(): """The conflict arm adds the increment alone so two pods that both seed the same new window cannot add the SpendLogs base twice.""" db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=9.0) + aggregate = _RecordingAggregate(total=9.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -260,7 +264,7 @@ async def test_upsert_sql_adds_for_a_current_window_and_replaces_for_a_newer_one @pytest.mark.asyncio async def test_upsert_never_interpolates_values_into_the_sql(): db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=0.0) + aggregate = _RecordingAggregate(total=0.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -278,7 +282,7 @@ async def test_upserts_are_ordered_by_primary_key_then_window_start(): """Cross-pod lock ordering, plus an older window must be applied before the roll that supersedes it or the roll would be undone.""" db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=0.0) + aggregate = _RecordingAggregate(total=0.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -306,7 +310,7 @@ async def test_upserts_are_ordered_by_primary_key_then_window_start(): @pytest.mark.asyncio async def test_existing_row_lookup_sends_every_primary_key_as_array_params(): db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=0.0) + aggregate = _RecordingAggregate(total=0.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), @@ -344,9 +348,7 @@ async def test_unknown_entity_type_contributes_no_seed(): anything else starts from its increment alone.""" db = _FakeDB(existing_rows=[]) - async def no_such_column( - prisma_client, entity_type, entity_id, window_start, exclude_request_ids, exclude_started_at - ): + async def no_such_column(prisma_client, entity_type, entity_id, window_start, batch_started_at): return None await commit_window_spend_updates( @@ -363,7 +365,7 @@ async def test_unknown_entity_type_contributes_no_seed(): async def test_unavailable_spend_logs_aggregate_seeds_zero_rather_than_failing(): db = _FakeDB(existing_rows=[]) - async def unavailable(prisma_client, entity_type, entity_id, window_start, exclude_request_ids, exclude_started_at): + async def unavailable(prisma_client, entity_type, entity_id, window_start, batch_started_at): return None await commit_window_spend_updates( @@ -399,32 +401,31 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o @pytest.mark.asyncio -async def test_seed_receives_the_batch_request_ids_and_earliest_start_to_exclude(): +async def test_seed_receives_the_batch_earliest_start_as_its_cutoff(): db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=0.0) + aggregate = _RecordingAggregate(total=0.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("req-1", "req-2", "req-3"), 3.0),), + transactions=(_batch(3.0),), spend_logs_aggregate=aggregate, ) - assert aggregate.calls[0]["exclude_request_ids"] == ("req-1", "req-2", "req-3") - assert aggregate.calls[0]["exclude_started_at"] == BATCH_STARTED_AT + assert aggregate.calls[0]["batch_started_at"] == BATCH_STARTED_AT @pytest.mark.asyncio async def test_seed_passes_no_start_bound_when_the_batch_has_none(): db = _FakeDB(existing_rows=[]) - aggregate = _RecordingAggregate(value=0.0) + aggregate = _RecordingAggregate(total=0.0) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("req-1",), 1.0, started_at=None),), + transactions=(_batch(1.0, started_at=None),), spend_logs_aggregate=aggregate, ) - assert aggregate.calls[0]["exclude_started_at"] is None + assert aggregate.calls[0]["batch_started_at"] is None @pytest.mark.asyncio @@ -444,7 +445,7 @@ async def test_new_row_is_not_double_counted_when_the_batch_logs_already_flushed await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("req-1", "req-2", "req-3"), 0.000141),), + transactions=(_batch(0.000141),), spend_logs_aggregate=already_flushed, ) @@ -460,7 +461,7 @@ async def test_new_row_still_covers_spend_that_predates_the_batch(): await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("req-1",), 0.000047),), + transactions=(_batch(0.000047),), spend_logs_aggregate=spend_logs, ) @@ -469,22 +470,29 @@ async def test_new_row_still_covers_spend_that_predates_the_batch(): @pytest.mark.asyncio -async def test_replayed_request_id_cannot_erase_historical_spend_from_the_seed(): - """request_id can be chosen by the client via x-litellm-call-id. A request - that replays an id from before this batch writes no new LiteLLM_SpendLogs - row (the insert skips duplicates), so the seed must keep counting the - historical row that id belongs to; only its increment is new.""" +async def test_seed_keeps_spend_another_pod_persisted_after_this_batch_started(): + """A concurrent request on another pod can land its spend log after this + batch started but before this pod seeds the row. Dropping it on a plain + time cutoff would lose that spend for the rest of the window if that pod + died before flushing its increment, so the seed takes off only this batch's + own spend and keeps everything else.""" db = _FakeDB(existing_rows=[]) - spend_logs = _SpendLogsFake(rows=(("replayed", 0.5, BEFORE_BATCH),)) + spend_logs = _SpendLogsFake( + rows=( + ("older", 0.5, BEFORE_BATCH), + ("mine", 0.000047, BATCH_STARTED_AT), + ("other-pod", 0.25, BATCH_STARTED_AT + timedelta(seconds=1)), + ), + ) await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("replayed",), 0.000047),), + transactions=(_batch(0.000047),), spend_logs_aggregate=spend_logs, ) ((_, params),) = db.batcher.calls - assert params[INSERT_SPEND] == pytest.approx(0.500047) + assert params[INSERT_SPEND] == pytest.approx(0.750047) @pytest.mark.asyncio @@ -496,7 +504,7 @@ async def test_new_row_is_correct_when_the_batch_logs_have_not_flushed_yet(): await commit_window_spend_updates( prisma_client=_FakePrismaClient(db), - transactions=(_batch(("req-1", "req-2", "req-3"), 0.000141),), + transactions=(_batch(0.000141),), spend_logs_aggregate=nothing_flushed, ) @@ -509,57 +517,49 @@ async def test_new_row_is_correct_when_the_batch_logs_have_not_flushed_yet(): "entity_type, expected_column", [("key", "api_key = $1"), ("team", "team_id = $1")], ) -async def test_seed_aggregate_sql_excludes_the_request_ids_only_within_the_batch_start_bound( - entity_type, expected_column -): - db = _FakeDB(existing_rows=[{"total": 1.25}]) +async def test_seed_aggregate_sql_splits_the_window_at_the_batch_start(entity_type, expected_column): + db = _FakeDB(existing_rows=[{"total": 1.25, "before_batch": 0.75}]) - total = await spend_logs_total_excluding( + totals = await spend_logs_seed_totals( prisma_client=_FakePrismaClient(db), entity_type=entity_type, entity_id="e1", window_start=WINDOW_A, - exclude_request_ids=("req-1", "req-2"), - exclude_started_at=BATCH_STARTED_AT, + batch_started_at=BATCH_STARTED_AT, ) - assert total == pytest.approx(1.25) + assert totals == WindowSeedTotals(total=1.25, before_batch=0.75) ((query, params),) = db.query_raw_calls normalized = " ".join(query.split()) assert expected_column in normalized - assert "NOT (request_id = ANY($3::text[]) AND \"startTime\" >= ($4::timestamptz AT TIME ZONE 'UTC'))" in normalized + assert "FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC'))" in normalized assert 'FROM "LiteLLM_SpendLogs"' in normalized # startTime is TIMESTAMP(3): the bound is floored to the second so the # batch's own earliest row cannot round under it. - assert params == ("e1", WINDOW_A, ("req-1", "req-2"), datetime(2026, 8, 10, 12, 0, 0)) - # The ids are bound, never spliced into the statement. - assert "req-1" not in query + assert params == ("e1", WINDOW_A, datetime(2026, 8, 10, 12, 0, 0)) + # Nothing the caller supplied reaches the statement text. + assert "e1" not in query @pytest.mark.asyncio -@pytest.mark.parametrize( - "exclude_request_ids, exclude_started_at", - [(("req-1",), None), ((), BATCH_STARTED_AT)], -) -async def test_seed_aggregate_excludes_nothing_without_both_ids_and_a_start_bound( - exclude_request_ids, exclude_started_at -): - """Ids without a start bound would reopen the replayed-id hole, so the - seed counts everything instead; at worst that over-counts one batch.""" - db = _FakeDB(existing_rows=[{"total": 1.25}]) +async def test_seed_aggregate_sums_the_whole_window_without_a_start_bound(): + """A batch with no known start cannot place the split, so both halves are + the same sum and the seed counts everything; at worst that over-counts one + batch, which enforcement tolerates, where under-counting is a budget + bypass.""" + db = _FakeDB(existing_rows=[{"total": 1.25, "before_batch": 1.25}]) - total = await spend_logs_total_excluding( + totals = await spend_logs_seed_totals( prisma_client=_FakePrismaClient(db), entity_type="key", entity_id="e1", window_start=WINDOW_A, - exclude_request_ids=exclude_request_ids, - exclude_started_at=exclude_started_at, + batch_started_at=None, ) - assert total == pytest.approx(1.25) + assert totals == WindowSeedTotals(total=1.25, before_batch=1.25) ((query, params),) = db.query_raw_calls - assert "request_id" not in query + assert '"startTime" <' not in query assert params == ("e1", WINDOW_A) @@ -567,16 +567,15 @@ async def test_seed_aggregate_excludes_nothing_without_both_ids_and_a_start_boun async def test_seed_aggregate_returns_none_for_an_entity_type_with_no_spend_logs_column(): db = _FakeDB(existing_rows=[]) - total = await spend_logs_total_excluding( + totals = await spend_logs_seed_totals( prisma_client=_FakePrismaClient(db), entity_type="user", entity_id="u1", window_start=WINDOW_A, - exclude_request_ids=(), - exclude_started_at=None, + batch_started_at=None, ) - assert total is None + assert totals is None assert db.query_raw_calls == [] @@ -584,13 +583,12 @@ async def test_seed_aggregate_returns_none_for_an_entity_type_with_no_spend_logs async def test_seed_aggregate_treats_an_entity_with_no_rows_as_zero(): db = _FakeDB(existing_rows=[]) - total = await spend_logs_total_excluding( + totals = await spend_logs_seed_totals( prisma_client=_FakePrismaClient(db), entity_type="key", entity_id="k-unknown", window_start=WINDOW_A, - exclude_request_ids=(), - exclude_started_at=None, + batch_started_at=None, ) - assert total == 0.0 + assert totals == WindowSeedTotals(total=0.0, before_batch=0.0) diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index d28cf8c9c6a..11ef911de3e 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2581,7 +2581,6 @@ async def test_failed_window_spend_commit_requeues_the_increments_and_continues_ window_duration="30d", window_start=datetime(2026, 8, 1, tzinfo=timezone.utc), spend=0.5, - request_id="req-1", ) await db_writer.window_spend_update_queue.add_update(transaction) db = _WindowSpendFakeDB() @@ -2611,7 +2610,6 @@ async def test_failed_window_spend_commit_from_redis_is_restored_to_redis(): window_duration="7d", window_start=datetime(2026, 8, 1, tzinfo=timezone.utc), spend=2.0, - request_id="req-1", ), ) mock_redis_update_buffer = AsyncMock() @@ -2638,74 +2636,6 @@ async def test_failed_window_spend_commit_from_redis_is_restored_to_redis(): db_writer.pod_lock_manager.release_lock.assert_awaited_once() -@pytest.mark.asyncio -async def test_update_database_returns_the_spend_log_request_id(): - """The budget-window seed excludes the log rows its increments already - cover, so the caller needs the id this call was recorded under. It cannot - be re-derived: cache hits append time.time() to the id.""" - db_writer = DBSpendUpdateWriter() - db_writer._insert_spend_log_to_db = AsyncMock() - db_writer._enqueue_tool_usage_transaction = AsyncMock() - - with ( - patch.multiple( # test-quality-ok: update_database lazily imports these proxy_server globals; no injection seam - "litellm.proxy.proxy_server", - disable_spend_logs=False, - prisma_client=MagicMock(), - litellm_proxy_budget_name="test-budget", - ) - ): - request_id = await db_writer.update_database( - token="test-token", - user_id="test-user", - end_user_id=None, - team_id="test-team", - org_id=None, - kwargs={"model": "gpt-4", "custom_llm_provider": "openai", "litellm_call_id": "call-xyz"}, - completion_response=MagicMock(), - start_time=datetime.now(), - end_time=datetime.now(), - response_cost=0.1, - ) - await asyncio.sleep(0) - - assert request_id is not None - # Same id the spend log row was queued under. - assert request_id == db_writer._insert_spend_log_to_db.call_args[1]["payload"]["request_id"] - - -@pytest.mark.asyncio -async def test_update_database_returns_none_when_the_payload_cannot_be_built(): - db_writer = DBSpendUpdateWriter() - - with ( - patch.multiple( # test-quality-ok: update_database lazily imports these proxy_server globals; no injection seam - "litellm.proxy.proxy_server", - disable_spend_logs=False, - prisma_client=MagicMock(), - litellm_proxy_budget_name="test-budget", - ), - patch( # test-quality-ok: the payload builder is called by name inside update_database; no injection seam - "litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", - side_effect=Exception("payload boom"), - ), - ): - request_id = await db_writer.update_database( - token="test-token", - user_id="test-user", - end_user_id=None, - team_id="test-team", - org_id=None, - kwargs={"model": "gpt-4"}, - completion_response=MagicMock(), - start_time=datetime.now(), - end_time=datetime.now(), - response_cost=0.1, - ) - - assert request_id is None - - @pytest.mark.asyncio async def test_commit_spend_updates_to_db_does_not_stamp_key_settings_updated_at(): """Spend flushes must leave settings_updated_at alone, or it decays into diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py index b140082a3bf..f5d51a601d7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py @@ -432,7 +432,7 @@ class TestHiddenlayerGuardrail: @pytest.mark.asyncio async def test_apply_guardrail_request_with_image(self, monkeypatch: pytest.MonkeyPatch): - """Test apply_guardrail sends multimodal content (image) to HiddenLayer v1.""" + """Test apply_guardrail strips images from multimodal content before sending to HiddenLayer v1.""" monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( @@ -485,12 +485,13 @@ class TestHiddenlayerGuardrail: logging_obj=logging_obj, ) - # v1 API requires string content — multimodal list is stringified + # v1 API requires string content — image_url items are stripped and the + # remaining (text-only) content is stringified before being sent. mock_post.assert_called_once() call_kwargs = mock_post.call_args.kwargs sent_content = call_kwargs["json"]["input"]["messages"][0]["content"] assert isinstance(sent_content, str) - assert sent_content == str(multimodal_content) + assert sent_content == str([{"type": "text", "text": "how much is on this receipt?"}]) # Result should be returned without error assert result is not None diff --git a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py index 26beaa78a46..ab4e15ff423 100644 --- a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py @@ -1,16 +1,16 @@ -from fastapi.exceptions import HTTPException -from unittest.mock import patch, AsyncMock -from httpx import Response, Request +import asyncio import base64 +from unittest.mock import AsyncMock, patch import pytest - -from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import ( - PromptSecurityGuardrailMissingSecrets, - PromptSecurityGuardrail, -) +from fastapi.exceptions import HTTPException +from httpx import ReadTimeout, Request, Response import litellm +from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import ( + PromptSecurityGuardrail, + PromptSecurityGuardrailMissingSecrets, +) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 @@ -30,6 +30,7 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch): "guardrail": "prompt_security", "mode": "during_call", "default_on": True, + "file_sanitization_fail_open": False, }, } ], @@ -41,6 +42,10 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch): assert registered[0].guardrail_name == "prompt_security" assert registered[0].default_on is True assert registered[0].event_hook == "during_call" + assert registered[0].file_sanitization_fail_open is False + config_model = registered[0].get_config_model() + assert config_model is not None + assert config_model().file_sanitization_fail_open is True def test_prompt_security_guard_config_no_api_key(monkeypatch: pytest.MonkeyPatch): @@ -374,6 +379,86 @@ async def test_file_sanitization(monkeypatch: pytest.MonkeyPatch): assert result is not None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "timeout", + ( + litellm.Timeout( + message="Prompt Security upload timed out", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ), + ReadTimeout( + "Prompt Security poll timed out", + request=Request(method="GET", url="https://test.prompt.security/api/sanitizeFile"), + ), + ), + ids=("litellm", "httpx"), +) +@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed")) +async def test_file_sanitization_request_timeout_policy( + monkeypatch: pytest.MonkeyPatch, timeout: Exception, fail_open: bool +): + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") + + guardrail = PromptSecurityGuardrail( + guardrail_name="test-guard", + event_hook="pre_call", + default_on=True, + file_sanitization_fail_open=fail_open, + ) + + with patch.object(guardrail.async_handler, "post", AsyncMock(side_effect=timeout)): + if not fail_open: + with pytest.raises(HTTPException) as exc_info: + await guardrail.sanitize_file_content(b"file-content", "document.pdf") + assert exc_info.value.status_code == 408 + assert exc_info.value.detail == "File sanitization timeout" + return + + result = await guardrail.sanitize_file_content(b"file-content", "document.pdf") + + assert result == { + "action": "allow", + "content": None, + "metadata": {}, + "violations": (), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed")) +async def test_file_sanitization_overall_timeout_policy(monkeypatch: pytest.MonkeyPatch, fail_open: bool): + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") + + guardrail = PromptSecurityGuardrail( + guardrail_name="test-guard", + event_hook="pre_call", + default_on=True, + file_sanitization_timeout=0.01, + file_sanitization_fail_open=fail_open, + ) + + async def hanging_post(*_args: object, **_kwargs: object) -> None: + await asyncio.sleep(60) + raise AssertionError("sanitization request should have been cancelled") + + with patch.object(guardrail.async_handler, "post", side_effect=hanging_post): + if not fail_open: + with pytest.raises(HTTPException) as exc_info: + await guardrail.sanitize_file_content(b"file-content", "document.pdf") + assert exc_info.value.status_code == 408 + assert exc_info.value.detail == "File sanitization timeout" + return + + result = await guardrail.sanitize_file_content(b"file-content", "document.pdf") + + assert result["action"] == "allow" + assert result["content"] is None + + @pytest.mark.asyncio async def test_file_sanitization_block(monkeypatch: pytest.MonkeyPatch): """Test that file sanitization blocks malicious files""" @@ -544,7 +629,7 @@ async def test_role_filtering(monkeypatch: pytest.MonkeyPatch): return mock_response with patch.object(guardrail.async_handler, "post", side_effect=mock_post): - result = await guardrail.apply_guardrail( + await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, input_type="request", diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 2a540f4f522..8043a1aca3f 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -567,9 +567,7 @@ async def test_update_database_and_spend_counters_preserves_db_exception_when_re @pytest.mark.asyncio async def test_update_database_and_spend_counters_updates_counters_after_db_update(): proxy_logging_obj = MagicMock() - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( - return_value="chatcmpl-abc123" - ) + proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock() increment_spend_counters = AsyncMock() budget_reservation = {"reserved_cost": 0.5, "entries": []} start_time = datetime.now() @@ -602,7 +600,6 @@ async def test_update_database_and_spend_counters_updates_counters_after_db_upda budget_reservation=budget_reservation, end_user_id="test_end_user_id", tags=["tag-a"], - request_id="chatcmpl-abc123", request_started_at=start_time, model_access_groups=("premium",), ) @@ -1884,61 +1881,6 @@ async def test_track_cost_callback_logs_unauthenticated_pass_through_request( ) -@pytest.mark.asyncio -async def test_update_database_and_spend_counters_forwards_the_spend_log_request_id(): - """The budget-window flush excludes the log rows its increments already - cover. That only works if the id update_database recorded the row under is - handed to the counter update, so this seam is load-bearing.""" - proxy_logging_obj = MagicMock() - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( - return_value="chatcmpl-abc123" - ) - increment_spend_counters = AsyncMock() - - await _update_database_and_spend_counters( - proxy_logging_obj=proxy_logging_obj, - increment_spend_counters=increment_spend_counters, - user_api_key="test_api_key", - user_id="test_user_id", - end_user_id=None, - team_id="test_team_id", - org_id="test_org_id", - kwargs={}, - completion_response=None, - start_time=datetime.now(), - end_time=datetime.now(), - response_cost=0.2, - budget_reservation=None, - ) - - assert increment_spend_counters.await_args.kwargs["request_id"] == "chatcmpl-abc123" - - -@pytest.mark.asyncio -async def test_update_database_and_spend_counters_forwards_a_missing_request_id_as_none(): - proxy_logging_obj = MagicMock() - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(return_value=None) - increment_spend_counters = AsyncMock() - - await _update_database_and_spend_counters( - proxy_logging_obj=proxy_logging_obj, - increment_spend_counters=increment_spend_counters, - user_api_key="test_api_key", - user_id="test_user_id", - end_user_id=None, - team_id="test_team_id", - org_id="test_org_id", - kwargs={}, - completion_response=None, - start_time=datetime.now(), - end_time=datetime.now(), - response_cost=0.2, - budget_reservation=None, - ) - - assert increment_spend_counters.await_args.kwargs["request_id"] is None - - class _FakeDeploymentLookup: """Deployment lookup returning the access groups each deployment declares.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 726e09f3162..a74aa553449 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -878,7 +878,8 @@ def _leg_record(**overrides: object) -> MagicMock: defaults = { "id": "leg-1", "group_id": "job-1", - "api_key_id": "key-hash", + "target_type": "key", + "target_id": "key-hash", "router_name": "my-router", "direction": "forward", "baseline_model": None, @@ -912,8 +913,28 @@ def _key_record( return record +def _team_record(team_id: str, team_alias: str | None) -> MagicMock: + record = MagicMock(spec=["team_id", "team_alias"]) + record.team_id = team_id + record.team_alias = team_alias + return record + + +def _user_record(user_id: str, user_email: str | None) -> MagicMock: + record = MagicMock(spec=["user_id", "user_email"]) + record.user_id = user_id + record.user_email = user_email + return record + + def _shadow_prisma( - legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2"), key_teams=None + legs=(), + agg_rows=None, + by_leg_rows=None, + known_keys=("key-hash", "key-hash-2"), + key_teams=None, + known_teams=None, + known_users=None, ) -> MagicMock: """The job-table fake honours the filters it is handed, so a read that forgets stopped_at sees rows the partial index would have released, one that forgets @@ -921,6 +942,8 @@ def _shadow_prisma( group read that matched on a leg id would come back empty.""" prisma = MagicMock() teams: Final = key_teams or {} + team_aliases: Final = known_teams or {} + user_emails: Final = known_users or {} async def find_tokens(*, where): """Honours the token filter, like the job-table fake below: the endpoint derives the @@ -931,6 +954,17 @@ def _shadow_prisma( prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=find_tokens) + async def find_teams(*, where): + requested = where["team_id"]["in"] + return [_team_record(t, alias) for t, alias in team_aliases.items() if t in requested] + + async def find_users(*, where): + requested = where["user_id"]["in"] + return [_user_record(u, email) for u, email in user_emails.items() if u in requested] + + prisma.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_teams) + prisma.db.litellm_usertable.find_many = AsyncMock(side_effect=find_users) + async def execute_raw(sql: str, *params: object): if "SET stopped_by" in sql: group = [row for row in stored if row.group_id == params[0]] @@ -959,9 +993,19 @@ def _shadow_prisma( async def find_many_legs(where=None, **_: object): current = list(stored) w = dict(where or {}) - if "api_key_id" in w: - wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]] - current = [row for row in current if row.api_key_id in wanted] + if "OR" in w: + pairs = [ + ( + branch["target_type"], + branch["target_id"]["in"] if isinstance(branch["target_id"], dict) else [branch["target_id"]], + ) + for branch in w["OR"] + ] + current = [ + row + for row in current + if any(row.target_type == target_type and row.target_id in ids for target_type, ids in pairs) + ] if "direction" in w: current = [row for row in current if row.direction == w["direction"]] if "stopped_at" in w: @@ -983,7 +1027,8 @@ def _shadow_prisma( fields = ( "id", "group_id", - "api_key_id", + "target_type", + "target_id", "router_name", "direction", "baseline_model", @@ -1009,7 +1054,11 @@ def _shadow_prisma( if "AS attempt_count" in sql: return prisma.attempt_rows if "GROUP BY group_id" in sql: - scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]] + scoped = [ + row + for row in stored + if "target_type = $2" not in sql or (row.target_type == params[1] and row.target_id == params[2]) + ] keep = set(newest_groups(scoped, params[0])) return [leg_dict(row) for row in stored if row.group_id in keep] if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: @@ -1058,7 +1107,7 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) - sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args + sweep_sql, sweep_ids, sweep_type = prisma.db.execute_raw.call_args.args assert "stopped_at IS NULL" in sweep_sql assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql @@ -1066,12 +1115,13 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert "j.max_budget IS NOT NULL" in sweep_sql assert ">= j.max_budget" in sweep_sql assert "SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost)" in sweep_sql - assert "j.api_key_id = ANY($1::text[])" in sweep_sql - assert sweep_keys == ["key-hash", "key-hash-2"] + assert "j.target_type = $2 AND j.target_id = ANY($1::text[])" in sweep_sql + assert sweep_ids == ["key-hash", "key-hash-2"] + assert sweep_type == "key" prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] - assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] - assert len({frozenset((k, v) for k, v in row.items() if k not in ("api_key_id", "id")) for row in rows}) == 1 + assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")] + assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1 assert len({row["id"] for row in rows}) == len(rows) assert len({row["group_id"] for row in rows}) == 1 assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows) @@ -1080,11 +1130,12 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert response.job_id == rows[0]["group_id"] assert response.status == "running" assert response.judged_count is None - assert [(key.api_key_id, key.max_budget, key.key_alias) for key in response.keys] == [ + assert [(target.target_id, target.max_budget, target.target_alias) for target in response.targets] == [ ("key-hash", 5.0, "prod-alpha"), ("key-hash-2", 5.0, "prod-alpha"), ] - assert all(key.max_turns == SHADOW_EVAL_TURN_VALVE for key in response.keys) + assert all(target.target_type == "key" for target in response.targets) + assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets) @pytest.mark.asyncio @@ -1232,7 +1283,7 @@ async def test_start_shadow_eval_rejections( import litellm.proxy.proxy_server as proxy_server _configure_anthropic_sdk_judge(monkeypatch) - prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed]) + prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", target_id=key) for key in claimed]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) @@ -1315,14 +1366,14 @@ async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pyt import litellm.proxy.proxy_server as proxy_server _configure_anthropic_sdk_judge(monkeypatch) - prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")]) + prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", target_id="key-hash-2")]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) with pytest.raises(HTTPException) as exc: await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) assert exc.value.status_code == 409 - assert "key-hash-2 (job job-7)" in exc.value.detail + assert "key key-hash-2 (job job-7)" in exc.value.detail @pytest.mark.asyncio @@ -1441,6 +1492,180 @@ def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set(): _start_request(api_key_ids=tuple(f"k{i}" for i in range(101))) +def test_start_request_bounds_the_combined_target_count_across_types(): + """The 1..100 bound counts keys, teams, and users together, so a caller cannot dodge + it by spreading targets over the three fields, and a request naming no target of any + type samples nothing and is rejected.""" + with pytest.raises(ValidationError, match="at least one target"): + _start_request(api_key_ids=(), team_ids=(), user_ids=()) + with pytest.raises(ValidationError, match="at most 100 targets"): + _start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(41))) + mixed = _start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(40))) + assert len(mixed.api_key_ids) + len(mixed.team_ids) == 100 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides,prisma_kwargs,expected_target", + [ + ( + {"api_key_ids": (), "team_ids": ("team-eng",)}, + {"known_teams": {"team-eng": "Engineering"}}, + ("team", "team-eng", "Engineering"), + ), + ( + {"api_key_ids": (), "user_ids": ("dev-alice",)}, + {"known_users": {"dev-alice": "alice@example.com"}}, + ("user", "dev-alice", "alice@example.com"), + ), + ], + ids=["team-target-labeled-by-team-alias", "user-target-labeled-by-user-email"], +) +async def test_start_shadow_eval_creates_typed_legs_for_team_and_user_targets( + monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_target +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(**prisma_kwargs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(**overrides), ADMIN) + + target_type, target_id, target_alias = expected_target + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [(row["target_type"], row["target_id"]) for row in rows] == [(target_type, target_id)] + assert response.status == "running" + target = response.targets[0] + assert (target.target_type, target.target_id, target.target_alias, target.key_name) == ( + target_type, + target_id, + target_alias, + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides,prisma_kwargs,expected_detail", + [ + ( + {"api_key_ids": (), "team_ids": ("team-eng", "team-ghost")}, + {"known_teams": {"team-eng": "Engineering"}}, + "team_ids not on this proxy: team-ghost", + ), + ( + {"api_key_ids": (), "user_ids": ("dev-alice", "dev-ghost")}, + {"known_users": {"dev-alice": "alice@example.com"}}, + "user_ids not on this proxy: dev-ghost", + ), + ], + ids=["unknown-team", "unknown-user"], +) +async def test_start_shadow_eval_rejects_teams_and_users_this_proxy_does_not_know( + monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_detail +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(**prisma_kwargs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(**overrides), ADMIN) + assert exc.value.status_code == 400 + assert expected_detail in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_mixed_targets_create_both_legs_and_sweep_once_per_type( + monkeypatch: pytest.MonkeyPatch, +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(known_teams={"team-eng": "Engineering"}) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(team_ids=("team-eng",)), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("team", "team-eng")] + assert len({row["group_id"] for row in rows}) == 1 + sweeps = [ + call.args + for call in prisma.db.execute_raw.await_args_list + if "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in call.args[0] + ] + assert [(ids, target_type) for _, ids, target_type in sweeps] == [(["key-hash"], "key"), (["team-eng"], "team")] + assert [(t.target_type, t.target_id, t.target_alias) for t in response.targets] == [ + ("key", "key-hash", "prod-alpha"), + ("team", "team-eng", "Engineering"), + ] + + +@pytest.mark.asyncio +async def test_start_shadow_eval_names_the_busy_team_target(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma( + legs=[_leg_record(id="leg-t", group_id="job-7", target_type="team", target_id="team-eng")], + known_teams={"team-eng": "Engineering"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN) + assert exc.value.status_code == 409 + assert "team team-eng (job job-7)" in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_claim_matches_exact_target_pairs_not_bare_ids(monkeypatch: pytest.MonkeyPatch): + """A key whose hash happens to spell a team's id must not hold the team's slot: the + claim matches (target_type, target_id) pairs, never ids across kinds.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma( + legs=[_leg_record(id="leg-k", group_id="job-7", target_type="key", target_id="team-eng")], + known_teams={"team-eng": "Engineering"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN) + + assert response.status == "running" + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_list_shadow_eval_jobs_rejects_a_lone_filter_half(monkeypatch: pytest.MonkeyPatch): + """target_type and target_id only mean anything together: a bare id could name a key + or a team, and a bare type filters nothing.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record()]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as id_only: + await list_shadow_eval_jobs(VIEWER, target_type=None, target_id="key-hash", limit=50) + assert id_only.value.status_code == 400 + + with pytest.raises(HTTPException) as type_only: + await list_shadow_eval_jobs(VIEWER, target_type="key", target_id=None, limit=50) + assert type_only.value.status_code == 400 + prisma.db.query_raw.assert_not_called() + + @pytest.mark.asyncio async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server @@ -1530,7 +1755,7 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", max_turns=50)], agg_rows=tier_rows, by_leg_rows=leg_rows, ) @@ -1549,8 +1774,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 assert response.results.overall_shadow_win_rate_pct == 40.0 assert response.results.overall_tie_rate_pct == 20.0 - assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)] - assert response.results.by_key[0].shadow_win_rate_pct == 66.7 + verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets} + assert verdicts_by_target[("key", "key-hash")].turn_count == 6 + assert verdicts_by_target[("key", "key-hash")].shadow_win_rate_pct == 66.7 + assert verdicts_by_target[("key", "key-hash-2")].turn_count == 4 agg_sql = next(call.args[0] for call in prisma.db.query_raw.await_args_list if "real_spend" in call.args[0]) assert agg_sql.count("FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit)") == 2 assert response.results.by_tier[0].real_spend == 0.08 @@ -1561,7 +1788,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert response.results.not_sampled_count is None assert response.results.unjudgeable_count is None assert response.results.shed_count is None - assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)] + assert [(target.target_id, target.max_turns) for target in response.targets] == [ + ("key-hash", 200), + ("key-hash-2", 50), + ] totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]] assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])] error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"] @@ -1595,7 +1825,7 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke _leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)), _leg_record( id="leg-2", - api_key_id="key-hash-2", + target_id="key-hash-2", stopped_at=stamp, created_at=datetime(2026, 8, 13, tzinfo=timezone.utc), ), @@ -1615,14 +1845,14 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert [(job.job_id, job.status) for job in jobs] == [ ("job-1", "running"), ("job-2", "stopped"), ("job-3", "completed"), ] - assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"] assert all(job.judged_count is None and job.results is None for job in jobs) legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql @@ -1648,17 +1878,20 @@ async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypa prisma = _shadow_prisma( legs=[ _leg_record(), - _leg_record(id="leg-2", api_key_id="key-hash-2"), - _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"), + _leg_record(id="leg-2", target_id="key-hash-2"), + _leg_record(id="leg-3", group_id="job-2", target_id="key-hash-2"), _leg_record(id="leg-4", group_id="job-3"), ] ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type="key", target_id="key-hash-2", limit=50) assert [job.job_id for job in jobs] == ["job-1", "job-2"] - assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"] + legs_sql, *legs_params = prisma.db.query_raw.await_args_list[0].args + assert "WHERE target_type = $2 AND target_id = $3" in legs_sql + assert legs_params == [50, "key", "key-hash-2"] @pytest.mark.parametrize( @@ -1682,7 +1915,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop legs=[ _leg_record( id=f"leg-{index}", - api_key_id=f"key-{index}", + target_id=f"key-{index}", stopped_at=stamp if stopped else None, ends_at=datetime.now(timezone.utc) + timedelta(days=days_left), ) @@ -1691,7 +1924,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert [job.status for job in jobs] == [expected] @@ -1707,9 +1940,9 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch prisma = _shadow_prisma( legs=[ _leg_record(max_turns=5), - _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5), - _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5), - _leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5), + _leg_record(id="leg-2", target_id="key-hash-2", max_turns=5), + _leg_record(id="leg-3", group_id="job-2", target_id="key-hash", max_turns=5), + _leg_record(id="leg-4", group_id="job-2", target_id="key-hash-2", max_turns=5), ] ) prisma.attempt_rows = [ @@ -1720,13 +1953,13 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch ] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) by_id = {job.job_id: job for job in jobs} assert by_id["job-1"].status == "completed" - assert all(key.stopped_at is None for key in by_id["job-1"].keys) + assert all(target.stopped_at is None for target in by_id["job-1"].targets) assert by_id["job-2"].status == "running" - assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3} + assert {t.target_id: t.attempt_count for t in by_id["job-2"].targets} == {"key-hash": 5, "key-hash-2": 3} @pytest.mark.asyncio @@ -1740,7 +1973,7 @@ async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: py prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "stopped" assert jobs[0].stopped_by == "admin" @@ -1760,7 +1993,7 @@ async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pyt prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "stopped" @@ -1808,6 +2041,56 @@ def test_max_budget_migration_is_additive_and_leaves_legacy_rows_null(): @pytest.mark.asyncio +@pytest.mark.asyncio +async def test_verdicts_keep_same_id_targets_of_different_kinds_distinct(monkeypatch): + """A team and a user can legitimately share an id; their slices must not merge.""" + from litellm.proxy import proxy_server + + leg_rows = [ + { + "grp": "leg-1", + "turn_count": 6, + "real_wins": 2, + "shadow_wins": 4, + "ties": 0, + "avg_confidence": 0.8, + "real_spend": 0.02, + "shadow_spend": 0.01, + "cache_hit_turns": 0, + }, + { + "grp": "leg-2", + "turn_count": 4, + "real_wins": 3, + "shadow_wins": 0, + "ties": 1, + "avg_confidence": 0.6, + "real_spend": 0.05, + "shadow_spend": 0.04, + "cache_hit_turns": 1, + }, + ] + prisma = _shadow_prisma( + legs=[ + _leg_record(target_type="team", target_id="dev-alice"), + _leg_record(id="leg-2", target_type="user", target_id="dev-alice"), + ], + agg_rows=leg_rows[:1], + by_leg_rows=leg_rows, + known_teams={"dev-alice": "alias"}, + known_users={"dev-alice": "alice@example.com"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets} + assert verdicts_by_target[("team", "dev-alice")].turn_count == 6 + assert verdicts_by_target[("team", "dev-alice")].shadow_win_rate_pct == 66.7 + assert verdicts_by_target[("user", "dev-alice")].turn_count == 4 + assert verdicts_by_target[("user", "dev-alice")].shadow_win_rate_pct == 0.0 + + async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server @@ -1832,9 +2115,9 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk prisma = _shadow_prisma( legs=[ _leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), - _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), + _leg_record(id="leg-2", target_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), _leg_record( - id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0 + id="leg-3", group_id="job-2", target_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0 ), ] ) @@ -1845,13 +2128,13 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk ] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) by_id = {job.job_id: job for job in jobs} assert by_id["job-1"].status == "completed" assert by_id["job-2"].status == "running" - assert {key.api_key_id: key.spend for key in by_id["job-1"].keys} == {"key-hash": 1.0, "key-hash-2": 1.25} - assert all(key.max_budget == 1.0 for key in by_id["job-1"].keys) + assert {t.target_id: t.spend for t in by_id["job-1"].targets} == {"key-hash": 1.0, "key-hash-2": 1.25} + assert all(target.max_budget == 1.0 for target in by_id["job-1"].targets) @pytest.mark.asyncio @@ -1880,11 +2163,11 @@ async def test_legacy_jobs_without_a_dollar_budget_stay_turn_gated(monkeypatch: prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 40, "spend": 250.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "running" - assert jobs[0].keys[0].max_budget is None - assert jobs[0].keys[0].spend == 250.0 + assert jobs[0].targets[0].max_budget is None + assert jobs[0].targets[0].spend == 250.0 @pytest.mark.asyncio @@ -1892,14 +2175,14 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="deleted-key-hash")], known_keys=("key-hash", "key-hash-2"), ) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [ + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) + assert [(target.target_alias, target.key_name) for target in jobs[0].targets] == [ (None, None), ("prod-alpha", "sk-...lpha"), ] @@ -1907,7 +2190,7 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}} detail = await get_shadow_eval_job("job-1", VIEWER) - assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"] + assert [target.target_alias for target in detail.targets] == [None, "prod-alpha"] @pytest.mark.asyncio @@ -1919,7 +2202,7 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin import litellm.proxy.proxy_server as proxy_server earned = datetime.now(timezone.utc) - timedelta(hours=1) - prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)]) + prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", stopped_at=earned)]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) stopped = await stop_shadow_eval_job("job-1", ADMIN) @@ -1938,9 +2221,9 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin assert datetime.fromisoformat(stop_stamp).tzinfo is None assert prisma.db.execute_raw.await_count == 1 prisma.db.litellm_shadowevaljob.update_many.assert_not_called() - by_key = {key.api_key_id: key.stopped_at for key in stopped.keys} - assert by_key["key-hash-2"] == earned - assert by_key["key-hash"] is not None and by_key["key-hash"] != earned + by_target = {target.target_id: target.stopped_at for target in stopped.targets} + assert by_target["key-hash-2"] == earned + assert by_target["key-hash"] is not None and by_target["key-hash"] != earned done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) prisma_done = _shadow_prisma(legs=[done_leg]) @@ -2331,7 +2614,7 @@ async def test_get_shadow_eval_job_sums_funnel_rows_across_legs(monkeypatch: pyt }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")], agg_rows=tier_rows, ) prisma.funnel_rows = [{"legs_with_rows": 2, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 3}] @@ -2366,7 +2649,7 @@ async def test_partially_seeded_funnel_reads_as_unknown_coverage(monkeypatch: py }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")], agg_rows=tier_rows, ) prisma.funnel_rows = [{"legs_with_rows": 1, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 0}] diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py index 5c163c44cb3..1225cb80224 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -83,6 +83,50 @@ def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth): assert response.json()["alias"] == "Updated Test User" +def test_update_customer_unblock(mock_prisma_client, mock_user_api_key_auth): + mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=False) + + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user) + mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user) + + response = client.post( + "/customer/update", + json={"user_id": "test-user-1", "blocked": False}, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + assert response.json()["blocked"] is False + update_mock = mock_prisma_client.db.litellm_endusertable.update + update_mock.assert_called_once() + assert update_mock.call_args.kwargs["data"]["blocked"] is False + + +def test_update_customer_keeps_blocked_when_omitted(mock_prisma_client, mock_user_api_key_auth): + """ + Regression test: updating a blocked customer without supplying `blocked` + must NOT reset it to unblocked. `blocked=False` is the model default and + should only be applied when explicitly provided by the caller. + """ + mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True) + + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user) + mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user) + + response = client.post( + "/customer/update", + json={"user_id": "test-user-1", "alias": "Updated Test User"}, + headers={"Authorization": "Bearer test-key"}, + ) + + assert response.status_code == 200 + update_mock = mock_prisma_client.db.litellm_endusertable.update + update_mock.assert_called_once() + assert "blocked" not in update_mock.call_args.kwargs["data"] + + def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth): """ Test that update_end_user raises a 404 ProxyException when user_id does not exist. diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 42e56ceabd3..1c80aa5683f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -8312,15 +8312,17 @@ async def test_key_does_not_override_explicit_budget_duration(): @patch( "litellm.proxy.management_endpoints.key_management_endpoints.rotate_mcp_server_credentials_master_key" ) -async def test_rotate_master_key_model_data_valid_for_prisma( +async def test_rotate_master_key_reencrypts_model_params_in_place( mock_rotate_mcp, ): """ - Test that _rotate_master_key produces valid data for Prisma create_many(). - - Regression test for: master key rotation fails with Prisma validation error - because created_at/updated_at are None (non-nullable DateTime) and - litellm_params/model_info are JSON strings (create_many expects dicts). + Regression test for: master key rotation wipes every non-credential column + on LiteLLM_ProxyModelTable. Rotation used to rebuild the table via + delete_many + create_many from Deployment objects, which carry no + blocked/created_at/created_by/updated_at/updated_by, so every rotation + reset blocked to False (silently unblocking blocked models) and rewrote the + audit columns. Rotation must instead update only litellm_params (the sole + encrypted column) on each existing row, keyed by model_id. """ from unittest.mock import AsyncMock, MagicMock @@ -8352,6 +8354,7 @@ async def test_rotate_master_key_model_data_valid_for_prisma( mock_tx.litellm_proxymodeltable = MagicMock() mock_tx.litellm_proxymodeltable.delete_many = AsyncMock() mock_tx.litellm_proxymodeltable.create_many = AsyncMock() + mock_tx.litellm_proxymodeltable.update_many = AsyncMock() mock_prisma_client.db.tx = MagicMock( return_value=AsyncMock( __aenter__=AsyncMock(return_value=mock_tx), @@ -8400,36 +8403,33 @@ async def test_rotate_master_key_model_data_valid_for_prisma( new_master_key="sk-new-master-key", ) - # Verify create_many was called - mock_tx.litellm_proxymodeltable.create_many.assert_called_once() + # Rotation must never rewrite whole rows: no delete + recreate + mock_tx.litellm_proxymodeltable.delete_many.assert_not_called() + mock_tx.litellm_proxymodeltable.create_many.assert_not_called() - # Get the data passed to create_many - call_args = mock_tx.litellm_proxymodeltable.create_many.call_args - created_models = call_args.kwargs.get("data") or call_args[1].get("data") + mock_tx.litellm_proxymodeltable.update_many.assert_called_once() + call_args = mock_tx.litellm_proxymodeltable.update_many.call_args - assert len(created_models) == 1 - model_data = created_models[0] + assert call_args.kwargs["where"] == { + "model_id": "model-1" + }, "the re-encrypted params must land on the same row, keyed by model_id" - # Verify timestamps are NOT present (Prisma @default(now()) should apply) - assert ( - "created_at" not in model_data - ), "created_at should be excluded so Prisma @default(now()) applies" - assert ( - "updated_at" not in model_data - ), "updated_at should be excluded so Prisma @default(now()) applies" + update_data = call_args.kwargs["data"] + assert set(update_data.keys()) == {"litellm_params"}, ( + "rotation must touch only the encrypted litellm_params column; writing any " + f"other column wipes it (blocked, audit columns), got {sorted(update_data.keys())}" + ) - # Verify litellm_params and model_info are prisma.Json wrappers, NOT JSON strings import prisma assert isinstance( - model_data["litellm_params"], prisma.Json - ), f"litellm_params should be prisma.Json for create_many(), got {type(model_data['litellm_params'])}" - assert isinstance( - model_data["model_info"], prisma.Json - ), f"model_info should be prisma.Json for create_many(), got {type(model_data['model_info'])}" - - # Verify delete_many was called inside the transaction (before create_many) - mock_tx.litellm_proxymodeltable.delete_many.assert_called_once() + update_data["litellm_params"], prisma.Json + ), f"litellm_params should be prisma.Json for update_many(), got {type(update_data['litellm_params'])}" + reencrypted_params = update_data["litellm_params"].data + assert set(reencrypted_params.keys()) >= {"model", "api_key"} + assert ( + reencrypted_params["api_key"] != "sk-decrypted-key" + ), "api_key must be stored re-encrypted under the new master key, not in plaintext" async def test_default_key_generate_params_duration(monkeypatch): diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index dc9fede1f65..393953ccf68 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -4466,3 +4466,65 @@ class TestEnforceRpmTpmOnModelAdd: _raise_if_rate_limits_required_but_missing(litellm_params=params, enforced=True) assert expected_missing in str(exc_info.value.message) assert exc_info.value.code == "400" + + +class TestBlockModelResponseSerialization: + @pytest.mark.parametrize( + ("route", "blocked"), [("/model/block", True), ("/model/unblock", False)] + ) + def test_block_routes_serialize_prisma_row_to_200(self, route, blocked): + from datetime import datetime, timezone + + from prisma import models as prisma_models + + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import app + + written_at = datetime(2026, 8, 29, tzinfo=timezone.utc) + row_fields = { + "model_id": "m-block-1", + "model_name": "gpt-4o-mini", + "litellm_params": json.dumps({"model": "openai/gpt-4o-mini", "api_key": "encrypted-value"}), + "model_info": json.dumps({"id": "m-block-1"}), + "created_at": written_at, + "created_by": "admin", + "updated_at": written_at, + "updated_by": "admin", + } + existing_row = prisma_models.LiteLLM_ProxyModelTable(blocked=not blocked, **row_fields) + updated_row = prisma_models.LiteLLM_ProxyModelTable(blocked=blocked, **row_fields) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + mock_prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row) + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + app.dependency_overrides[ps.user_api_key_auth] = lambda: admin + try: + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point + "litellm.proxy.proxy_server.llm_router", + MagicMock(**{"get_model_ids.return_value": ["m-block-1"]}), + ), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( # test-quality-ok: stubs the cache write so the test observes only response serialization + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + patch( # test-quality-ok: audit logging is a background side effect outside this test's contract + "litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log", + new=AsyncMock(return_value=None), + ), + ): + client = TestClient(app) + response = client.post(route, json={"model_id": "m-block-1"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["model_id"] == "m-block-1" + assert body["blocked"] is blocked + assert body["litellm_params"] == {"model": "openai/gpt-4o-mini", "api_key": "encrypted-value"} diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 09bab1dc416..1d4b0264879 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -12,11 +12,13 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest from fastapi import HTTPException, Request, Response +from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient from starlette.datastructures import FormData import litellm +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, @@ -30,6 +32,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( get_azure_ai_search_index_from_endpoint, get_vertex_base_url, is_azure_ai_search_service_level_index_create, + gigachat_proxy_route, llm_passthrough_factory_proxy_route, milvus_proxy_route, mistral_proxy_route, @@ -178,7 +181,7 @@ class TestBaseOpenAIPassThroughHandler: assert result["api-key"] == "test_api_key" assert result["test-header"] == "value" - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" ) async def test_base_openai_pass_through_handler(self, mock_create_pass_through): @@ -2022,15 +2025,15 @@ class TestLLMPassthroughFactoryProxyRoute: class TestVLLMProxyRoute: @pytest.mark.asyncio - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body", return_value={"model": "router-model", "stream": False}, ) - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", return_value=True, ) - @patch("litellm.proxy.proxy_server.llm_router") + @patch("litellm.proxy.proxy_server.llm_router") # test-quality-ok: patching litellm internal for unit test isolation async def test_vllm_proxy_route_with_router_model( self, mock_llm_router, mock_is_router, mock_get_body ): @@ -2055,15 +2058,15 @@ class TestVLLMProxyRoute: mock_llm_router.allm_passthrough_route.assert_awaited_once() @pytest.mark.asyncio - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body", return_value={"model": "other-model"}, ) - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", return_value=False, ) - @patch( + @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.llm_passthrough_factory_proxy_route" ) async def test_vllm_proxy_route_fallback_to_factory( @@ -2085,6 +2088,312 @@ class TestVLLMProxyRoute: mock_factory_route.assert_awaited_once() +class TestGigachatProxyRoute: + @pytest.mark.asyncio + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body", + return_value={"model": "router-model", "stream": False}, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", + return_value=True, + ) + @patch("litellm.proxy.proxy_server.llm_router") # test-quality-ok: patching litellm internal for unit test isolation + async def test_gigachat_proxy_route_with_router_model( + self, mock_llm_router, mock_is_router, mock_get_body + ): + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_fastapi_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + mock_llm_router.allm_passthrough_route = AsyncMock( + return_value=httpx.Response(200, json={"response": "success"}) + ) + + result = await gigachat_proxy_route( + endpoint="/chat/completions", + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + mock_is_router.assert_called_once() + mock_llm_router.allm_passthrough_route.assert_awaited_once() + assert isinstance(result, Response) + + @pytest.mark.asyncio + async def test_gigachat_router_handler_keeps_cached_body_and_payload_metadata_pristine(self): + """Regression: auth-metadata injection must not leak into the cached parsed body or the upstream payload.""" + from litellm.proxy.common_utils.http_parsing_utils import get_request_body + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_gigachat_passthrough_router_model, + ) + + body = json.dumps( + { + "model": "gigachat-router", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"client_tag": "user-supplied"}, + } + ).encode() + scope = { + "type": "http", + "method": "POST", + "headers": [(b"content-type", b"application/json")], + "query_string": b"", + "path": "/gigachat/chat/completions", + } + + async def receive(): + return {"type": "http.request", "body": body, "more_body": False} + + request = Request(scope, receive) + request_body = await get_request_body(request) + + captured: dict = {} + + class _CapturingProcessor: + def __init__(self, data: dict): + captured["data"] = data + + async def base_passthrough_process_llm_request(self, **kwargs): + return Response(content=b"{}", status_code=200) + + with patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + _CapturingProcessor, + ): + await handle_gigachat_passthrough_router_model( + model="gigachat-router", + endpoint="/chat/completions", + request=request, + request_body=request_body, + fastapi_response=Response(), + llm_router=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(user_id="user-1", team_id="team-1"), + proxy_logging_obj=MagicMock(), + general_settings={}, + proxy_config=MagicMock(), + select_data_generator=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + data = captured["data"] + assert data["json"] is request_body + assert request_body["metadata"] == {"client_tag": "user-supplied"} + assert data["metadata"]["client_tag"] == "user-supplied" + assert data["metadata"]["user_api_key_user_id"] == "user-1" + assert data["metadata"]["user_api_key_team_id"] == "team-1" + cached_reread = await get_request_body(request) + assert cached_reread["metadata"] == {"client_tag": "user-supplied"} + + @pytest.mark.asyncio + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body", + return_value={"model": "other-model"}, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", + return_value=False, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn", + new_callable=AsyncMock, + return_value=False, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.authenticator.get_access_token", + return_value="gigachat-test-token", + ) + async def test_gigachat_proxy_route_fallback_forwards_to_gigachat_api( + self, + mock_get_token, + mock_is_streaming, + mock_is_router, + mock_get_body, + monkeypatch, + ): + monkeypatch.delenv("GIGACHAT_API_BASE", raising=False) + mock_request = MagicMock(spec=Request) + mock_fastapi_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + captured_kwargs = {} + + async def fake_endpoint(request, fastapi_response, user_api_key_dict): + return Response(content=b'{"response": "success"}', status_code=200) + + def fake_create_pass_through_route(**kwargs): + captured_kwargs.update(kwargs) + return fake_endpoint + + with patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + side_effect=fake_create_pass_through_route, + ): + result = await gigachat_proxy_route( + endpoint="/chat/completions", + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + assert isinstance(result, Response) + assert result.status_code == 200 + assert captured_kwargs["target"] == "https://gigachat.devices.sberbank.ru/api/v1/chat/completions" + assert captured_kwargs["custom_headers"] == {"Authorization": "Bearer gigachat-test-token"} + + @pytest.mark.asyncio + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.get_request_body", + return_value={}, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn", + new_callable=AsyncMock, + return_value=False, + ) + @patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.llms.gigachat.authenticator.get_access_token", + return_value="gigachat-test-token", + ) + async def test_gigachat_proxy_route_models_endpoint_without_model( + self, + mock_get_token, + mock_is_streaming, + mock_get_body, + monkeypatch, + ): + monkeypatch.delenv("GIGACHAT_API_BASE", raising=False) + mock_request = MagicMock(spec=Request) + mock_fastapi_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + captured_kwargs = {} + + async def fake_endpoint(request, fastapi_response, user_api_key_dict): + return Response(content=b'{"data": []}', status_code=200) + + def fake_create_pass_through_route(**kwargs): + captured_kwargs.update(kwargs) + return fake_endpoint + + with patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + side_effect=fake_create_pass_through_route, + ): + result = await gigachat_proxy_route( + endpoint="models", + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + assert isinstance(result, Response) + assert result.status_code == 200 + assert captured_kwargs["target"] == "https://gigachat.devices.sberbank.ru/api/v1/models" + assert captured_kwargs["custom_headers"] == {"Authorization": "Bearer gigachat-test-token"} + + @pytest.mark.asyncio + async def test_allm_passthrough_streaming_preserves_upstream_headers(self): + async def _stream() -> bytes: + yield b'data: {"id":"1"}\n\n' + + class MockPassthroughStreamingResponse: + def __init__(self): + self.status_code = 201 + self.headers = { + "content-type": "text/event-stream; charset=utf-8", + "x-request-id": "req-123", + "x-ratelimit-remaining-requests": "77", + "transfer-encoding": "chunked", + "content-encoding": "gzip", + } + self._iterator = _stream() + + def __aiter__(self): + return self + + async def __anext__(self): + return await self._iterator.__anext__() + + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": "some-provider/model", + "stream": True, + "litellm_call_id": "call-123", + "litellm_logging_obj": MagicMock(litellm_call_id="call-123"), + } + ) + + mock_request = MagicMock(spec=Request) + mock_request.headers = {"content-type": "application/json"} + mock_fastapi_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.allowed_model_region = "" + mock_user_api_key_dict.spend = 0.0 + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + mock_proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + mock_proxy_logging_obj.post_call_response_headers_hook = AsyncMock( + return_value={"x-test-callback-header": "callback-value"} + ) + + streaming_response = MockPassthroughStreamingResponse() + + async def _fake_route_request(*args, **kwargs): + async def _inner(): + return streaming_response + + return _inner() + + with patch.object( + processor, + "common_processing_pre_call_logic", + new=AsyncMock( + return_value=( + processor.data, + processor.data["litellm_logging_obj"], + ) + ), + ), patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.common_request_processing.route_request", + new=_fake_route_request, + ), patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers", + return_value={"x-litellm-call-id": "call-123"}, + ): + result = await processor.base_passthrough_process_llm_request( + request=mock_request, + fastapi_response=mock_fastapi_response, + user_api_key_dict=mock_user_api_key_dict, + proxy_logging_obj=mock_proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(), + select_data_generator=MagicMock(), + llm_router=None, + model="some-provider/model", + version="test-version", + ) + + assert isinstance(result, StreamingResponse) + assert result.status_code == 201 + assert result.headers["content-type"] == "text/event-stream; charset=utf-8" + assert result.headers["x-request-id"] == "req-123" + assert result.headers["x-ratelimit-remaining-requests"] == "77" + assert result.headers["x-litellm-call-id"] == "call-123" + assert result.headers["x-test-callback-header"] == "callback-value" + assert "transfer-encoding" not in result.headers + assert "content-encoding" not in result.headers + + class TestForwardHeaders: """ Test cases for _forward_headers parameter in passthrough endpoints @@ -4627,3 +4936,76 @@ class TestPassthroughRouterModelBudgetReservation: ) self._assert_metadata_carries_attribution(captured, user_api_key_dict) + + +class TestAzureRouterModelStreamingDispatch: + """ + Regression: ``llm_router.allm_passthrough_route`` returns an awaited + ``AsyncPassthroughStreamingResponse`` for streaming calls, which is no + longer an async generator under ``inspect.isasyncgen``. The dispatch's + else branch therefore calls ``.aiter_bytes()`` / ``.status_code`` / + ``.headers`` on it. The router's ``set_response_headers`` also runs the + result through ``prepare_response_for_header_attachment``, which used to + wrap it in ``HiddenParamsAsyncIteratorWrapper`` (no ``aiter_bytes``), so + every streaming Azure router-model request 500'd with + ``AttributeError: aiter_bytes``; ``_hidden_params`` on the streaming + response keeps it unwrapped. + """ + + @pytest.mark.asyncio + async def test_azure_router_model_streaming_returns_streaming_response(self, monkeypatch): + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + + upstream_body = b"data: hello\n\n" + + async def _upstream_response() -> httpx.Response: + upstream_request = httpx.Request( + "POST", + "https://my-azure.openai.azure.com/openai/deployments/gpt-5/chat/completions", + ) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=upstream_body, + request=upstream_request, + ) + + logging_obj = MagicMock() + logging_obj.async_flush_passthrough_collected_chunks = AsyncMock() + + from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment + + class StreamingRouter: + async def allm_passthrough_route(self, **kwargs): + streaming_response = await AsyncPassthroughStreamingResponse( + response=_upstream_response(), + litellm_logging_obj=logging_obj, + provider_config=MagicMock(), + ) + return prepare_response_for_header_attachment(streaming_response) + + async def fake_get_request_body(_request): + return {"model": "gpt-5", "stream": True} + + monkeypatch.setattr(proxy_server, "llm_router", StreamingRouter()) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + + result = await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + + assert isinstance(result, StreamingResponse) + assert result.status_code == 200 + body = b"".join([chunk async for chunk in result.body_iterator]) + assert body == upstream_body diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a3f56adb86f..f5ae0fe5977 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5462,3 +5462,164 @@ def test_the_marker_check_distinguishes_the_two_route_kinds(): builtin = MagicMock(spec=Request) builtin.scope = {"endpoint": llm_passthrough_endpoints.anthropic_proxy_route} assert request_dispatched_to_pass_through_endpoint(builtin) is False + + +async def _drive_passthrough_request_and_capture_logging(user_api_key_dict: UserAPIKeyAuth) -> tuple[int, object]: + import litellm + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + def transport_handler(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": True}) + + real_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": resolve_pass_through_request_timeout(None)}, + ) + cache_dict = litellm.in_memory_llm_clients_cache.cache_dict + cache_key = next((key for key, cached in cache_dict.items() if cached is real_handler), None) + assert cache_key is not None + cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + mock_request.body = AsyncMock(return_value=b'{"model": "gemini-2.0-flash"}') + + captured_data: dict = {} + + async def capture_pre_call_hook(user_api_key_dict, data, call_type): + captured_data.update(data) + return data + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=capture_pre_call_hook) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + mock_proxy_logging.get_proxy_hook = MagicMock(return_value=None) + + try: + with patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside pass_through_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ): + response = await pass_through_request( + request=mock_request, + target="https://upstream.example.test/v1/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + finally: + cache_dict[cache_key] = real_handler + + return response.status_code, captured_data.get("litellm_logging_obj") + + +@pytest.mark.asyncio +async def test_pass_through_request_wires_team_callbacks(): + """LIT-5152 regression: pass_through_request must resolve team-level logging + callbacks from key/team metadata and wire them into the Logging object, the + same way add_litellm_data_to_request does for normal LLM routes.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": { + "langfuse_public_key": "pk_test", + "langfuse_secret_key": "sk_test", + "langfuse_host": "https://langfuse.example.test", + }, + } + ] + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert logging_obj.dynamic_success_callbacks, "team success callbacks not wired into Logging" + assert logging_obj.dynamic_failure_callbacks, "team failure callbacks not wired into Logging" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") == "pk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_secret_key") == "sk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_host") == "https://langfuse.example.test" + assert ("langfuse_public_key", "pk_test") in logging_obj._trusted_callback_vars + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_malformed_team_logging_metadata(): + """LIT-5152 fail-open: a malformed team ``logging`` value (here a non-iterable) + raises inside callback resolution; the passthrough request must still succeed, + just without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={"logging": 5}, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_env_reference_in_deprecated_callback_settings(): + """LIT-5152 fail-open: the deprecated ``callback_settings`` team metadata skips + AddTeamCallback validation, so an ``os.environ/`` callback var would otherwise + blow up inside ``Logging.__init__`` and fail the request; the passthrough must + instead succeed without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "callback_settings": { + "success_callback": ["langfuse"], + "failure_callback": ["langfuse"], + "callback_vars": { + "langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY", + "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY", + "langfuse_host": "https://langfuse.example.test", + }, + } + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + assert not logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") + + +@pytest.mark.asyncio +async def test_resolve_team_callback_wiring_fails_open_on_operational_error(): + """LIT-5152 fail-open: an operational error while resolving callback metadata + (e.g. team config lookup hitting a dead secret manager) must not raise; the + request proceeds without dynamic callbacks and the error is logged.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + _resolve_team_callback_wiring, + ) + from litellm.proxy.proxy_server import ProxyConfig + + class RaisingTeamConfig(ProxyConfig): + def load_team_config(self, team_id: str) -> dict: + raise RuntimeError("secret manager unavailable") + + wiring = _resolve_team_callback_wiring( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", team_id="test-team"), + proxy_config=RaisingTeamConfig(), + route_description="pass_through_endpoint", + ) + + assert wiring.success_callbacks is None + assert wiring.failure_callbacks is None + assert wiring.logging_kwargs is None diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 791d64c6428..d7010de6405 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py """ import unittest +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from httpx import Response import litellm from litellm.proxy.proxy_server import app @@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText( - type="output_text", text="Hello from Cursor!" - ) - ], + content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")], ) ], ) @@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.llm_router") @patch("litellm.proxy.proxy_server.user_api_key_auth") - async def test_responses_api_key_spend_header_includes_response_cost( - self, mock_auth, mock_router - ): + async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router): """ Test that x-litellm-key-spend header includes the current request's response_cost for /v1/responses endpoint. @@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase): ResponseOutputMessage( type="message", role="assistant", - content=[ - ResponseOutputText(type="output_text", text="Test response") - ], + content=[ResponseOutputText(type="output_text", text="Test response")], ) ], ) @@ -356,6 +350,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "model": "gpt-4o", "input": "hello"} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -363,6 +358,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}} assert _extract_model_from_first_ws_event(event) == "gpt-4o" @@ -370,6 +366,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = { "type": "response.create", "model": "flat-model", @@ -381,6 +378,7 @@ class TestWSModelExtraction: from litellm.proxy.response_api_endpoints.endpoints import ( _extract_model_from_first_ws_event, ) + event = {"type": "response.create", "input": "hello"} assert _extract_model_from_first_ws_event(event) is None @@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation: ws.send_text.assert_awaited_once() ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message") error_payload = json.loads(ws.send_text.await_args.args[0]) - assert ( - error_payload["error"]["message"] - == "First message must be a response.create JSON object." - ) + assert error_payload["error"]["message"] == "First message must be a response.create JSON object." @pytest.mark.asyncio async def test_rejects_non_object_json_first_frame(self): @@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth: ws.url = "ws://testserver/v1/responses" ws.accept = AsyncMock() ws.receive_text = AsyncMock( - return_value=json.dumps( - {"type": "response.create", "model": "gpt-4o-mini", "input": []} - ) + return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []}) ) ws.close = AsyncMock() processor = MagicMock() - processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "gpt-4o-mini"}, MagicMock()) - ) + processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock())) async def fake_llm_call(): return None @@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth: _enforce_responses_ws_first_frame_model_auth, ) - request = Request( - {"type": "http", "method": "POST", "path": "/v1/responses", "headers": []} - ) + request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) user_api_key_dict = MagicMock() llm_router = MagicMock() @@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None ws.send_text.assert_not_awaited() - ws.close.assert_awaited_once_with( - code=1008, reason="Timed out waiting for first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message") @pytest.mark.asyncio async def test_invalid_json_sends_error_and_closes(self): @@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None payload = json.loads(ws.send_text.await_args.args[0]) assert payload["error"]["message"] == "First message is not valid JSON." - ws.close.assert_awaited_once_with( - code=1008, reason="Invalid JSON in first message" - ) + ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message") @pytest.mark.asyncio async def test_missing_model_sends_error_and_closes(self): @@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors: ) ws = MagicMock() - ws.receive_text = AsyncMock( - return_value=json.dumps({"type": "response.create", "input": []}) - ) + ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []})) ws.send_text = AsyncMock() ws.close = AsyncMock() @@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider: assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True def test_different_provider_is_not_same(self): - assert ( - self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") - is False - ) + assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False def test_inject_credentials_keeps_provider_for_same_provider_model(self): handler = self._handler("gpt-4o", custom_llm_provider="openai") @@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider: assert "custom_llm_provider" not in call_kwargs def test_unresolvable_connection_model_falls_back_to_custom_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") assert handler._same_provider("gpt-4o-mini") is True call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="gpt-4o-mini") assert call_kwargs["custom_llm_provider"] == "openai" def test_unresolvable_connection_model_still_drops_cross_provider(self): - handler = self._handler( - "my-custom-deployment", custom_llm_provider="openai" - ) + handler = self._handler("my-custom-deployment", custom_llm_provider="openai") call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash") assert "custom_llm_provider" not in call_kwargs @@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s type="message", role="assistant", status="completed", - content=[ - ResponseOutputText(type="output_text", text="agent reply", annotations=[]) - ], + content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])], ) ], ) @@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s app.dependency_overrides[user_api_key_auth] = _auth_override try: - with patch.object(ps, "llm_router", mock_router), patch( - "litellm.proxy.response_api_endpoints.endpoints._read_request_body", - side_effect=capturing_read_request_body, + with ( + patch.object(ps, "llm_router", mock_router), + patch( + "litellm.proxy.response_api_endpoints.endpoints._read_request_body", + side_effect=capturing_read_request_body, + ), ): client = TestClient(app) response = client.post( @@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock: mock_router.router_general_settings.pass_through_all_models = False mock_router.default_deployment = None mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]} - mock_router.pattern_router.get_pattern.side_effect = ( - lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None + mock_router.pattern_router.get_pattern.side_effect = lambda model: ( + [{"model_name": "anthropic/*"}] if model == base_model else None ) return mock_router @@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups: from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant router = Router( - model_list=[ - {"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}} - ], + model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}], routing_groups=[ {"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"} ], @@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage: assert usage["input_tokens"] == 0 assert usage["output_tokens"] == 0 assert usage["total_tokens"] == 0 + + +class TestResponsesInputTokens: + """Regression tests for POST /v1/responses/input_tokens. + + The docs promise OpenAI-format token counting on the proxy, but the route was + never registered, so the POST fell through to the GET/DELETE-only + /v1/responses/{response_id} route and returned 405.""" + + def _post_input_tokens( + self, + body: dict[str, Any], + path: str = "/v1/responses/input_tokens", + counter: AsyncMock | None = None, + ) -> tuple[Response, AsyncMock]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter + from litellm.types.utils import TokenCountResponse + + token_counter_mock = ( + counter + if counter is not None + else AsyncMock( + return_value=TokenCountResponse( + total_tokens=13, + request_model=body.get("model", ""), + model_used=body.get("model", ""), + tokenizer_type="openai_api", + ) + ) + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path) + app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock + try: + client = TestClient(app) + response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"}) + return response, token_counter_mock + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + app.dependency_overrides.pop(_proxy_token_counter, None) + + def test_string_input_returns_openai_input_tokens_shape(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) + + assert response.status_code == 200, response.text + assert response.json() == {"object": "response.input_tokens", "input_tokens": 13} + counter.assert_awaited_once() + assert counter.call_args.kwargs["call_endpoint"] is True + token_request = counter.call_args.kwargs["request"] + assert token_request.model == "gpt-4o" + assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}] + + def test_every_route_alias_is_registered(self): + for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"): + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path) + assert response.status_code == 200, f"{path}: {response.status_code} {response.text}" + + def test_input_items_instructions_and_tools_are_forwarded(self): + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ] + response, counter = self._post_input_tokens( + { + "model": "gpt-4o", + "input": [{"role": "user", "content": "What is the weather in Paris?"}], + "instructions": "You are terse.", + "tools": tools, + } + ) + + assert response.status_code == 200, response.text + token_request = counter.call_args.kwargs["request"] + assert token_request.messages == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "What is the weather in Paris?"}, + ] + assert token_request.tools == tools + + def test_missing_model_returns_openai_400(self): + response, counter = self._post_input_tokens({"input": "Hello"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'model'.", + "type": "invalid_request_error", + "param": "model", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_missing_input_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o"}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": "Missing required parameter: 'input'.", + "type": "invalid_request_error", + "param": "input", + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + @pytest.mark.parametrize("empty_input", ["", []]) + def test_empty_input_returns_openai_400(self, empty_input): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input}) + + assert response.status_code == 400, response.text + assert response.json() == { + "error": { + "message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""", + "type": "invalid_request_error", + "param": None, + "code": "missing_required_parameter", + } + } + counter.assert_not_awaited() + + def test_invalid_tools_returns_openai_400(self): + response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"}) + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + counter.assert_not_awaited() + + def test_provider_error_maps_status_code(self): + from litellm.proxy._types import ProxyException + + failing_counter = AsyncMock( + side_effect=ProxyException( + message="rate limited", + type="token_counting_error", + param="model", + code="429", + ) + ) + response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter) + + assert response.status_code == 429, response.text + assert response.json()["error"]["message"] == "rate limited" diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py new file mode 100644 index 00000000000..f65f68812a2 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -0,0 +1,48 @@ +from typing import Final + +import pytest + +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request +from litellm.proxy.utils import ProxyLogging + +TOKEN_COUNTING_ROUTES: Final = ( + "/responses/input_tokens", + "/v1/responses/input_tokens", + "/openai/v1/responses/input_tokens", + "/utils/token_counter", +) + + +def _budgeted_token() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0) + + +async def _reserve(route: str) -> dict | None: + return await reserve_budget_for_request( + request_body={"model": "gpt-4o", "input": "hello"}, + route=route, + llm_router=None, + valid_token=_budgeted_token(), + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES) +async def test_token_counting_routes_are_exempt_from_budget_reservation(route): + assert await _reserve(route) is None + + +@pytest.mark.asyncio +async def test_non_exempt_llm_route_still_reserves_budget(): + reservation: Final = await _reserve("/v1/responses") + + assert reservation is not None + assert reservation["reserved_cost"] > 0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 10c3e5fecf8..a0dcbf802ef 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2865,7 +2865,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2961,7 +2961,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -3055,7 +3055,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5022dab32be..9e5917637a8 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2,6 +2,7 @@ import asyncio import datetime import json from datetime import timezone +from collections.abc import Mapping from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -3956,3 +3957,71 @@ def test_passthrough_caching_carries_no_injection_marker(): ) metadata = json.loads(payload["metadata"]) assert metadata["litellm_gateway_injected_cache"] is None + + +def _routed_call_kwargs(model_info: Mapping[str, object]) -> dict[str, object]: + return { + "model": "claude-haiku-4-5", + "custom_llm_provider": "azure_ai", + "litellm_call_id": "router-corr-123", + "litellm_params": { + "metadata": { + "user_api_key": "test-key", + "model_group": "internal-router/gpt-5.4", + "deployment": "azure_ai/claude-haiku-4-5", + "model_info": model_info, + } + }, + } + + +def test_router_metadata_stamped_for_internal_router_model_deployment(): + """A deployment flagged model_info.internal_router_model gets a router_metadata + block correlating the requested model group with the selected deployment.""" + payload = get_logging_payload( + kwargs=_routed_call_kwargs({"id": "mi-1", "internal_router_model": True}), + response_obj=litellm.ModelResponse(id="chatcmpl-router-meta", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] == { + "requested_model": "internal-router/gpt-5.4", + "selected_model": "azure_ai/claude-haiku-4-5", + "selected_provider": "azure_ai", + "router_correlation_id": "router-corr-123", + } + + +def test_router_metadata_absent_without_internal_router_model_flag(): + payload = get_logging_payload( + kwargs=_routed_call_kwargs({"id": "mi-1"}), + response_obj=litellm.ModelResponse(id="chatcmpl-unflagged", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] is None + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_caller_forged_router_metadata_is_discarded(bucket): + """The raw request bucket is client-writable and _get_spend_logs_metadata projects + every SpendLogsMetadata key from it, so the server-derived value must overwrite + unconditionally or a caller could plant router provenance the router never produced.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": { + bucket: { + "user_api_key": "test-key", + "router_metadata": {"requested_model": "forged", "router_correlation_id": "forged-id"}, + } + }, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-forged-router-meta", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["router_metadata"] is None diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 629ae6084b8..df14224af5c 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4694,7 +4694,7 @@ class TestAllmPassthroughStreamingProviderGate: } return ProxyBaseLLMRequestProcessing(data=data) - async def _run(self, processing_obj, monkeypatch, chunks): + async def _run(self, processing_obj, monkeypatch, chunks, stream=None): import litellm.proxy.common_request_processing as crp from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth @@ -4702,9 +4702,11 @@ class TestAllmPassthroughStreamingProviderGate: for chunk in chunks: yield chunk + upstream_stream = stream if stream is not None else streaming_response() + async def fake_route_request(**kwargs): async def _llm_call(): - return streaming_response() + return upstream_stream return _llm_call() @@ -4729,6 +4731,40 @@ class TestAllmPassthroughStreamingProviderGate: skip_pre_call_logic=True, ) + @pytest.mark.asyncio + async def test_client_disconnect_closes_unbuffered_passthrough_stream(self, monkeypatch): + """Starlette abandons the body iterator when the client disconnects, so the + unbuffered passthrough branch must return _UpstreamClosingStreamingResponse, + whose shielded cleanup closes the upstream stream; that close is what flushes + buffered passthrough usage into spend logs.""" + processing_obj = self._build_processing_obj("gigachat") + monkeypatch.setattr(litellm, "callbacks", []) + upstream_closed = asyncio.Event() + + async def hanging_stream(): + try: + yield b"chunk-1" + await asyncio.Event().wait() + finally: + upstream_closed.set() + + result = await self._run(processing_obj, monkeypatch, [], stream=hanging_stream()) + + assert isinstance(result, _UpstreamClosingStreamingResponse) + + first_chunk_sent = asyncio.Event() + + async def receive(): + await first_chunk_sent.wait() + return {"type": "http.disconnect"} + + async def send(message): + if message["type"] == "http.response.body" and message.get("body"): + first_chunk_sent.set() + + await result({"type": "http"}, receive, send) + await asyncio.wait_for(upstream_closed.wait(), timeout=5) + @pytest.mark.asyncio async def test_non_bedrock_stream_is_not_buffered(self, monkeypatch): processing_obj = self._build_processing_obj("anthropic") diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 1de3ed6e56d..43280258153 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11399,35 +11399,10 @@ async def test_no_window_spend_row_enqueued_without_budget_limits(): assert enqueued == [] -@pytest.mark.asyncio -async def test_window_spend_row_carries_the_spend_log_request_id(): - """The flush excludes these ids from its one-time seed, so the id threaded - here has to be the same one the LiteLLM_SpendLogs row was written under.""" - from litellm.proxy.proxy_server import increment_spend_counters - - reset_at = datetime.now(timezone.utc) + timedelta(days=10) - key_obj = MagicMock() - key_obj.budget_limits = [ - {"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()} - ] - - with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", - team_id=None, - user_id=None, - response_cost=0.25, - request_id="chatcmpl-abc123", - ) - enqueued = await _drain(queue) - - assert enqueued[0]["request_ids"] == ("chatcmpl-abc123",) - - @pytest.mark.asyncio async def test_window_spend_row_carries_the_request_start_time(): - """The seed only excludes a batch id whose LiteLLM_SpendLogs.startTime is at - or after this, so it must be the same start the spend log was written with.""" + """The seed sums LiteLLM_SpendLogs only up to this point, so it must be the + same start the spend log row was written with.""" from litellm.proxy.proxy_server import increment_spend_counters reset_at = datetime.now(timezone.utc) + timedelta(days=10) @@ -11442,7 +11417,6 @@ async def test_window_spend_row_carries_the_request_start_time(): team_id=None, user_id=None, response_cost=0.25, - request_id="chatcmpl-abc123", request_started_at=datetime(2026, 8, 10, 12, 0, 0, 500_000, tzinfo=timezone.utc), ) enqueued = await _drain(queue) @@ -11451,26 +11425,7 @@ async def test_window_spend_row_carries_the_request_start_time(): @pytest.mark.asyncio -async def test_window_spend_row_without_a_request_id_excludes_nothing(): - from litellm.proxy.proxy_server import increment_spend_counters - - reset_at = datetime.now(timezone.utc) + timedelta(days=10) - key_obj = MagicMock() - key_obj.budget_limits = [ - {"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()} - ] - - with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", team_id=None, user_id=None, response_cost=0.25 - ) - enqueued = await _drain(queue) - - assert enqueued[0]["request_ids"] == () - - -@pytest.mark.asyncio -async def test_team_window_spend_row_carries_the_request_id(): +async def test_team_window_spend_row_carries_the_request_start_time(): from litellm.proxy.proxy_server import increment_spend_counters reset_at = datetime.now(timezone.utc) + timedelta(days=3) @@ -11485,11 +11440,11 @@ async def test_team_window_spend_row_carries_the_request_id(): team_id="team-1", user_id=None, response_cost=1.5, - request_id="chatcmpl-team", + request_started_at=datetime(2026, 8, 10, 12, 0, 0, 500_000, tzinfo=timezone.utc), ) enqueued = await _drain(queue) - assert enqueued[0]["request_ids"] == ("chatcmpl-team",) + assert enqueued[0]["started_at"] == "2026-08-10T12:00:00.500000" def _mock_startup_prisma_client(health_check_error=None, connect_error=None): diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b96d2eb5322..bb59e576568 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig: assert "extra_field" not in result["file"] assert "another_field" not in result["file"] + def test_transform_input_file_item_to_file_item_keeps_filename(self): + """OpenAI rejects file_data with no filename beside it, so dropping it 400s the request""" + result = ( + LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item( + { + "type": "input_file", + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0=", + } + ) + ) + assert result == { + "type": "file", + "file": { + "file_data": "data:application/pdf;base64,JVBERi0=", + "filename": "report.pdf", + }, + } + def test_transform_input_file_item_to_file_item_with_file_url(self): """file_url should be mapped to file_id for downstream URL handling""" result = ( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index eee4e9aa185..1ec8be88c9b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4425,6 +4425,265 @@ class _DummyPlugin: return context +class TestClassificationMode: + """Test classification_mode='user_turn': classify only requests whose newest turn is a new + human ask; tool-loop continuation turns replay the session's held routing decision.""" + + REASONING_ASK = { + "role": "user", + "content": "Let's think step by step and reason through this problem carefully.", + } + SIMPLE_ASK = {"role": "user", "content": "Hello!"} + ASSISTANT_ANSWER = {"role": "assistant", "content": "the answer"} + TOOL_CALL_1 = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}], + } + TOOL_RESULT_1 = {"role": "tool", "tool_call_id": "call_1", "content": "file contents"} + TOOL_CALL_2 = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_2", "type": "function", "function": {"name": "run_tests", "arguments": "{}"}}], + } + TOOL_RESULT_2 = {"role": "tool", "tool_call_id": "call_2", "content": "3 passed"} + + @pytest.fixture + def user_turn_config(self, basic_config) -> dict: + return {**basic_config, "classification_mode": "user_turn"} + + @staticmethod + def _request_kwargs(session_id: str) -> dict: + return {"metadata": {"session_id": session_id}} + + def _router(self, mock_router_instance, config: dict) -> ComplexityRouter: + mock_router_instance.cache = DualCache() + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + def _tool_loop_turns(self) -> list[list[dict]]: + return [ + [self.REASONING_ASK], + [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1], + [self.REASONING_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1, self.TOOL_CALL_2, self.TOOL_RESULT_2], + ] + + def test_default_mode_is_every_request(self, complexity_router): + assert complexity_router.config.classification_mode == "every_request" + + def test_invalid_classification_mode_rejected(self, mock_router_instance, basic_config): + with pytest.raises(ValidationError): + ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "classification_mode": "sometimes"}, + ) + + @pytest.mark.asyncio + async def test_user_turn_mode_classifies_tool_loop_once(self, mock_router_instance, user_turn_config): + """The mutation check: a 3-request tool loop drives exactly one classification, and both + continuation turns hold the classified model under the user_turn_continuation cause.""" + router = self._router(mock_router_instance, user_turn_config) + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + responses = [ + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("loop-1"), messages=turn + ) + for turn in self._tool_loop_turns() + ] + assert spy.call_count == 1 + assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] + assert [r.routing_decision["cause"] for r in responses[1:]] == [ + "user_turn_continuation", + "user_turn_continuation", + ] + + @pytest.mark.asyncio + async def test_every_request_default_classifies_every_tool_loop_turn(self, mock_router_instance, basic_config): + """Pins today's default: every request classifies, including tool-loop continuations.""" + router = self._router(mock_router_instance, basic_config) + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + responses = [ + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("loop-2"), messages=turn + ) + for turn in self._tool_loop_turns() + ] + assert spy.call_count == 3 + assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] + assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses) + + @pytest.mark.asyncio + async def test_continuation_without_session_id_still_classifies(self, mock_router_instance, user_turn_config): + """No resolvable session id means no held decision to replay, so every request classifies.""" + router = self._router(mock_router_instance, user_turn_config) + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + responses = [ + await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=turn) + for turn in self._tool_loop_turns() + ] + assert spy.call_count == 3 + assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] + assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses) + + @pytest.mark.asyncio + async def test_plugins_suppress_user_turn_gate(self, mock_router_instance, basic_config): + """A replayed decision would bypass the plugin pipeline, so plugins force every request + through _classify_and_route, exactly as they do for session_affinity.""" + router = self._router( + mock_router_instance, + {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]}, + ) + with patch.object(router, "_classify_and_route", wraps=router._classify_and_route) as spy: + responses = [ + await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("loop-3"), messages=turn + ) + for turn in self._tool_loop_turns() + ] + assert spy.call_count == 3 + assert [r.model for r in responses] == ["o1-preview", "o1-preview", "o1-preview"] + assert all(r.routing_decision["cause"] != "user_turn_continuation" for r in responses) + + @pytest.mark.asyncio + async def test_new_human_ask_reclassifies_and_repins(self, mock_router_instance, user_turn_config): + """Unlike session_affinity, a new human ask never short-circuits on the pin: the session + re-classifies, moves tier, and the moved decision becomes the next held decision.""" + router = self._router(mock_router_instance, user_turn_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("s-repin"), messages=[self.REASONING_ASK] + ) + second = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-repin"), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK], + ) + third = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-repin"), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, self.TOOL_CALL_1, self.TOOL_RESULT_1], + ) + assert first.model == "o1-preview" + assert second.model == "gpt-4o-mini" + assert third.model == "gpt-4o-mini" + assert third.routing_decision["cause"] == "user_turn_continuation" + + @pytest.mark.asyncio + async def test_new_ask_with_trailing_system_reminder_reclassifies(self, mock_router_instance, user_turn_config): + """Claude Code appends a system-role reminder after the human turn; that trailing plumbing + must not turn a new ask into a continuation, and a continuation turn carrying the same + trailing reminder stays a continuation.""" + router = self._router(mock_router_instance, user_turn_config) + reminder = {"role": "system", "content": "100 tokens left"} + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("s-reminder"), messages=[self.REASONING_ASK] + ) + second = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-reminder"), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK, reminder], + ) + third = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-reminder"), + messages=[ + self.REASONING_ASK, + self.ASSISTANT_ANSWER, + self.SIMPLE_ASK, + reminder, + self.TOOL_CALL_1, + self.TOOL_RESULT_1, + reminder, + ], + ) + assert first.model == "o1-preview" + assert second.model == "gpt-4o-mini" + assert second.routing_decision["cause"] != "user_turn_continuation" + assert third.model == "gpt-4o-mini" + assert third.routing_decision["cause"] == "user_turn_continuation" + + @pytest.mark.asyncio + async def test_escalation_keyword_turn_is_a_new_ask(self, mock_router_instance, user_turn_config): + """An escalation keyword arrives as human text, so the turn classifies and escalates + instead of replaying the held decision.""" + router = self._router(mock_router_instance, user_turn_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("s-esc"), messages=[self.SIMPLE_ASK] + ) + second = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-esc"), + messages=[self.SIMPLE_ASK, self.ASSISTANT_ANSWER, {"role": "user", "content": "LITELLM ESCALATE"}], + ) + assert first.model == "gpt-4o-mini" + assert second.model == "gpt-4o" + assert second.routing_decision["escalated"] is True + + @pytest.mark.asyncio + async def test_messages_surface_tool_result_shapes(self, mock_router_instance, user_turn_config): + """Messages-surface shapes: a tool_result-only user turn is a continuation, while an ask + riding alongside a tool_result in the same turn is a new ask.""" + router = self._router(mock_router_instance, user_turn_config) + tool_use = {"role": "assistant", "content": [{"type": "tool_use", "id": "x", "name": "t", "input": {}}]} + tool_result = {"type": "tool_result", "tool_use_id": "x", "content": "ok"} + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("s-msgs"), messages=[self.REASONING_ASK] + ) + pure = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-msgs"), + messages=[self.REASONING_ASK, tool_use, {"role": "user", "content": [tool_result]}], + ) + hybrid = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-msgs"), + messages=[ + self.REASONING_ASK, + tool_use, + {"role": "user", "content": [tool_result, {"type": "text", "text": "Hello!"}]}, + ], + ) + assert first.model == "o1-preview" + assert pure.model == "o1-preview" + assert pure.routing_decision["cause"] == "user_turn_continuation" + assert hybrid.model == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_session_affinity_wins_when_both_knobs_are_on(self, mock_router_instance, user_turn_config): + """With session_affinity also on, the pin short-circuits new asks too and keeps its own + cause, so the session stays on turn 1's model.""" + router = self._router(mock_router_instance, {**user_turn_config, "session_affinity": True}) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=self._request_kwargs("s-both"), messages=[self.REASONING_ASK] + ) + second = await router.async_pre_routing_hook( + model="test-model", + request_kwargs=self._request_kwargs("s-both"), + messages=[self.REASONING_ASK, self.ASSISTANT_ANSWER, self.SIMPLE_ASK], + ) + assert first.model == "o1-preview" + assert second.model == "o1-preview" + assert second.routing_decision["cause"] == "session_affinity_pin" + + def test_user_turn_mode_enables_tier_and_deployment_pins(self, mock_router_instance, basic_config): + """user_turn implies the tier pin machinery (the pin write is what gives a continuation + a held decision) and the tier pin implies the deployment pin; plugins suppress both.""" + default = self._router(mock_router_instance, basic_config) + enabled = self._router(mock_router_instance, {**basic_config, "classification_mode": "user_turn"}) + suppressed = self._router( + mock_router_instance, + {**basic_config, "classification_mode": "user_turn", "plugins": [_DummyPlugin()]}, + ) + assert default._uses_tier_pin is False + assert enabled._uses_tier_pin is True + assert enabled._uses_deployment_pin is True + assert suppressed._uses_tier_pin is False + assert suppressed._uses_deployment_pin is False + + class TestRoutingPlugins: """Test the `complexity_router_config.plugins` field: narrows the classified tier's candidate pool before a model is picked. Discussion: @@ -9762,3 +10021,719 @@ class TestHeuristicFirst: ) outcome = await router.aclassify(NO_SIGNAL_PROMPT) assert outcome.cause == "default_model_fallback" + + +def _windowed_router(*deployments: tuple) -> Router: + """Real Router; each deployment is (group, provider_model, declared window or None). + None means no declared override on a model the cost map does not know: unresolvable.""" + return Router( + model_list=[ + { + "model_name": group, + "litellm_params": {"model": provider_model, "mock_response": "ok"}, + **({"model_info": {"max_input_tokens": window}} if window is not None else {}), + } + for group, provider_model, window in deployments + ] + ) + + +_SMALL = ("small-model", "openai/gpt-3.5-turbo", 16385) +_BIG = ("big-model", "openai/gpt-4o-mini", 200000) + +# A long agentic session whose newest ask is trivial: low-density filler the heuristic scores +# SIMPLE, sized well past a 16,385-token window so the fit check must move it. +_CONTEXT_FILLER = "The meeting notes were saved to the shared folder for later review this week. " * 2000 +_OVERSIZED_TURNS = [ + {"role": "user", "content": "Here is everything discussed so far. " + _CONTEXT_FILLER}, + {"role": "assistant", "content": "Noted, I have read all of it."}, + {"role": "user", "content": "ok continue"}, +] +# ~40k CJK chars: chars/4 says ~10k tokens, the real tokenizer says several times that. A +# character-based shortcut would skip counting and dispatch this to a 16k window. +_CJK_TURNS = [ + {"role": "user", "content": "会议记录已经保存到共享文件夹里，供大家本周晚些时候查阅和讨论使用。" * 1300}, + {"role": "user", "content": "ok continue"}, +] + + +def _tier_config(**overrides) -> Dict: + return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides} + + +class TestContextWindowEscalation: + """A tier decided on complexity alone must still hold the prompt, or the provider 400s. + + The classifier never weighs prompt size (token count is a 0.10-weight scoring dimension, + below every tier boundary), so a long session ending in a trivial ask lands on the + smallest tier and dies upstream with no retry. The gate checks fit pre-dispatch, against + windows resolved through the real Router deployment chain. + """ + + @pytest.mark.asyncio + async def test_an_oversized_simple_prompt_escalates_to_the_lowest_tier_that_fits(self): + """The LIT-6503 regression: SIMPLE verdict, 17k-token prompt, 16,385-token tier model. + + Unfixed, this dispatched to the small model and the provider rejected it with a + context-window 400 that neither the retry layer nor tier-keyed fallbacks catch. + """ + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(), + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalated"] is True + assert result.routing_decision["context_escalation_original_tier"] == "SIMPLE" + assert result.routing_decision["tier"] == "COMPLEX" + assert "context_escalation" in result.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_a_prompt_that_fits_routes_exactly_as_before(self): + """The gate must be invisible for normal traffic: same model, no escalation facts.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(), + ) + + result = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=[{"role": "user", "content": "ok continue"}] + ) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + assert "context_escalation_original_tier" not in result.routing_decision + + @pytest.mark.asyncio + async def test_the_pick_prefers_a_fitting_group_inside_the_decided_tier(self): + """A tier holding both a small and a large group keeps the request and picks the one + that fits, which is cheaper than escalating and preserves the classifier's decision.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, ("mid-model", "openai/gpt-4o-mini", 200000), _BIG), + complexity_router_config={"tiers": {"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}}, + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "mid-model" + assert result.routing_decision["tier"] == "SIMPLE" + assert "context_escalated" not in result.routing_decision + + @pytest.mark.asyncio + async def test_a_group_is_only_as_safe_as_its_smallest_deployment(self): + """One group name can front deployments with different windows, and the core router + picks among them with no fit check, so retaining the group on its largest member + turns the pick into a coin flip against a 400. The gate judges the group by its + smallest resolvable window and escalates past it.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=Router( + model_list=[ + { + "model_name": "mixed-pool", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 16385}, + }, + { + "model_name": "mixed-pool", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 200000}, + }, + { + "model_name": "big-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 200000}, + }, + ] + ), + complexity_router_config={"tiers": {"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}}, + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalated"] is True + + @pytest.mark.asyncio + async def test_token_dense_text_cannot_slip_past_the_counting_shortcut(self): + """CJK text runs several tokens per four characters, so a chars/4 shortcut would skip + the real count and dispatch an oversized prompt. The skip is gated on the UTF-8 byte + length, which the token count can never exceed.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(), + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_CJK_TURNS) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalated"] is True + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "deployments,tiers,expected_model", + [ + ( + (("small-model", "openai/unmapped-model-under-test", None), _BIG), + {"SIMPLE": "small-model", "COMPLEX": "big-model"}, + "small-model", + ), + ( + (_SMALL, ("mid-model", "openai/another-unmapped-model", None), _BIG), + {"SIMPLE": "small-model", "MEDIUM": "mid-model", "COMPLEX": "big-model"}, + "big-model", + ), + ((_SMALL,), {"SIMPLE": "small-model"}, "small-model"), + ], + ids=["unknown-window-stays", "unproven-target-skipped", "nothing-fits-stays"], + ) + async def test_unknown_windows_are_never_acted_on(self, deployments, tiers, expected_model): + """No faith in either direction: a model with no resolvable window is never escalated + away from (its misfit is unprovable) and never escalated onto (its fit is unprovable); + when nothing provably fits, the classified tier stands and the client owns overflow.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(*deployments), + complexity_router_config={"tiers": tiers}, + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == expected_model + + @pytest.mark.asyncio + async def test_the_disabled_gate_dispatches_on_complexity_alone(self): + """The escape hatch: enable_context_window_escalation false restores today's behavior.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(enable_context_window_escalation=False), + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "small-model" + assert "context_escalated" not in result.routing_decision + + @pytest.mark.asyncio + async def test_out_of_band_system_and_tools_count_against_the_window(self): + """The Claude Code shape that live-testing caught: a tiny ask riding a top-level + `system` block and tool definitions that together dwarf the message list. None of + that reaches resolved messages on /v1/messages, so a gate reading only messages + dispatches a provably oversized request and the provider 400s anyway.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(), + ) + + result = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={ + "proxy_server_request": { + "body": { + "system": _CONTEXT_FILLER, + "tools": [{"name": f"tool_{i}", "description": _CONTEXT_FILLER[:500]} for i in range(20)], + } + } + }, + messages=[{"role": "user", "content": "reply with exactly: rig check ok"}], + ) + + assert result is not None + assert result.model == "big-model" + assert result.routing_decision["context_escalated"] is True + + @pytest.mark.asyncio + async def test_an_escalated_first_turn_never_becomes_the_session_pin(self): + """Escalation describes the prompt's size, not the session: once the client compacts, + the next turn fits again, so pinning the big-window tier would hold the whole session + on it for the TTL. The escalated turn routes big, and the next fitting turn classifies + fresh instead of inheriting a pin.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(session_affinity=True), + ) + session_kwargs = lambda: {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} # noqa: E731 + + first = await router.async_pre_routing_hook( + model="test-router", request_kwargs=session_kwargs(), messages=_OVERSIZED_TURNS + ) + second = await router.async_pre_routing_hook( + model="test-router", request_kwargs=session_kwargs(), messages=[{"role": "user", "content": "ok continue"}] + ) + + assert first is not None and first.model == "big-model" + assert second is not None and second.model == "small-model" + assert second.routing_decision["cause"] != "session_affinity_pin" + + @pytest.mark.asyncio + async def test_a_pinned_session_escalates_per_request_and_keeps_its_pin(self): + """The pin fast path skips classification, not physics: an oversized turn on a session + pinned to the small tier is served by the fitting tier, while the stored pin keeps the + session's own model so the first turn that fits again routes exactly as pinned.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=_windowed_router(_SMALL, _BIG), + complexity_router_config=_tier_config(session_affinity=True), + ) + session_kwargs = lambda: {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} # noqa: E731 + + pinned = await router.async_pre_routing_hook( + model="test-router", request_kwargs=session_kwargs(), messages=[{"role": "user", "content": "ok continue"}] + ) + oversized = await router.async_pre_routing_hook( + model="test-router", request_kwargs=session_kwargs(), messages=_OVERSIZED_TURNS + ) + back_to_small = await router.async_pre_routing_hook( + model="test-router", request_kwargs=session_kwargs(), messages=[{"role": "user", "content": "ok continue"}] + ) + + assert pinned is not None and pinned.model == "small-model" + assert oversized is not None and oversized.model == "big-model" + assert oversized.routing_decision["cause"] == "session_affinity_pin" + assert oversized.routing_decision["context_escalated"] is True + assert oversized.routing_decision["context_escalation_original_tier"] == "SIMPLE" + assert back_to_small is not None and back_to_small.model == "small-model" + assert back_to_small.routing_decision["cause"] == "session_affinity_pin" + + @pytest.mark.asyncio + async def test_the_adaptive_cold_start_never_samples_a_model_that_cannot_hold_the_prompt(self): + """The bandit's exploration is still bounded by physics: with the whole classified tier + unobserved, cold start samples only among models whose window holds the prompt.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=Router( + model_list=[ + { + "model_name": "small-model", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 16385}, + }, + { + "model_name": "mid-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 200000}, + }, + ] + ), + complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["small-model", "mid-model"]}}, + ) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "mid-model" + + @pytest.mark.asyncio + async def test_the_gate_never_resolves_an_authenticating_provider(self, monkeypatch, tmp_path): + """Resolving github_copilot runs its OAuth device flow, so a window question must adopt + the declaration instead of resolving: the copilot group reads as unknown-window and the + request stays put, with zero copilot resolutions recorded.""" + import json + import time + + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "api-key.json").write_text(json.dumps({"token": "tid=test", "expires_at": int(time.time()) + 3600})) + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=Router( + model_list=[ + {"model_name": "cop-pool", "litellm_params": {"model": "github_copilot/gpt-4o"}}, + { + "model_name": "big-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 200000}, + }, + ] + ), + complexity_router_config={"tiers": {"SIMPLE": "cop-pool", "COMPLEX": "big-model"}}, + ) + real_get_llm_provider = litellm.get_llm_provider + copilot_resolutions: List = [] + + def _guarded(*args, **kwargs): + target = str(kwargs.get("model") or (args[0] if args else "")) + str(kwargs.get("custom_llm_provider") or "") + if "github_copilot" in target: + copilot_resolutions.append(target) + raise RuntimeError("the gate must not resolve an authenticating provider") + return real_get_llm_provider(*args, **kwargs) + + monkeypatch.setattr(litellm, "get_llm_provider", _guarded) + + result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + + assert result is not None + assert result.model == "cop-pool" + assert copilot_resolutions == [] + + @pytest.mark.asyncio + async def test_the_full_routing_path_serves_the_escalated_deployment(self): + """End to end through Router.async_get_available_deployment: the auto-router alias with + an oversized prompt resolves to the big tier's deployment, and a small prompt to the + small tier's, with no mocking anywhere in the resolution chain.""" + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}}, + }, + }, + { + "model_name": "small-model", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 16385}, + }, + { + "model_name": "big-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"max_input_tokens": 200000}, + }, + ] + ) + + oversized = await router.async_get_available_deployment( + model="smart-router", request_kwargs={}, messages=_OVERSIZED_TURNS + ) + small = await router.async_get_available_deployment( + model="smart-router", request_kwargs={}, messages=[{"role": "user", "content": "ok continue"}] + ) + + assert oversized["model_name"] == "big-model" + assert small["model_name"] == "small-model" + + +IMG_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}} +PLAN_BODY = { + "messages": [{"role": "system", "content": [{"type": "text", "text": "Plan mode is active. Do not execute."}]}] +} + + +class TestModalityRouting: + """modality_routing: the response gate replaces a routed model that cannot take images.""" + + IMAGE_MESSAGE = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, IMG_PART]}] + BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"} + BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True} + + @staticmethod + def _router(mock_router_instance, config, vision_by_model): + """vision_by_model: model name -> True/False (deployment model_info) or None (undeclared).""" + + def get_model_list(model_name=None): + if model_name not in vision_by_model: + return [] + declared = vision_by_model[model_name] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}"}, + "model_info": {} if declared is None else {"supports_vision": declared}, + } + ] + + mock_router_instance.get_model_list = get_model_list + return ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "config_extra, vision, send_image, expected_model, expect_marker", + [ + ({}, {"text-cheap": False}, True, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": False}, False, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": None}, True, "text-cheap", False), + ], + ids=["flag_off", "no_image", "undeclared_model_stays_routable"], + ) + async def test_gate_leaves_ungated_requests_untouched( + self, mock_router_instance, config_extra, vision, send_image, expected_model, expect_marker + ): + router = self._router(mock_router_instance, {"tiers": dict(self.BASE_TIERS), **config_extra}, vision) + request = self.IMAGE_MESSAGE if send_image else [{"role": "user", "content": "What color is the sky?"}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=request) + assert result.model == expected_model + assert result.routing_decision["cause"] == "heuristic_scorer" + assert ("modality:image" in (result.routing_decision.get("signals") or ())) is expect_marker + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "part", + [ + IMG_PART, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + {"type": "tool_result", "tool_use_id": "tu_1", "content": [dict(IMG_PART, type="image")]}, + ], + ids=["image_url", "input_image", "anthropic_image", "tool_result_nested"], + ) + async def test_every_image_dialect_escalates(self, mock_router_instance, part): + router = self._router( + mock_router_instance, {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, dict(self.BASE_VISION) + ) + message = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, part]}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=message) + assert result.model == "vision-mid" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "path, expected_model, expected_cause", + [ + ("classifier_escalates", "vision-mid", "modality_escalation"), + ("same_tier_repick_keeps_cause", "vision-cheap", "heuristic_scorer"), + ("keyword_tier_escalates", "vision-mid", "modality_escalation"), + ("no_ask_capable_default_kept", "vision-default", "default_fallback"), + ("no_ask_text_default_displaced", "vision-mid", "modality_escalation"), + ("custom_tiers_walk", "premium-model", "modality_escalation"), + ("pin_kept_bypasses", "text-cheap", "session_affinity_pin"), + ("pin_replacement_gated", "vision-big", "modality_escalation"), + ("adaptive_pick_rewritten", "vision-mid", "modality_escalation"), + ], + ) + async def test_placements_across_decision_paths(self, mock_router_instance, path, expected_model, expected_cause): + config = {"tiers": dict(self.BASE_TIERS), "modality_routing": True} + vision = dict(self.BASE_VISION) + request_kwargs = {} + messages = self.IMAGE_MESSAGE + if path == "same_tier_repick_keeps_cause": + config["tiers"]["SIMPLE"] = ["text-cheap", "vision-cheap"] + vision["vision-cheap"] = True + with patch( # test-quality-ok: the mixed-pool repick is unreachable deterministically without pinning the first random pick + "litellm.router_strategy.complexity_router.complexity_router.random.choice", + side_effect=lambda pool: sorted(pool)[0], + ): + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + assert result.routing_decision["signals"][-1] == "modality:image" + return + if path == "keyword_tier_escalates": + config["keyword_tier_rules"] = [{"keywords": ["quick lookup"], "tier": "SIMPLE"}] + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path == "no_ask_capable_default_kept": + config["default_model"] = "vision-default" + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "no_ask_text_default_displaced": + config["default_model"] = "text-default" + vision["text-default"] = False + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "custom_tiers_walk": + config = { + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-4o-mini"}, + "fallback_tier": "cheap", + "tier_definitions": [ + {"name": "cheap", "description": "trivial asks"}, + {"name": "premium", "description": "hard asks"}, + ], + "tiers": {"cheap": "cheap-model", "premium": "premium-model"}, + "keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "cheap"}], + "modality_routing": True, + } + vision = {"cheap-model": False, "premium-model": True} + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path in ("pin_kept_bypasses", "pin_replacement_gated"): + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + config["session_affinity"] = True + request_kwargs = {"metadata": {"session_id": "s1"}} + if path == "pin_replacement_gated": + config["tiers"]["MEDIUM"] = "text-mid" + vision["text-mid"] = False + messages = [ + {"role": "user", "content": [{"type": "text", "text": "LITELLM ESCALATE describe this"}, IMG_PART]} + ] + elif path == "adaptive_pick_rewritten": + config["adaptive"] = True + mock_router_instance.model_list = [] + mock_router_instance.model_name_to_deployment_indices = {} + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + if path == "adaptive_pick_rewritten": + assert request_kwargs["metadata"]["adaptive_router_chosen_model"] == expected_model + + @pytest.mark.asyncio + async def test_plan_floored_decision_never_falls_to_default_model(self, mock_router_instance): + """An upward-only walk cannot undercut the floor; default_model must not either.""" + config = { + "tiers": {"SIMPLE": "vision-cheap", "MEDIUM": "text-mid"}, + "default_model": "vision-default", + "plan_mode_min_tier": "MEDIUM", + "modality_routing": True, + } + vision = {"vision-cheap": True, "text-mid": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + async def test_at_floor_plan_turn_never_falls_to_default_model(self, mock_router_instance): + """A sentinel turn whose classified tier already satisfies the floor keeps its ordinary + cause, so the record carries no floor marker; the default arm must still refuse it.""" + config = { + "tiers": {"SIMPLE": "text-a", "MEDIUM": "text-b"}, + "default_model": "vision-default", + "plan_mode_min_tier": "SIMPLE", + "modality_routing": True, + } + vision = {"text-a": False, "text-b": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "default_model, default_vision, expect_error", + [(None, None, True), ("text-default", False, True), ("vision-default", True, False)], + ids=["no_default", "text_only_default", "vision_default_serves"], + ) + async def test_no_capable_tier_above_uses_default_or_rejects( + self, mock_router_instance, default_model, default_vision, expect_error + ): + config = {"tiers": {"SIMPLE": "text-cheap", "COMPLEX": "text-big"}, "modality_routing": True} + vision = {"text-cheap": False, "text-big": False} + if default_model is not None: + config["default_model"] = default_model + vision[default_model] = default_vision + router = self._router(mock_router_instance, config, vision) + if expect_error: + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + return + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-default" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_mixed_deployment_group_is_treated_text_only(self, mock_router_instance): + def get_model_list(model_name=None): + declared = {"mixed-group": [True, False], "vision-big": [True]}.get(model_name) + if declared is None: + return [] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}-{i}"}, + "model_info": {"supports_vision": accepts}, + } + for i, accepts in enumerate(declared) + ] + + mock_router_instance.get_model_list = get_model_list + router = ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "mixed-group", "COMPLEX": "vision-big"}, + "modality_routing": True, + }, + ) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-big" + assert result.routing_decision["cause"] == "modality_escalation" + + @pytest.mark.asyncio + async def test_continuation_turn_screenshot_escalates_past_the_held_model(self, mock_router_instance): + """classification_mode user_turn replays the held model on continuation turns; a + continuation carrying a screenshot must still be re-placed when that model is text-only.""" + mock_router_instance.cache = DualCache() + config = { + "tiers": dict(self.BASE_TIERS), + "classification_mode": "user_turn", + "modality_routing": True, + } + router = self._router(mock_router_instance, config, dict(self.BASE_VISION)) + first = await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": "cont-1"}}, + messages=[{"role": "user", "content": "hi there"}], + ) + assert first.model == "text-cheap" + continuation = [ + {"role": "user", "content": "hi there"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "screenshot", "input": {}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + } + ], + }, + ] + second = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "cont-1"}}, messages=continuation + ) + assert second.model == "vision-mid" + assert second.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in second.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_rewrite_carries_the_context_escalation_record(self, mock_router_instance): + """A context-window escalation and a modality re-place are separate facts on one + record; rewriting for the image must not drop the sibling gate's fields.""" + from litellm.types.router import PreRoutingHookResponse + + router = self._router( + mock_router_instance, + {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, + dict(self.BASE_VISION), + ) + decision = router._build_routing_decision( + routed_model="text-cheap", + cause="heuristic_scorer", + tier=ComplexityTier.SIMPLE, + context_escalation_original_tier=ComplexityTier.SIMPLE, + ) + response = PreRoutingHookResponse(model="text-cheap", messages=None, routing_decision=decision) + rewritten = await router._gate_response_modality(response, None, self.IMAGE_MESSAGE, {}) + assert rewritten.model == "vision-mid" + assert rewritten.routing_decision["cause"] == "modality_escalation" + assert rewritten.routing_decision["context_escalated"] is True + assert rewritten.routing_decision["context_escalation_original_tier"] == "SIMPLE" + + def test_modality_escalation_is_never_pinnable(self): + from litellm.router_strategy.complexity_router.complexity_router import _decision_is_pinnable + + assert _decision_is_pinnable({"cause": "modality_escalation"}) is False + assert _decision_is_pinnable({"cause": "heuristic_scorer"}) is True diff --git a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py new file mode 100644 index 00000000000..7e94205fb09 --- /dev/null +++ b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py @@ -0,0 +1,35 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_friendli_glm_5_3_flash_model_info(): + model = "friendliai/zai-org/GLM-5.3-Flash" + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + info = model_cost.get(model) + assert ( + info is not None + ), f"{model} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "friendliai" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] == 1.5e-07 + assert info["output_cost_per_token"] == 5e-07 + assert info["cache_read_input_token_cost"] == 3e-08 + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 1048576 + assert info["supports_function_calling"] is True + assert info["supports_reasoning"] is True + assert info["reasoning_effort_levels"] == ["low", "high", "max"] + assert info["supports_tool_choice"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_vision"] is True + assert info["supports_image_input"] is True + assert info["supports_video_input"] is True + + routed_model, provider, _, _ = get_llm_provider(model=model) + assert routed_model == "zai-org/GLM-5.3-Flash" + assert provider == "friendliai" diff --git a/tests/test_litellm/test_friendli_glm_5_3_model_metadata.py b/tests/test_litellm/test_friendli_glm_5_3_model_metadata.py new file mode 100644 index 00000000000..5282b0f589e --- /dev/null +++ b/tests/test_litellm/test_friendli_glm_5_3_model_metadata.py @@ -0,0 +1,34 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_friendli_glm_5_3_model_info(): + model = "friendliai/zai-org/GLM-5.3" + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + info = model_cost.get(model) + assert ( + info is not None + ), f"{model} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "friendliai" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] == 1.26e-06 + assert info["output_cost_per_token"] == 3.96e-06 + assert info["cache_read_input_token_cost"] == 2.34e-07 + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 1048576 + assert info["supports_function_calling"] is True + assert info["supports_reasoning"] is True + assert info["reasoning_effort_levels"] == ["low", "high", "max"] + assert info["supports_tool_choice"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_vision"] is False + assert info["supports_image_input"] is False + + routed_model, provider, _, _ = get_llm_provider(model=model) + assert routed_model == "zai-org/GLM-5.3" + assert provider == "friendliai" diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 826beb74a27..a96e8541e06 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -1,3 +1,4 @@ +import inspect import json from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -600,6 +601,72 @@ def test_reconnect_kwargs_in_cluster_kwargs(): assert "socket_keepalive" in kwargs +def test_retry_attempts_in_cluster_kwargs(): + """cluster_error_retry_attempts must survive the cluster kwarg allow-list so + operators can bound worst-case retry latency on a Redis Cluster: it was being + silently dropped because the allow-list was built from redis.RedisCluster's + decorated __init__ without unwrapping it, so getfullargspec saw an empty + (self, *args, **kwargs) wrapper signature.""" + kwargs = _get_redis_cluster_kwargs() + assert "cluster_error_retry_attempts" in kwargs + + +def test_async_only_kwargs_in_cluster_kwargs_when_async_client_requested(): + """decode_responses is on the async cluster client's constructor and not the sync + one, on every redis-py the matrix covers. Introspecting the sync class regardless + of which client is actually built silently drops it for every async cluster caller.""" + sync_kwargs = _get_redis_cluster_kwargs() + async_kwargs = _get_redis_cluster_kwargs(async_redis.RedisCluster) + + assert "decode_responses" not in sync_kwargs + assert "decode_responses" in async_kwargs + + +@patch( # test-quality-ok: redis-py >= 6 keeps no cluster_error_retry_attempts attribute on the built client, so the constructor call is the only place the value is observable + "litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class" +) +def test_async_cluster_forwards_retry_attempts(mock_get_cluster_class): + """Regression: cluster_error_retry_attempts must reach the constructed async + cluster client. Silently dropping it removes an operator's only lever for + bounding a stuck node's worst-case retry latency, and the client falls back + to redis-py's own default (3 retries) instead.""" + mock_cluster_cls = mock_get_cluster_class.return_value + get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + cluster_error_retry_attempts=2, + ) + + call_kwargs = mock_cluster_cls.call_args[1] + assert call_kwargs["cluster_error_retry_attempts"] == 2 + + +def test_async_cluster_passes_async_only_kwargs(): + """Regression: decode_responses is an async-cluster-only constructor arg. When + the allow-list came from the sync class it was filtered out and values came + back as bytes instead of str.""" + client = get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + decode_responses=True, + ) + + assert client.connection_kwargs["decode_responses"] is True + + +@pytest.mark.parametrize("cluster_client", [redis.RedisCluster, async_redis.RedisCluster], ids=["sync", "async"]) +def test_cluster_kwargs_exclude_variadic_parameters(cluster_client): + """*args / **kwargs are signature placeholders, not connection settings, and + must never land in the allow-list regardless of which cluster client is + introspected.""" + variadic = { + name + for name, param in inspect.signature(cluster_client).parameters.items() + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD) + } + + leaked = variadic & set(_get_redis_cluster_kwargs(cluster_client)) + assert not leaked, f"variadic params leaked into the allow-list: {leaked}" + + @patch("litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class") def test_async_cluster_sets_reconnect_defaults(mock_get_cluster_class): """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 97286017ffe..44c1cdbff06 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8155,6 +8155,71 @@ class TestUpsertDeploymentRollback: assert len(router.model_list) == 1 +class TestUpsertDeploymentRename: + """ + Issue #38360: renaming a model wrote the new `model_name` to the db, but the reload's + `upsert_deployment` compared only `litellm_params` and `model_info`. A rename with no + other edit therefore compared equal and the router kept the old name until a restart, + so `/model/info` and `/v1/models` served the stale name and the new one was unroutable. + """ + + @staticmethod + def _router() -> "litellm.Router": + return litellm.Router( + model_list=[ + { + "model_name": "old-name", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}, + "model_info": {"id": "rename-1", "db_model": True}, + } + ] + ) + + @staticmethod + def _deployment(model_name: str, tpm: int | None = None): + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + return Deployment( + model_name=model_name, + litellm_params=LiteLLM_Params(model="openai/gpt-4o", api_key="sk-test", tpm=tpm), + model_info=ModelInfo(id="rename-1", db_model=True), + ) + + def test_rename_only_updates_the_router(self): + router = self._router() + + assert router.upsert_deployment(deployment=self._deployment("new-name")) is not None + + assert [model["model_name"] for model in router.model_list] == ["new-name"] + renamed = router.get_deployment(model_id="rename-1") + assert renamed is not None + assert renamed.model_name == "new-name" + + def test_rename_only_makes_the_new_name_routable(self): + router = self._router() + + router.upsert_deployment(deployment=self._deployment("new-name")) + + assert router.get_model_ids(model_name="new-name") == ["rename-1"] + assert router.get_model_ids(model_name="old-name") == [] + + def test_rename_alongside_another_edit_still_updates(self): + router = self._router() + + router.upsert_deployment(deployment=self._deployment("new-name", tpm=1234)) + + assert router.get_model_ids(model_name="new-name") == ["rename-1"] + renamed = router.get_deployment(model_id="rename-1") + assert renamed is not None + assert renamed.litellm_params.tpm == 1234 + + def test_unchanged_deployment_is_still_a_no_op(self): + router = self._router() + + assert router.upsert_deployment(deployment=self._deployment("old-name")) is None + assert [model["model_name"] for model in router.model_list] == ["old-name"] + + class TestConsumedRequestTagsStamp: """Issue #36621: when a request's tags select a tagged pre-routing strategy, those tags are consumed by the selection; the hook must stamp the rewritten model group so diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 209d80df3e2..fabfa0c31d8 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -5850,3 +5850,33 @@ async def test_registered_guardrail_does_not_starve_vector_store_search_results( provider_fields = result.choices[0].message.provider_specific_fields assert provider_fields is not None assert provider_fields["search_results"] == search_results + + +class TestIsVisionExplicitlyDisabled: + """github_copilot and chatgpt run an OAuth device flow inside get_llm_provider; the + explicit-disable lookup must adopt the declared prefix instead of resolving it, exactly + as _supports_factory does, or a capability check on a copilot deployment blocks routing + on a device-code prompt.""" + + @pytest.mark.parametrize("model", ["github_copilot/gpt-4o", "chatgpt/gpt-5"]) + def test_never_resolves_an_authenticating_prefix(self, model, monkeypatch): + from litellm.utils import is_vision_explicitly_disabled + + lookups: list = [] + + def _record(*args, **kwargs): + lookups.append((args, kwargs)) + raise RuntimeError("provider resolution must not run for an authenticating provider") + + monkeypatch.setattr(litellm, "get_llm_provider", _record) + + assert is_vision_explicitly_disabled(model) is False + assert lookups == [] + + def test_explicit_false_detected_and_absent_reads_enabled(self): + from litellm.utils import is_vision_explicitly_disabled + + assert ( + is_vision_explicitly_disabled("fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731") is True + ) + assert is_vision_explicitly_disabled("anthropic/claude-sonnet-4-5") is False diff --git a/type-discipline-budget.json b/type-discipline-budget.json index ab34775c460..6eb7e115586 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,15 +1,15 @@ { "LIT001": { - "limit": 22521 + "limit": 22519 }, "LIT002": { - "limit": 26820 + "limit": 26818 }, "LIT003": { "limit": 269 }, "LIT004": { - "limit": 43 + "limit": 40 }, "LIT005": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16546 + "limit": 16544 }, "LIT011": { "limit": 5575 diff --git a/ui/litellm-dashboard/public/assets/logos/gigachat.svg b/ui/litellm-dashboard/public/assets/logos/gigachat.svg new file mode 100644 index 00000000000..e7abe47b221 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/gigachat.svg @@ -0,0 +1,27 @@ + + + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index ef6e224761d..8b397f20552 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -39,6 +39,35 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ })), })); +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useInfiniteTeams: vi.fn(() => ({ + data: { pages: [{ teams: [{ team_id: "team-eng", team_alias: "engineering" }], page: 1, total_pages: 1 }] }, + isLoading: false, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + })), +})); + +vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ + useInfiniteUsers: vi.fn(() => ({ + data: { + pages: [ + { + users: [{ user_id: "dev-alice", user_alias: null, user_email: "alice@example.com" }], + page: 1, + total_pages: 1, + }, + ], + }, + isPending: false, + isError: false, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + })), +})); + vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn(() => ({ data: [ @@ -60,7 +89,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ })), })); -import ShadowEvalSection, { shadowedKeyLabel } from "./ShadowEvalSection"; +import ShadowEvalSection, { shadowedTargetLabel } from "./ShadowEvalSection"; import { useShadowEvalJob, useShadowEvalJobs, @@ -77,14 +106,15 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ baseline_model: null, judge_model: "anthropic/claude-sonnet-5", shadow_percentage: 10, - keys: [ + targets: [ { - api_key_id: "hashed-key-abc", + target_type: "key", + target_id: "hashed-key-abc", max_turns: 10000, max_budget: 10, spend: 3.21, stopped_at: null, - key_alias: "prod-alpha", + target_alias: "prod-alpha", key_name: "sk-...alpha", }, ], @@ -129,7 +159,6 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ cache_hit_turns: 2, }, ], - by_key: [], overall_shadow_win_rate_pct: 48.0, overall_tie_rate_pct: 22.0, sampled_real_spend: 0.6, @@ -144,17 +173,18 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ ...overrides, }); -const keyEntry = ( - api_key_id: string, - overrides: Partial = {}, -): ShadowEvalJob["keys"][number] => ({ - api_key_id, +const targetEntry = ( + target_id: string, + overrides: Partial = {}, +): ShadowEvalJob["targets"][number] => ({ + target_type: "key", + target_id, max_turns: 10000, max_budget: 10, spend: 0, stopped_at: null, attempt_count: null, - key_alias: null, + target_alias: null, key_name: null, ...overrides, }); @@ -235,8 +265,8 @@ describe("ShadowEvalSection", () => { it("gives every active job its own card with a stop button, with the form still offered", () => { mockHooks({ jobs: [ - job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }), - job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }), + job({ job_id: "job-a", status: "running", targets: [targetEntry("key-a")] }), + job({ job_id: "job-b", status: "running", targets: [targetEntry("key-b")] }), ], }); render(); @@ -347,7 +377,7 @@ describe("ShadowEvalSection", () => { }); it("shows spend without a budget cap for a job from before spend budgets existed", () => { - const j = job({ keys: [keyEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] }); + const j = job({ targets: [targetEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] }); mockHooks({ jobs: [j], detailsById: { "job-1": j } }); render(); expect(screen.getByText(/\$3\.21 eval spend/)).toBeInTheDocument(); @@ -417,6 +447,38 @@ describe("ShadowEvalSection", () => { const expectedBody = { api_key_ids: ["hash-alpha", "hash-beta"], + team_ids: [], + user_ids: [], + router_name: "gpt-auto", + direction: "forward", + shadow_percentage: 10, + duration_days: 7, + max_budget: 10, + judge_model: "anthropic/claude-sonnet-5", + }; + expect(start.mutate).toHaveBeenCalledWith(expectedBody); + }); + + it("submits a team-only job with team_ids and no keys", async () => { + const user = userEvent.setup(); + const { start } = mockHooks({}); + render(); + + expect(screen.getByText("Start shadow eval")).toBeDisabled(); + + await user.click(screen.getByPlaceholderText("Search teams by alias")); + const teamList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(teamList).getByText("engineering")); + await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(await screen.findByText("gpt-auto")); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByText("Start shadow eval")); + + const expectedBody = { + api_key_ids: [], + team_ids: ["team-eng"], + user_ids: [], router_name: "gpt-auto", direction: "forward", shadow_percentage: 10, @@ -453,6 +515,8 @@ describe("ShadowEvalSection", () => { const expectedBody = { api_key_ids: ["hash-alpha"], + team_ids: [], + user_ids: [], router_name: "gpt-auto", direction: "reverse", baseline_model: "prod-claude", @@ -485,9 +549,13 @@ describe("ShadowEvalSection", () => { }); it("labels the shadowed key by alias, then masked name, then truncated hash", () => { - expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha"); - expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); - expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…"); + expect(shadowedTargetLabel(job().targets[0])).toBe("prod-alpha"); + expect(shadowedTargetLabel(targetEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); + expect(shadowedTargetLabel(targetEntry("hashed-key-abc"))).toBe("hashed-key…"); + expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team" }))).toBe("team-eng"); + expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team", target_alias: "engineering" }))).toBe( + "engineering", + ); }); it("breaks results down per key, so one key exhausting its own budget is visible while a sibling runs on", () => { @@ -495,15 +563,12 @@ describe("ShadowEvalSection", () => { jobs: [ job({ judged_count: 205, - keys: [ - keyEntry("hash-spent", { max_budget: 2, spend: 1.5, stopped_at: "2026-08-08T00:00:00Z" }), - keyEntry("hash-hungry", { max_budget: 5, spend: 0.2 }), - ], - results: { - by_tier: [], - by_current_model: [], - by_key: [ - { + targets: [ + targetEntry("hash-spent", { + max_budget: 2, + spend: 1.5, + stopped_at: "2026-08-08T00:00:00Z", + verdicts: { group: "hash-spent", turn_count: 200, real_win_rate_pct: 20.0, @@ -514,7 +579,12 @@ describe("ShadowEvalSection", () => { shadow_spend: 0.5, cache_hit_turns: 0, }, - ], + }), + targetEntry("hash-hungry", { max_budget: 5, spend: 0.2 }), + ], + results: { + by_tier: [], + by_current_model: [], overall_shadow_win_rate_pct: 60.0, overall_tie_rate_pct: 20.0, sampled_real_spend: 0.9, @@ -539,7 +609,7 @@ describe("ShadowEvalSection", () => { expect(screen.getByText(/205 turns judged/)).toBeInTheDocument(); expect(screen.getByText(/Shadowing 10% of/)).toBeInTheDocument(); - expect(screen.getByText("2 keys")).toBeInTheDocument(); + expect(screen.getByText("2 targets")).toBeInTheDocument(); }); it("reads a key that spent its budget as completed even before the sweep stamps it", () => { @@ -547,9 +617,9 @@ describe("ShadowEvalSection", () => { mockHooks({ jobs: [ job({ - keys: [ - keyEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }), - keyEntry("hash-hungry", legacyTurnBudgetLeg), + targets: [ + targetEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }), + targetEntry("hash-hungry", legacyTurnBudgetLeg), ], }), ], @@ -571,9 +641,9 @@ describe("ShadowEvalSection", () => { job({ judged_count: 0, results: null, - keys: [ - keyEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }), - keyEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }), + targets: [ + targetEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }), + targetEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }), ], }), ], @@ -594,9 +664,9 @@ describe("ShadowEvalSection", () => { jobs: [ job({ status: "completed", - keys: [ - keyEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }), - keyEntry("hash-hungry", { max_turns: 500 }), + targets: [ + targetEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }), + targetEntry("hash-hungry", { max_turns: 500 }), ], }), ], diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index 39dee28390a..ec145bc617a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -3,10 +3,13 @@ import React, { useMemo, useState } from "react"; import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; +import TeamMultiSelect from "@/components/common_components/team_multi_select"; +import { userOptionLabel } from "@/components/common_components/UserDropdown"; import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -27,7 +30,7 @@ import { useStartShadowEval, useStopShadowEval, type ShadowEvalJob, - type ShadowEvalJobKey, + type ShadowEvalJobTarget, type ShadowEvalSlice, } from "./useShadowEval"; @@ -66,29 +69,31 @@ const routerMatchedOrBeatPct = ( ? 100 - results.overall_shadow_win_rate_pct : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct; -export const shadowedKeyLabel = (key: ShadowEvalJobKey): string => - key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`; +export const shadowedTargetLabel = (target: ShadowEvalJobTarget): string => + target.target_alias || + target.key_name || + (target.target_type === "key" ? `${target.target_id.slice(0, 10)}…` : target.target_id); -const shadowedKeysLabel = (job: ShadowEvalJob): string => - job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`; +const shadowedTargetsLabel = (job: ShadowEvalJob): string => + job.targets.length === 1 ? shadowedTargetLabel(job.targets[0]) : `${job.targets.length} targets`; const totalBudget = (job: ShadowEvalJob): number | null => - job.keys.reduce( - (sum, key) => (sum === null || key.max_budget == null ? null : sum + key.max_budget), + job.targets.reduce( + (sum, target) => (sum === null || target.max_budget == null ? null : sum + target.max_budget), 0, ); -const totalSpend = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + (key.spend ?? 0), 0); +const totalSpend = (job: ShadowEvalJob): number => job.targets.reduce((sum, target) => sum + (target.spend ?? 0), 0); -const keySpent = (key: ShadowEvalJobKey): boolean => { - const spendBudgetReached = key.max_budget != null && key.spend != null && key.spend >= key.max_budget; - const turnValveReached = key.attempt_count != null && key.attempt_count >= key.max_turns; +const targetSpent = (target: ShadowEvalJobTarget): boolean => { + const spendBudgetReached = target.max_budget != null && target.spend != null && target.spend >= target.max_budget; + const turnValveReached = target.attempt_count != null && target.attempt_count >= target.max_turns; return spendBudgetReached || turnValveReached; }; -const keyStatus = (job: ShadowEvalJob, key: ShadowEvalJobKey): string => { - if (job.status === "completed" || (key.stopped_at == null && keySpent(key))) return "completed"; - return key.stopped_at != null ? "stopped" : "running"; +const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string => { + if (job.status === "completed" || (target.stopped_at == null && targetSpent(target))) return "completed"; + return target.stopped_at != null ? "stopped" : "running"; }; const jobHeadline = (job: ShadowEvalJob): React.ReactNode => @@ -96,12 +101,12 @@ const jobHeadline = (job: ShadowEvalJob): React.ReactNode => <> Comparing {job.router_name} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} - {shadowedKeysLabel(job)} traffic + {shadowedTargetsLabel(job)} traffic ) : ( <> - Shadowing {job.shadow_percentage}% of {shadowedKeysLabel(job)} traffic - via {job.router_name} + Shadowing {job.shadow_percentage}% of {shadowedTargetsLabel(job)}{" "} + traffic via {job.router_name} ); @@ -255,13 +260,12 @@ const VerdictBar: React.FC<{ direction: ShadowEvalDirection; results: NonNullabl ); }; -const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { - const slices = new Map((job.results?.by_key ?? []).map((slice) => [slice.group, slice])); +const TargetTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { return ( - Key + Target Status {["Budget used", "Router wins", `${otherArmLabel(job.direction)} wins`].map((label) => ( @@ -271,18 +275,23 @@ const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { - {job.keys.map((key) => { - const slice = slices.get(key.api_key_id); + {job.targets.map((target) => { + const slice = target.verdicts; return ( - - {shadowedKeyLabel(key)} + + + {shadowedTargetLabel(target)} + {target.target_type !== "key" && ( + {target.target_type} + )} + - + - {key.max_budget != null - ? `${usd(key.spend ?? 0)} / ${usd(key.max_budget)}` - : `${(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${key.max_turns.toLocaleString()} turns`} + {target.max_budget != null + ? `${usd(target.spend ?? 0)} / ${usd(target.max_budget)}` + : `${(target.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${target.max_turns.toLocaleString()} turns`} {slice ? ( <> @@ -318,9 +327,9 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ const hasVerdicts = results != null && (results.by_tier.length > 0 || results.by_current_model.length > 0); return ( <> - {job.keys.length > 1 && ( + {job.targets.length > 1 && (
- +
)} {/* results == null re-stated for TS narrowing; hasVerdicts alone cannot narrow it */} @@ -454,9 +463,9 @@ const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[ const START_FORM_DESCRIPTION: Record = { forward: - "Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own spend budget. The router's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", reverse: - "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", }; const DURATION_OPTIONS = [ @@ -515,9 +524,46 @@ const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => voi ); }; +const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( + 50, + search || undefined, + ); + const options = useMemo( + () => + Array.from( + new Map( + (data?.pages ?? []) + .flatMap((page) => page.users) + .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), + ).values(), + ), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search users by email" + emptyText="No matching users" + errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + const StartForm: React.FC = () => { const { accessToken } = useAuthorized(); const [apiKeyIds, setApiKeyIds] = useState([]); + const [teamIds, setTeamIds] = useState([]); + const [userIds, setUserIds] = useState([]); const [routerName, setRouterName] = useState(""); const [direction, setDirection] = useState("forward"); const [baselineModel, setBaselineModel] = useState(""); @@ -542,12 +588,15 @@ const StartForm: React.FC = () => { const parsedMaxBudget = Number.parseFloat(maxBudget); const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; const baselinePicked = direction === "forward" || baselineModel !== ""; - const filled = apiKeyIds.length > 0 && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; + const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0; + const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; const boundsValid = percentageValid && maxBudgetValid; const valid = Boolean(accessToken) && filled && boundsValid; const handleStart = () => { const startBody = { api_key_ids: apiKeyIds, + team_ids: teamIds, + user_ids: userIds, router_name: routerName, direction, ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), @@ -587,6 +636,12 @@ const StartForm: React.FC = () => { + + + + + + { value={maxBudget} onChange={(e) => setMaxBudget(e.target.value)} /> - max shadow + judge spend, per key + max shadow + judge spend, per target {maxBudget.trim() !== "" && !maxBudgetValid && (

Enter a value from 0.01 to 10000

@@ -774,8 +829,9 @@ const ShadowEvalSection: React.FC = () => {

Shadow eval

- Blind-judge the auto-router on your real traffic: against the models a key uses today before switching, or - against a fixed baseline after it has switched. + Blind-judge the auto-router on the real traffic of a key, team, or user (teams and users cover + JWT-authenticated traffic): against the models they use today before switching, or against a fixed baseline + after they have switched.

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts index eef98320e67..107df0f594a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts @@ -7,7 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"]; -export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"]; +export type ShadowEvalJobTarget = components["schemas"]["ShadowEvalJobTargetResponse"]; export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"]; export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"]; diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 2947e29319b..84f61c95eca 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -111,7 +111,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = How Classification Works {scoringExplanation(value)} {scorerRuns && ranges && ( -
    +
    • {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium}
      • diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 33ce1169c46..218c99e32c5 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -80,6 +80,15 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); }); + it("leaves the score threshold list color to the theme instead of an inline style", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const list = screen.getByText(/Score < 0.15/).closest("ul"); + expect(list).toBeInTheDocument(); + expect(list).toHaveClass("text-muted-foreground"); + expect(list?.style.color).toBe(""); + }); + it("should default to heuristic and hide classifier model/timeout fields", () => { renderWithProviders(); expect(screen.getByText("Advanced: Classification Method")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 153afa0b586..111a7c9f10a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -29,6 +29,7 @@ import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import { Restricted, restrictedBy } from "./TierRestrictions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -392,6 +393,13 @@ export interface ComplexityRouterConfigValue { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + /** + * Context-window escalation gate. Undefined means untouched, which keeps both keys out of the + * payload so the router tracks the backend defaults (enabled, 0.95 buffer); an explicit false + * is a real opt-out and must survive the edit round-trip. + */ + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; /** * Heuristic scorer knobs. Undefined means the operator never touched them, which keeps the key out of the * payload so the router tracks the backend defaults rather than freezing today's numbers. @@ -827,6 +835,11 @@ const ComplexityRouterConfig: React.FC = ({ ), }, + { + key: "context-window", + label: Advanced: Context Window Escalation, + children: , + }, { key: "response", label: Advanced: Response Format, diff --git a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx new file mode 100644 index 00000000000..c0a65076d20 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx @@ -0,0 +1,60 @@ +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const ContextWindowEscalationConfig: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => { + const enabled = value.enable_context_window_escalation ?? true; + // A number input renders Number("0.") as "0", so a decimal cannot be typed without a local draft. + const [bufferDraft, setBufferDraft] = React.useState(null); + const commitBuffer = (raw: string) => { + setBufferDraft(null); + if (raw.trim() === "") { + onChange({ ...value, context_window_escalation_buffer: undefined }); + return; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return; + onChange({ ...value, context_window_escalation_buffer: Math.min(1, Math.max(0.01, parsed)) }); + }; + return ( + <> +
      + onChange({ ...value, enable_context_window_escalation: next })} + aria-label="Escalate oversized prompts to a tier that fits" + /> + Escalate oversized prompts to a tier that fits +
      + + When a prompt provably cannot fit the decided tier's context windows, route it to the lowest tier whose + window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone. + + {enabled && ( +
      + + setBufferDraft(event.target.value)} + onBlur={(event) => commitBuffer(event.target.value)} + /> + + Fraction of a model's window the counted prompt must fit within, above 0 up to 1. Empty tracks the + backend default of 0.95. + +
      + )} + + ); +}; + +export default ContextWindowEscalationConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 01cb41bcb95..71b454dbb06 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -373,6 +373,71 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries a context-window escalation opt-out through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); + expect(toggle).toBeChecked(); + await user.click(toggle); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + enable_context_window_escalation: false, + }); + }); + + it("clamps the context-window buffer to 1 and keeps an untouched buffer out of the payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "1.5" } }); + fireEvent.blur(buffer, { target: { value: "1.5" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; + expect(config).toMatchObject({ context_window_escalation_buffer: 1 }); + expect(config).not.toHaveProperty("enable_context_window_escalation"); + }); + + it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "0.8" } }); + fireEvent.blur(buffer, { target: { value: "0.8" } }); + fireEvent.change(buffer, { target: { value: "" } }); + fireEvent.blur(buffer, { target: { value: "" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty( + "context_window_escalation_buffer", + ); + }); + // The scalar floor is the one scorer knob with no group dict behind it, so its wiring into the create // payload is only proven end to end. 0 is the case a truthy check would silently drop. it("carries a reasoning override floor of 0 through to the create payload", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4e5e5e8d460..ed584a4882b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -367,6 +367,8 @@ const AddAutoRouterTab: React.FC = ({ tokenThresholds: complexityRouterConfig.token_thresholds, dimensionWeights: complexityRouterConfig.dimension_weights, reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, + enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, + contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, }; const submitRecommendedRouter = async (name: string) => { diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 84406a2093e..feddaaa0eac 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -59,6 +59,16 @@ describe("buildComplexityRouterConfig", () => { expect(config).toEqual(expected); }); + it("carries an explicit context-window escalation opt-out and buffer, false included", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + enableContextWindowEscalation: false, + contextWindowEscalationBuffer: 0.9, + }); + expect(config.enable_context_window_escalation).toBe(false); + expect(config.context_window_escalation_buffer).toBe(0.9); + }); + it("trims escalation keywords and drops blank entries", () => { const config = buildComplexityRouterConfig({ ...baseParams, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index ba500d116ce..9a14e956207 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -123,6 +123,8 @@ export interface BuildComplexityRouterConfigParams { dimensionWeights?: DimensionWeights; reasoningOverrideMinScore?: number; tierModelParams?: TierModelParamsByTier; + enableContextWindowEscalation?: boolean; + contextWindowEscalationBuffer?: number; } /** @@ -174,6 +176,8 @@ export interface ComplexityRouterConfigPayload { token_thresholds?: TokenThresholds; dimension_weights?: DimensionWeights; reasoning_override_min_score?: number; + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; tier_model_configs?: Record; } @@ -407,6 +411,8 @@ export const buildComplexityRouterConfig = ({ dimensionWeights, reasoningOverrideMinScore, tierModelParams, + enableContextWindowEscalation, + contextWindowEscalationBuffer, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const serializedTierModelConfigs = customTierSet ? serializeTierModelConfigs( @@ -463,6 +469,12 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), + ...(enableContextWindowEscalation !== undefined && { + enable_context_window_escalation: enableContextWindowEscalation, + }), + ...(contextWindowEscalationBuffer !== undefined && { + context_window_escalation_buffer: contextWindowEscalationBuffer, + }), ...scorerKnobs, }; if (!customTierSet) return payload; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index a7bd4b8eab4..d8c2987ead5 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -487,6 +487,8 @@ describe("managed keys survive an untouched open-and-save", () => { token_thresholds: { simple: 20, complex: 500 }, dimension_weights: { tokenCount: 0.1 }, reasoning_override_min_score: 0.3, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, }; // tier_definitions, fallback_tier and classification_prompt cannot sit beside heuristic_first, which diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 425d5d51f06..4751c2e64b7 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -107,6 +107,8 @@ export interface StoredComplexityRouterConfig { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + enable_context_window_escalation?: unknown; + context_window_escalation_buffer?: unknown; } /** @@ -178,6 +180,14 @@ export const hydrateComplexityRouterConfig = ( tier_distance_penalty: parsedConfig.tier_distance_penalty, adaptive_eligible: parsedConfig.adaptive_eligible || "all", return_raw_model_name: parsedConfig.return_raw_model_name || false, + enable_context_window_escalation: + typeof parsedConfig.enable_context_window_escalation === "boolean" + ? parsedConfig.enable_context_window_escalation + : undefined, + context_window_escalation_buffer: + typeof parsedConfig.context_window_escalation_buffer === "number" + ? parsedConfig.context_window_escalation_buffer + : undefined, }; }; @@ -208,6 +218,8 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "token_thresholds", "dimension_weights", "reasoning_override_min_score", + "enable_context_window_escalation", + "context_window_escalation_buffer", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -307,6 +319,8 @@ export const buildUpdatedComplexityRouterConfig = ( dimensionWeights: value.dimension_weights, reasoningOverrideMinScore: value.reasoning_override_min_score, tierModelParams: value.tier_model_params, + enableContextWindowEscalation: value.enable_context_window_escalation, + contextWindowEscalationBuffer: value.context_window_escalation_buffer, }; const built = buildComplexityRouterConfig(builderParams); diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index b0bb0e8a5b5..c3e1f924d09 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -213,6 +213,20 @@ describe("Sidebar (leftnav)", () => { expect(screen.getByText("Search Tools")).toBeInTheDocument(); }); }); + it("reports whether a nested tab is expanded", async () => { + renderWithProviders(); + + const toggle = screen.getByText("Tools").closest("button")!; + expect(toggle).toHaveAttribute("aria-expanded", "false"); + + act(() => { + fireEvent.click(toggle); + }); + await waitFor(() => { + expect(toggle).toHaveAttribute("aria-expanded", "true"); + }); + }); + it("keeps Router Settings as a single Settings child", () => { // Router Settings is admin-only, so getAvailablePages() filters it out entirely and the // page_utils duplicate-key guard cannot see it. Walk menuGroups directly, otherwise a diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 42389facac2..51ba36348e1 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -570,6 +570,7 @@ const Sidebar_: React.FC = ({ toggleGroup(item.key)} title={collapsed ? labelText(item) : undefined} > diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index c97bb8ede3d..5baa8138960 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -20,6 +20,7 @@ import falAiLogo from "../../public/assets/logos/fal_ai.jpg"; import featherlessLogo from "../../public/assets/logos/featherless.svg"; import fireworksLogo from "../../public/assets/logos/fireworks.svg"; import friendliLogo from "../../public/assets/logos/friendli.svg"; +import gigachatLogo from "../../public/assets/logos/gigachat.svg"; import githubCopilotLogo from "../../public/assets/logos/github_copilot.svg"; import googleLogo from "../../public/assets/logos/google.svg"; import groqLogo from "../../public/assets/logos/groq.svg"; @@ -107,6 +108,7 @@ export enum Providers { FireworksAI = "Fireworks AI", FRIENDLIAI = "Friendliai", GALADRIEL = "Galadriel", + GIGACHAT = "GigaChat", GITHUB_COPILOT = "Github Copilot", Google_AI_Studio = "Google AI Studio", GradientAI = "GradientAI", @@ -218,6 +220,7 @@ export const provider_map: Record = { FireworksAI: "fireworks_ai", FRIENDLIAI: "friendliai", GALADRIEL: "galadriel", + GIGACHAT: "gigachat", GITHUB_COPILOT: "github_copilot", Google_AI_Studio: "gemini", GradientAI: "gradient_ai", @@ -323,6 +326,7 @@ export const providerLogoMap: Partial> = { [Providers.FEATHERLESS_AI]: featherlessLogo.src, [Providers.FireworksAI]: fireworksLogo.src, [Providers.FRIENDLIAI]: friendliLogo.src, + [Providers.GIGACHAT]: gigachatLogo.src, [Providers.GITHUB_COPILOT]: githubCopilotLogo.src, [Providers.Google_AI_Studio]: googleLogo.src, [Providers.Groq]: groqLogo.src, diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index e084fdf37e6..bf6a20a4af8 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -186,6 +186,12 @@ describe("RoutingDecisionCard", () => { expect(screen.queryByText("housekeeping")).not.toBeInTheDocument(); }); + it("labels a modality escalation instead of showing the raw cause token", () => { + render(); + expect(screen.getByText("Escalated for image input")).toBeInTheDocument(); + expect(screen.queryByText("modality_escalation")).not.toBeInTheDocument(); + }); + it("shows the escalation keyword", () => { render( , diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 8c77b2db630..aa1d45a859e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -89,6 +89,8 @@ const CONSTANT_CAUSE_LABELS: Record = { semantic_keyword_match: "Semantic keyword match", session_affinity_pin: "Pinned to session", session_affinity_escalation: "Escalated from session pin", + user_turn_continuation: "Continuation turn, classifier skipped", + modality_escalation: "Escalated for image input", quality_tier: "Quality tier mapping", bandit: "Adaptive bandit", default_fallback: "Default model, no route matched", diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.test.tsx index 5aee6b33ec5..6cd9f476628 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.test.tsx @@ -53,6 +53,24 @@ describe("SectionHeader", () => { expect(onToggleCollapse).toHaveBeenCalledTimes(1); }); + it("reports its collapsed state to assistive technology", () => { + const { rerender } = render( + , + ); + + expect(screen.getByRole("button", { name: /^Input/ })).toHaveAttribute("aria-expanded", "true"); + + rerender(); + + expect(screen.getByRole("button", { name: /^Input/ })).toHaveAttribute("aria-expanded", "false"); + }); + + it("names each copy button for the section it belongs to", () => { + render(); + + expect(screen.getByRole("button", { name: "Copy output" })).toBeInTheDocument(); + }); + it("stays inert when no toggle handler is given", async () => { const onCopy = vi.fn(); render(); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx index 93e9953b2ee..6a18aaf8642 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SectionHeader.tsx @@ -17,6 +17,8 @@ interface SectionHeaderProps { turnCount?: number; } +const SUMMARY_CLASSES = "flex flex-1 items-center gap-4"; + export function SectionHeader({ type, tokens, @@ -26,42 +28,53 @@ export function SectionHeader({ onToggleCollapse, turnCount, }: SectionHeaderProps) { + const summary = ( + <> + {onToggleCollapse && + (isCollapsed ? ( + + ) : ( + + ))} + +
      + {type === "input" ? ( + + ) : ( + ✨ + )} + {type === "input" ? "Input" : "Output"} +
      + + {tokens !== undefined && Tokens: {tokens.toLocaleString()}} + + {cost !== undefined && Cost: ${cost.toFixed(6)}} + + {turnCount !== undefined && turnCount > 0 && ( + Turns: {turnCount} + )} + + ); + return (
      -
      - {onToggleCollapse && - (isCollapsed ? ( - - ) : ( - - ))} - -
      - {type === "input" ? ( - - ) : ( - ✨ - )} - {type === "input" ? "Input" : "Output"} -
      - - {tokens !== undefined && ( - Tokens: {tokens.toLocaleString()} - )} - - {cost !== undefined && Cost: ${cost.toFixed(6)}} - - {turnCount !== undefined && turnCount > 0 && ( - Turns: {turnCount} - )} -
      + {onToggleCollapse ? ( + + ) : ( +
      {summary}
      + )} { e.stopPropagation(); onCopy(); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 2de1ac11db2..f14d6279e32 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -573,6 +573,22 @@ describe("autorouter_presets", () => { expect(prefill.escalationKeywords).toEqual([]); }); + it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => { + const prefill = buildPresetPrefill( + { + tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic", + session_affinity: false, + deployment_affinity: true, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, + }, + groupsOnly(["gpt-5-nano"]), + ); + expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false); + expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); + }); + it("falls back to the defaults when a preset omits match_threshold and escalation_keywords", () => { const prefill = buildPresetPrefill( { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index a35b868db5e..f96dd5ddb4c 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -295,6 +295,8 @@ export const buildPresetPrefill = ( tier_distance_penalty: config.tier_distance_penalty, adaptive_eligible: config.adaptive_eligible, return_raw_model_name: config.return_raw_model_name, + enable_context_window_escalation: config.enable_context_window_escalation, + context_window_escalation_buffer: config.context_window_escalation_buffer, }, customTechnicalKeywords: config.custom_technical_keywords ?? [], keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8a564a07489..fb81191cfbf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1234,8 +1234,8 @@ export interface paths { }; /** * List Shadow Eval Jobs - * @description List shadow eval jobs, newest first, each key with its attempt count so status is - * accurate. Judged counts, spend, and results ride the detail endpoint only. + * @description List shadow eval jobs, newest first, each target with its attempt count so status + * is accurate. Judged counts, spend, and results ride the detail endpoint only. */ get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"]; put?: never; @@ -1257,22 +1257,29 @@ export interface paths { put?: never; /** * Start Shadow Eval - * @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against - * a second arm, judge the two responses blind, and stratify win rates by tier, by the model - * that served the real arm, and by key. + * @description Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic + * against a second arm, judge the two responses blind, and stratify win rates by tier, + * by the model that served the real arm, and by target. * - * A forward job answers whether the keys should adopt router_name: it samples the requests - * the router did not serve and duplicates them through it. A reverse job answers whether a - * key already on the router still gains from it: it samples the requests the router did - * serve and duplicates them against baseline_model. A key can hold one active job per - * direction, so both questions can run at once. + * A target is a virtual key, a team, or a user. Team and user targets match on the + * identity every request resolves to at auth time, so they cover JWT-authenticated + * traffic, which presents no virtual key; a user target samples that user's traffic + * across all their teams, whether it arrives on a JWT or a key they own. * - * Shadow responses are never served to users. Each key samples until its recorded eval - * spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's - * window ends, or the job is stopped, so one key running out of budget does not end - * sampling for the others; sampling changes propagate to pods within about 10 seconds. - * Shadow and judge calls bill to the shadowed key but are excluded from request counts - * and auto-router adoption metrics. + * A forward job answers whether the targets should adopt router_name: it samples the + * requests the router did not serve and duplicates them through it. A reverse job + * answers whether a target already on the router still gains from it: it samples the + * requests the router did serve and duplicates them against baseline_model. A target + * can hold one active job per direction, so both questions can run at once, and a + * request matching several jobs' targets (say its key and its team) is sampled by + * each, separately budgeted. + * + * Shadow responses are never served to users. Each target samples until its recorded + * eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the + * job's window ends, or the job is stopped, so one target running out of budget does + * not end sampling for the others; sampling changes propagate to pods within about 10 + * seconds. Shadow and judge calls bill to the sampled request's own identity but are + * excluded from request counts and auto-router adoption metrics. */ post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; delete?: never; @@ -1312,8 +1319,8 @@ export interface paths { put?: never; /** * Stop Shadow Eval Job - * @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept; - * sampling halts within ~10s. Keys that already stopped on their own budget keep the + * @description Stop an active shadow eval job, every target it scopes at once. Attempts are kept; + * sampling halts within ~10s. Targets that already stopped on their own budget keep the * stopped_at they earned. The statement is the whole state machine: it claims the job * only while a leg still samples inside the window with no stop recorded, so a racing * operator, a same-instant budget spend, and a repeat stop all read the same 400 with @@ -5258,6 +5265,42 @@ export interface paths { patch?: never; trace?: never; }; + "/gigachat/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Gigachat Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + */ + get: operations["gigachat_proxy_route_gigachat__endpoint__get"]; + /** + * Gigachat Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + */ + put: operations["gigachat_proxy_route_gigachat__endpoint__put"]; + /** + * Gigachat Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + */ + post: operations["gigachat_proxy_route_gigachat__endpoint__post"]; + /** + * Gigachat Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + */ + delete: operations["gigachat_proxy_route_gigachat__endpoint__delete"]; + options?: never; + head?: never; + /** + * Gigachat Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/gigachat) + */ + patch: operations["gigachat_proxy_route_gigachat__endpoint__patch"]; + trace?: never; + }; "/global/activity": { parameters: { query?: never; @@ -9700,6 +9743,37 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses/{response_id}": { parameters: { query?: never; @@ -12619,6 +12693,37 @@ export interface paths { patch?: never; trace?: never; }; + "/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/responses/{response_id}": { parameters: { query?: never; @@ -15505,7 +15610,7 @@ export interface paths { * Use this if if you want different teams to have different success/failure callbacks * * Parameters: - * - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add + * - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials * - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of: * - "success": Callback for successful LLM calls * - "failure": Callback for failed LLM calls @@ -15521,6 +15626,8 @@ export interface paths { * - langsmith_api_key: The API key for the Langsmith callback * - langsmith_project: The project for the Langsmith callback * - langsmith_base_url: The base URL for the Langsmith callback + * - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400 + * - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key * * Example curl: * ``` @@ -16210,6 +16317,11 @@ export interface paths { * * Meant to optimize querying spend data for analytics for a user. * + * Reads daily spend records that only ever accumulate and are never affected by budget + * resets. Their total can legitimately exceed the `spend` field returned by + * `/v2/user/info`, which is a running budget counter that every budget reset sets back + * to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). + * * Returns: * (by date) * - spend @@ -16241,6 +16353,11 @@ export interface paths { * Get User Daily Activity Aggregated * @description Aggregated analytics for a user's daily activity without pagination. * Returns the same response shape as the paginated endpoint with page metadata set to single-page. + * + * Reads daily spend records that only ever accumulate and are never affected by budget + * resets. Their total can legitimately exceed the `spend` field returned by + * `/v2/user/info`, which is a running budget counter that every budget reset sets back + * to zero (or to the overage above `max_budget` when `budget_rollover` is enabled). */ get: operations["get_user_daily_activity_aggregated_user_daily_activity_aggregated_get"]; put?: never; @@ -19182,6 +19299,37 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/responses/input_tokens": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Responses Input Tokens + * @description Count the input tokens of a Responses API request without calling the model. + * + * Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens + * + * ```bash + * curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ + * "model": "gpt-4o", + * "input": "Hello, how are you?" + * }' + * ``` + * + * Returns: `{"object": "response.input_tokens", "input_tokens": }` + */ + post: operations["responses_input_tokens_v1_responses_input_tokens_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/responses/{response_id}": { parameters: { query?: never; @@ -21004,6 +21152,14 @@ export interface paths { * This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem * where the old endpoint loaded all keys and teams into memory. * + * Note on `spend`: this is the user's running budget counter, which the budget reset job + * resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default, + * or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT + * lifetime or per-period historical spend. For historical spend over a date range, use + * `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend + * records that only ever accumulate and are never reset. The two values are expected to + * diverge once a budget reset has occurred within the queried period. + * * Access control: * - Proxy admins can query any user * - Team admins can query users within their teams @@ -34169,6 +34325,13 @@ export interface components { adaptive_eligible: "all" | "classified_tier"; /** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */ adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"]; + /** + * Classification Mode + * @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline. + * @default every_request + * @enum {string} + */ + classification_mode: "every_request" | "user_turn"; /** * Classification Prompt * @description Replaces the opening instructions of the LLM classifier rubric (the judging-criteria prose) for a custom tier set. The per-tier bullets and the trust-boundary paragraph telling the classifier to ignore tier requests embedded in quoted caller text are always appended after it and cannot be overridden. Requires tier_definitions; a built-in-tier router customizes its prompt via classifier_llm_config.system_prompt or classification_rubric instead. @@ -34229,6 +34392,12 @@ export interface components { * @description Keywords indicating code-related content */ code_keywords?: string[] | null; + /** + * Context Window Escalation Buffer + * @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens. + * @default 0.95 + */ + context_window_escalation_buffer: number; /** * Custom Technical Keywords * @description Domain-specific technical keywords appended to the effective base list (technical_keywords if set, otherwise DEFAULT_TECHNICAL_KEYWORDS). Order is preserved; duplicates are removed case-insensitively against the base list and within this list. @@ -34257,6 +34426,12 @@ export interface components { * @description Embedding model (LiteLLM model name) used when semantic_keyword_matching is enabled */ embedding_model?: string | null; + /** + * Enable Context Window Escalation + * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Set false to dispatch on complexity alone, as before. + * @default true + */ + enable_context_window_escalation: boolean; /** * Escalation Keywords * @description Case-sensitive phrases a user can include to force a bump to the next-higher complexity tier when they aren't satisfied with results (they can force a stronger model, but not choose which one). Defaults to ['LITELLM ESCALATE'] when unset; set to an empty list to disable. @@ -34288,6 +34463,12 @@ export interface components { * @default 0.5 */ match_threshold: number; + /** + * Modality Routing + * @description Route image-bearing requests only to models that can accept image input. The classifier reads text alone, so an image request whose text classifies cheap otherwise lands on a text-only model and fails with a provider 400. When enabled, a routed model explicitly declared supports_vision false (deployment model_info or the model cost map; unmapped names stay routable) is replaced by the nearest HIGHER tier holding a capable model, then default_model, else a clear 400. A kept session-affinity pin still wins even when an image arrives. + * @default false + */ + modality_routing: boolean; /** * Plan Mode Min Tier * @description When set, requests carrying a coding-agent plan-mode sentinel (Claude Code plan mode, VS Code Copilot Plan mode, Copilot CLI's exit_plan_mode tool) are routed to at least this tier: the classified tier still wins when it is higher, and the floor also overrides a session-affinity pin to a lower tier for exactly the turns carrying the sentinel, without rewriting the pin -- the first turn after plan mode exits routes as if plan mode had never happened. Names a built-in tier, or with tier_definitions set, one of the defined tier names (list order is ascending severity, same as keyword_tier_rules). Unset disables detection entirely. The sentinels ride in client-injected prompt text, so a caller who pastes one can spend up to this tier's models -- never down, and never outside the configured pools. @@ -35094,56 +35275,10 @@ export interface components { /** Timeout */ timeout?: number | null; }; - /** - * ShadowEvalJobKeyResponse - * @description One key a job shadows, with its own budget and stop state. - */ - ShadowEvalJobKeyResponse: { - /** - * Api Key Id - * @description The hashed virtual key whose traffic this entry scopes - */ - api_key_id: string; - /** - * Attempt Count - * @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it - */ - attempt_count?: number | null; - /** - * Key Alias - * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted - */ - key_alias?: string | null; - /** - * Key Name - * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias - */ - key_name?: string | null; - /** - * Max Budget - * @description This key's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds - */ - max_budget?: number | null; - /** - * Max Turns - * @description This key's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise - */ - max_turns: number; - /** - * Spend - * @description This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count - */ - spend?: number | null; - /** - * Stopped At - * @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset - */ - stopped_at?: string | null; - }; /** * ShadowEvalJobResponse - * @description A shadow-eval job over one or more keys, each with its own budget and stop state; - * status is derived from stopped_by, the keys' stop and budget state, and ends_at, + * @description A shadow-eval job over one or more targets, each with its own budget and stop state; + * status is derived from stopped_by, the targets' stop and budget state, and ends_at, * never stored, so no writer anywhere can produce an inconsistent one. Aggregate * fields are populated by the detail endpoint only and stay None on list responses. */ @@ -35185,11 +35320,6 @@ export interface components { * @description Verdicts recorded; detail endpoint only */ judged_count?: number | null; - /** - * Keys - * @description The keys whose traffic this job evaluates, and only those keys', each with its own budget - */ - keys: components["schemas"]["ShadowEvalJobKeyResponse"][]; /** * Last Error * @description Most recent attempt error; detail endpoint only @@ -35205,8 +35335,8 @@ export interface components { * Status * @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration * backfills it for every job that displayed stopped when the column arrived, so the - * pre-column population is closed), completion is the window passing or every key - * spending its budget, and anything else is running. The all-keys-stamped fallback + * pre-column population is closed), completion is the window passing or every target + * spending its budget, and anything else is running. The all-targets-stamped fallback * covers only stops written by pre-column pods during a rolling deploy. * @enum {string} */ @@ -35216,6 +35346,65 @@ export interface components { * @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed */ stopped_by?: string | null; + /** + * Targets + * @description The targets whose traffic this job evaluates, and only theirs, each with its own budget + */ + targets: components["schemas"]["ShadowEvalJobTargetResponse"][]; + }; + /** + * ShadowEvalJobTargetResponse + * @description One target a job shadows (a key, team, or user), with its own budget and stop state. + */ + ShadowEvalJobTargetResponse: { + /** + * Attempt Count + * @description This target's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the target is stamped, so in-flight attempts landing after a stop never reclassify it + */ + attempt_count?: number | null; + /** + * Key Name + * @description Masked display name (sk-...) for key targets, resolved at read time; None for teams and users + */ + key_name?: string | null; + /** + * Max Budget + * @description This target's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds + */ + max_budget?: number | null; + /** + * Max Turns + * @description This target's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise + */ + max_turns: number; + /** + * Spend + * @description This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count + */ + spend?: number | null; + /** + * Stopped At + * @description When this target's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset + */ + stopped_at?: string | null; + /** + * Target Alias + * @description Display label resolved from the target's own row at read time: the key's alias, the team's alias, or the user's email; None when unset or deleted + */ + target_alias?: string | null; + /** + * Target Id + * @description The hashed virtual key, team id, or user id whose traffic this entry scopes + */ + target_id: string; + /** + * Target Type + * @description What kind of entity this entry scopes + * @enum {string} + */ + target_type: "key" | "team" | "user"; + /** @description This target's own judged-verdict slice; detail endpoint only, None until a turn is judged */ + verdicts?: components["schemas"]["ShadowEvalSlice"] | null; }; /** * ShadowEvalResult @@ -35227,11 +35416,6 @@ export interface components { * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; - /** - * By Key - * @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero - */ - by_key: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** @@ -35273,8 +35457,9 @@ export interface components { }; /** * ShadowEvalSlice - * @description Judge outcomes for one slice of a job's verdicts (a router tier, or one of the - * models that served the real arm). + * @description Judge outcomes for one slice of a job's verdicts: a router tier, one of the + * models that served the real arm, or one scoped target (embedded on that target's + * own entry, so slices never need re-joining to a target by id). */ ShadowEvalSlice: { /** Avg Judge Confidence */ @@ -35441,11 +35626,15 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ classifier_model?: string; + /** Context Escalated */ + context_escalated?: boolean; + /** Context Escalation Original Tier */ + context_escalation_original_tier?: string; /** Conversation Continuing */ conversation_continuing?: boolean; /** Escalated */ @@ -35500,12 +35689,18 @@ export interface components { }; /** * StartShadowEvalRequest - * @description Start duplicating one or more keys' traffic for blind comparison against an auto-router. + * @description Start duplicating one or more targets' traffic for blind comparison against an auto-router. + * + * A target is a virtual key, a team, or a user; each becomes its own leg with its own + * budget and stop state. Team and user targets match on the identity every request + * carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover + * JWT-authenticated traffic, which presents no virtual key at all. */ StartShadowEvalRequest: { /** * Api Key Ids - * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. + * @description Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job needs at least one target and at most 100, which also bounds every read the job's endpoints make. Each target carries its own max_budget spend budget, so one exhausting its budget leaves the others sampling. + * @default [] */ api_key_ids: string[]; /** @@ -35534,7 +35729,7 @@ export interface components { judge_model: string; /** * Max Budget - * @description Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window + * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window * @default 10 */ max_budget: number; @@ -35545,9 +35740,21 @@ export interface components { router_name: string; /** * Shadow Percentage - * @description Percentage of the key's requests to duplicate through the router + * @description Percentage of each target's requests to duplicate through the router */ shadow_percentage: number; + /** + * Team Ids + * @description Teams whose traffic will be shadowed, matched on the team every authenticated request resolves to, so a team's JWT-auth and virtual-key traffic are both sampled + * @default [] + */ + team_ids: string[]; + /** + * User Ids + * @description Users whose traffic will be shadowed, matched on the user every authenticated request resolves to across all their teams: JWT requests carrying their subject claim and virtual keys they own + * @default [] + */ + user_ids: string[]; }; /** * SuccessfulKeyUpdate @@ -38418,6 +38625,8 @@ export interface components { input_cost_per_character?: number | null; /** Input Cost Per Token */ input_cost_per_token?: number | null; + /** Internal Router Model */ + internal_router_model?: boolean | null; /** Output Cost Per Character */ output_cost_per_character?: number | null; /** Output Cost Per Token */ @@ -40507,8 +40716,10 @@ export interface operations { list_shadow_eval_jobs_auto_router_shadow_eval_get: { parameters: { query?: { - /** @description Filter to jobs that shadow this key, alone or alongside others */ - api_key_id?: string | null; + /** @description Kind of target to filter on; requires target_id */ + target_type?: ("key" | "team" | "user") | null; + /** @description Filter to jobs that shadow this target, alone or alongside others */ + target_id?: string | null; /** @description Newest jobs to return */ limit?: number; }; @@ -46435,6 +46646,161 @@ export interface operations { }; }; }; + gigachat_proxy_route_gigachat__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + gigachat_proxy_route_gigachat__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + gigachat_proxy_route_gigachat__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + gigachat_proxy_route_gigachat__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + gigachat_proxy_route_gigachat__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_global_activity_global_activity_get: { parameters: { query?: { @@ -51474,6 +51840,26 @@ export interface operations { }; }; }; + responses_input_tokens_openai_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_openai_v1_responses__response_id__get: { parameters: { query?: never; @@ -54438,6 +54824,26 @@ export interface operations { }; }; }; + responses_input_tokens_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_responses__response_id__get: { parameters: { query?: never; @@ -62860,6 +63266,26 @@ export interface operations { }; }; }; + responses_input_tokens_v1_responses_input_tokens_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_response_v1_responses__response_id__get: { parameters: { query?: never; diff --git a/uv.lock b/uv.lock index 8ef72116466..7df1a35b28a 100644 --- a/uv.lock +++ b/uv.lock @@ -4306,6 +4306,8 @@ extra-proxy = [ { name = "google-cloud-iam" }, { name = "google-cloud-kms" }, { name = "prisma" }, + { name = "psycopg" }, + { name = "psycopg-binary" }, { name = "redisvl" }, { name = "resend" }, ] @@ -4544,6 +4546,8 @@ requires-dist = [ { name = "polars", marker = "extra == 'proxy'", specifier = ">=1.38.1,<2.0" }, { name = "prisma", marker = "extra == 'extra-proxy'", specifier = ">=0.11.0,<1.0" }, { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = ">=0.20.0,<1.0" }, + { name = "psycopg", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" }, + { name = "psycopg-binary", marker = "extra == 'extra-proxy'", specifier = ">=3.2,<4.0" }, { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" },