mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
merge: litellm_internal_staging into litellm_fix_nova_sonic_realtime_user_asr_usage
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI
commit
6809d537f0
54 changed files with 3072 additions and 713 deletions
230
.github/scripts/close_duplicate_issues.py
vendored
230
.github/scripts/close_duplicate_issues.py
vendored
|
|
@ -1,230 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Detect and close duplicate GitHub issues using title similarity.
|
||||
|
||||
Modes:
|
||||
--scan Compare all open issues against each other (batch)
|
||||
--issue-number N Check a single issue against older open issues
|
||||
|
||||
Requires the `gh` CLI to be authenticated.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
"""Strip common prefixes, lowercase, and collapse whitespace."""
|
||||
title = re.sub(
|
||||
r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
|
||||
"",
|
||||
title,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
return " ".join(title.lower().split())
|
||||
|
||||
|
||||
def gh(*args: str) -> str:
|
||||
"""Run a gh CLI command and return stdout."""
|
||||
result = subprocess.run(
|
||||
["gh", *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def fetch_open_issues(repo: str | None) -> list[dict]:
|
||||
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
|
||||
if repo:
|
||||
endpoint = (
|
||||
f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
|
||||
)
|
||||
else:
|
||||
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
|
||||
cmd = ["api", "--paginate", endpoint]
|
||||
|
||||
raw = gh(*cmd)
|
||||
# gh --paginate concatenates JSON arrays, so we may get multiple arrays
|
||||
issues = []
|
||||
for line in raw.strip().splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parsed = json.loads(line)
|
||||
if isinstance(parsed, list):
|
||||
issues.extend(parsed)
|
||||
else:
|
||||
issues.append(parsed)
|
||||
|
||||
# Filter out pull requests (they also appear in the issues endpoint)
|
||||
return [i for i in issues if "pull_request" not in i]
|
||||
|
||||
|
||||
def close_as_duplicate(
|
||||
issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
|
||||
) -> None:
|
||||
"""Close an issue as duplicate of another, adding a comment and label."""
|
||||
repo_args = ["--repo", repo] if repo else []
|
||||
|
||||
if dry_run:
|
||||
print(
|
||||
f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
|
||||
)
|
||||
return
|
||||
|
||||
# Add comment
|
||||
comment_body = (
|
||||
f"Closing as duplicate of #{duplicate_of}.\n\n"
|
||||
"If you believe this is not a duplicate, please reopen and add context "
|
||||
"explaining how this differs."
|
||||
)
|
||||
gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
|
||||
|
||||
# Add label
|
||||
gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
|
||||
|
||||
# Close with not_planned reason
|
||||
gh(
|
||||
"api",
|
||||
f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
|
||||
"-X",
|
||||
"PATCH",
|
||||
"-f",
|
||||
"state=closed",
|
||||
"-f",
|
||||
"state_reason=not_planned",
|
||||
)
|
||||
|
||||
print(f" Closed #{issue_number} as duplicate of #{duplicate_of}")
|
||||
|
||||
|
||||
def find_duplicate(
|
||||
issue: dict, candidates: list[dict], threshold: float
|
||||
) -> dict | None:
|
||||
"""Return the first candidate whose normalized title is above threshold."""
|
||||
norm = normalize_title(issue["title"])
|
||||
for candidate in candidates:
|
||||
if candidate["number"] == issue["number"]:
|
||||
continue
|
||||
cand_norm = normalize_title(candidate["title"])
|
||||
ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
|
||||
if ratio >= threshold:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def scan_all(
|
||||
issues: list[dict], threshold: float, repo: str | None, dry_run: bool
|
||||
) -> int:
|
||||
"""Compare every issue against all older issues. Returns count of duplicates found."""
|
||||
# Sort oldest first
|
||||
issues.sort(key=lambda i: i["number"])
|
||||
closed_count = 0
|
||||
|
||||
for idx, issue in enumerate(issues):
|
||||
older = issues[:idx]
|
||||
if not older:
|
||||
continue
|
||||
dup = find_duplicate(issue, older, threshold)
|
||||
if dup:
|
||||
ratio = difflib.SequenceMatcher(
|
||||
None,
|
||||
normalize_title(issue["title"]),
|
||||
normalize_title(dup["title"]),
|
||||
).ratio()
|
||||
print(
|
||||
f"#{issue['number']}: \"{issue['title']}\"\n"
|
||||
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
|
||||
f"({ratio:.0%} similar)"
|
||||
)
|
||||
close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
|
||||
closed_count += 1
|
||||
|
||||
return closed_count
|
||||
|
||||
|
||||
def check_single(
|
||||
issue_number: int,
|
||||
issues: list[dict],
|
||||
threshold: float,
|
||||
repo: str | None,
|
||||
dry_run: bool,
|
||||
) -> bool:
|
||||
"""Check a single issue against all older open issues. Returns True if duplicate found."""
|
||||
target = None
|
||||
for i in issues:
|
||||
if i["number"] == issue_number:
|
||||
target = i
|
||||
break
|
||||
|
||||
if target is None:
|
||||
print(f"Issue #{issue_number} not found among open issues.")
|
||||
return False
|
||||
|
||||
older = [i for i in issues if i["number"] < issue_number]
|
||||
dup = find_duplicate(target, older, threshold)
|
||||
if dup:
|
||||
ratio = difflib.SequenceMatcher(
|
||||
None,
|
||||
normalize_title(target["title"]),
|
||||
normalize_title(dup["title"]),
|
||||
).ratio()
|
||||
print(
|
||||
f"#{target['number']}: \"{target['title']}\"\n"
|
||||
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
|
||||
f"({ratio:.0%} similar)"
|
||||
)
|
||||
close_as_duplicate(issue_number, dup["number"], repo, dry_run)
|
||||
return True
|
||||
|
||||
print(f"#{issue_number}: no duplicate found above threshold {threshold}")
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Detect and close duplicate GitHub issues"
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
|
||||
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
|
||||
parser.add_argument(
|
||||
"--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--close",
|
||||
action="store_true",
|
||||
help="Actually close duplicates (default is dry-run)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
dry_run = not args.close
|
||||
|
||||
if dry_run:
|
||||
print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
|
||||
|
||||
print("Fetching open issues...")
|
||||
issues = fetch_open_issues(args.repo)
|
||||
print(f"Found {len(issues)} open issues.\n")
|
||||
|
||||
if args.scan:
|
||||
count = scan_all(issues, args.threshold, args.repo, dry_run)
|
||||
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
|
||||
else:
|
||||
found = check_single(
|
||||
args.issue_number, issues, args.threshold, args.repo, dry_run
|
||||
)
|
||||
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
69
.github/workflows/auto-close-duplicates.yml
vendored
Normal file
69
.github/workflows/auto-close-duplicates.yml
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
name: Auto-close duplicate issues
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Log which issues would close without closing anything
|
||||
type: boolean
|
||||
default: true
|
||||
grace_period_days:
|
||||
description: Days a duplicate notice must go unanswered before the close
|
||||
type: number
|
||||
default: 3
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/auto-close-duplicates.yml
|
||||
- scripts/auto-close-duplicates.ts
|
||||
- scripts/auto-close-duplicates.test.ts
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Test the sweep
|
||||
run: bun test scripts/auto-close-duplicates.test.ts
|
||||
|
||||
sweep:
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
# Exact version, never latest: the next step holds an issues: write token
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Close unanswered duplicates, reopen ones the reporter answered
|
||||
run: bun run scripts/auto-close-duplicates.ts
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DRY_RUN: ${{ inputs.dry_run == true }}
|
||||
GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }}
|
||||
40
.github/workflows/check_duplicate_issues.yml
vendored
40
.github/workflows/check_duplicate_issues.yml
vendored
|
|
@ -1,12 +1,19 @@
|
|||
name: Check Duplicate Issues
|
||||
|
||||
# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later,
|
||||
# and only when its title is identical to an older open issue and nobody replied.
|
||||
# The HTML marker below is the handshake between the two, so keep it in the template.
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-duplicate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
|
@ -19,35 +26,12 @@ jobs:
|
|||
threshold: 0.6
|
||||
reaction: eyes
|
||||
comment: |
|
||||
**⚠️ Potential duplicate detected**
|
||||
<!-- litellm:potential-duplicate candidates={{#issues}}{{number}},{{/issues}} -->
|
||||
**Potential duplicate detected**
|
||||
|
||||
This issue appears similar to existing issue(s):
|
||||
This looks similar to:
|
||||
{{#issues}}
|
||||
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
|
||||
- #{{number}} - {{title}}
|
||||
{{/issues}}
|
||||
|
||||
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
|
||||
|
||||
- name: Checkout close script
|
||||
if: github.event.action == 'opened'
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
if: github.event.action == 'opened'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Auto-close if high-confidence duplicate
|
||||
if: github.event.action == 'opened'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python3 .github/scripts/close_duplicate_issues.py \
|
||||
--issue-number ${{ github.event.issue.number }} \
|
||||
--repo ${{ github.repository }} \
|
||||
--threshold 0.85 \
|
||||
--close
|
||||
If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open.
|
||||
|
|
|
|||
18
Dockerfile
18
Dockerfile
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
|
|||
RUN apk add --no-cache \
|
||||
bash \
|
||||
gcc \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
rust \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
|
|
@ -49,10 +49,6 @@ RUN apk add --no-cache \
|
|||
npm \
|
||||
libsndfile
|
||||
|
||||
# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
|
||||
# uv resolves requires-python to the newest allowed minor, downloads a managed
|
||||
# interpreter under /root/.local/share/uv that the runtime stage never
|
||||
# receives, and the copied venv's python symlink dangles at runtime.
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_DOWNLOADS=0 \
|
||||
|
|
@ -70,7 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
# Copy full source tree
|
||||
COPY . .
|
||||
|
|
@ -91,7 +87,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
|
|
@ -106,7 +102,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
|
||||
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:${PATH}" \
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra proxy-runtime \
|
||||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
# Stage 2 — copy source and install the project + workspace members.
|
||||
COPY . .
|
||||
|
|
@ -57,7 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra proxy-runtime \
|
||||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
|
|
@ -71,7 +71,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"limit": 16171
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2226
|
||||
"limit": 2224
|
||||
},
|
||||
"reportAssignmentType": {
|
||||
"limit": 319
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -39,18 +39,14 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
|
|||
RUN apk add --no-cache \
|
||||
bash \
|
||||
gcc \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
libsndfile
|
||||
|
||||
# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
|
||||
# uv resolves requires-python to the newest allowed minor, downloads a managed
|
||||
# interpreter under /root/.local/share/uv that the runtime stage never
|
||||
# receives, and the copied venv's python symlink dangles at runtime.
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_DOWNLOADS=0 \
|
||||
|
|
@ -68,7 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
# Copy full source tree
|
||||
COPY . .
|
||||
|
|
@ -89,7 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
|
|
@ -103,7 +99,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
|
||||
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:${PATH}" \
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG PROXY_EXTRAS_SOURCE=published
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
|
|
@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
|
|||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
gcc \
|
||||
rust \
|
||||
bash \
|
||||
|
|
@ -50,10 +50,6 @@ RUN for i in 1 2 3; do \
|
|||
npm && break || sleep 5; \
|
||||
done
|
||||
|
||||
# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
|
||||
# uv resolves requires-python to the newest allowed minor, downloads a managed
|
||||
# interpreter under /root/.local/share/uv that the runtime stage never
|
||||
# receives, and the copied venv's python symlink dangles at runtime.
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_DOWNLOADS=0 \
|
||||
|
|
@ -74,7 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
# Copy full source tree
|
||||
COPY . .
|
||||
|
|
@ -101,7 +97,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3 \
|
||||
--python python3.13 \
|
||||
--no-sources-package litellm-proxy-extras; \
|
||||
else \
|
||||
uv sync --frozen --no-default-groups --no-editable \
|
||||
|
|
@ -110,7 +106,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--python python3; \
|
||||
--python python3.13; \
|
||||
fi
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
|
|
@ -129,7 +125,7 @@ RUN for i in 1 2 3; do \
|
|||
apk upgrade --no-cache && break || sleep 5; \
|
||||
done && \
|
||||
for i in 1 2 3; do \
|
||||
apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
|
||||
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
|
||||
done
|
||||
|
||||
# Copy only what runtime needs. The application is installed inside the venv;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra bedrock-realtime \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
# Stage 2 — copy source and install the project + workspace members.
|
||||
COPY . .
|
||||
|
|
@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra bedrock-realtime \
|
||||
--python python3
|
||||
--python python3.13
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
|
|
@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ class GenAIMapper:
|
|||
GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
|
||||
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
|
||||
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
|
||||
GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens,
|
||||
GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens,
|
||||
Error.TYPE: lambda d: d.error.error_type if d.error else None,
|
||||
Server.ADDRESS: lambda d: d.server.address if d.server else None,
|
||||
Server.PORT: lambda d: d.server.port if d.server else None,
|
||||
|
|
|
|||
|
|
@ -95,6 +95,22 @@ class LLMUsage:
|
|||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
cache_creation_input_tokens: int | None = None
|
||||
cache_read_input_tokens: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage:
|
||||
# Cache token counts only exist on the raw provider usage object under metadata
|
||||
metadata: Final[Mapping[str, object]] = payload.get("metadata") or {}
|
||||
raw_usage: Final = metadata.get("usage_object")
|
||||
usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {}
|
||||
return cls(
|
||||
input_tokens=as_int(payload.get("prompt_tokens")),
|
||||
output_tokens=as_int(payload.get("completion_tokens")),
|
||||
total_tokens=as_int(payload.get("total_tokens")),
|
||||
cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")),
|
||||
cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -363,11 +379,7 @@ class LLMCallSpanData:
|
|||
response_model=context.response_model,
|
||||
response_id=as_str(response.get("id")),
|
||||
request_params=LLMRequestParams.from_model_parameters(params),
|
||||
usage=LLMUsage(
|
||||
input_tokens=as_int(payload.get("prompt_tokens")),
|
||||
output_tokens=as_int(payload.get("completion_tokens")),
|
||||
total_tokens=as_int(payload.get("total_tokens")),
|
||||
),
|
||||
usage=LLMUsage.from_standard_logging_payload(payload),
|
||||
finish_reasons=finish_reasons,
|
||||
error=_parse_error(payload),
|
||||
response_cost=as_float(payload.get("response_cost")),
|
||||
|
|
|
|||
|
|
@ -110,6 +110,8 @@ class GenAI:
|
|||
# usage
|
||||
USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
|
||||
USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
|
||||
USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens"
|
||||
USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens"
|
||||
# content (opt-in, gated by capture mode)
|
||||
INPUT_MESSAGES: Final = "gen_ai.input.messages"
|
||||
OUTPUT_MESSAGES: Final = "gen_ai.output.messages"
|
||||
|
|
|
|||
|
|
@ -693,6 +693,26 @@ def _count_document_tokens(
|
|||
)
|
||||
|
||||
|
||||
def _count_file_tokens(
|
||||
file_value: object,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
) -> int:
|
||||
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
return 0
|
||||
filename: Final = file_value.get("filename")
|
||||
file_data: Final = file_value.get("file_data")
|
||||
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
|
||||
if not isinstance(file_data, str) or not file_data:
|
||||
return name_tokens
|
||||
return name_tokens + calculate_img_tokens(
|
||||
data=file_data,
|
||||
mode="auto",
|
||||
use_default_image_token_count=use_default_image_token_count,
|
||||
)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
content: Mapping[str, Any],
|
||||
count_function: TokenCounterFunction,
|
||||
|
|
@ -778,6 +798,12 @@ def _count_content_list(
|
|||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
)
|
||||
elif c["type"] == "file":
|
||||
num_tokens += _count_file_tokens(
|
||||
c.get("file"),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
c,
|
||||
|
|
@ -807,7 +833,7 @@ def _count_content_list(
|
|||
raise ValueError(
|
||||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -1631,6 +1631,11 @@ class AmazonConverseConfig(BaseConfig):
|
|||
bedrock_tool_config["toolChoice"] = tool_choice_values
|
||||
self._drop_tool_choice_type_conflicting_with_tool_config(additional_request_params)
|
||||
|
||||
config_block_entries: Final = tuple(
|
||||
(config_name, config_class, inference_params.pop(config_name, None))
|
||||
for config_name, config_class in self.get_config_blocks().items()
|
||||
)
|
||||
|
||||
data: Final[CommonRequestObject] = {
|
||||
"inferenceConfig": self._transform_inference_params(inference_params=inference_params),
|
||||
}
|
||||
|
|
@ -1641,9 +1646,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if system_content_blocks:
|
||||
data["system"] = system_content_blocks
|
||||
|
||||
# Handle all config blocks
|
||||
for config_name, config_class in self.get_config_blocks().items():
|
||||
config_value = inference_params.pop(config_name, None)
|
||||
for config_name, config_class, config_value in config_block_entries:
|
||||
if config_value is not None:
|
||||
data[config_name] = config_class(**config_value)
|
||||
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ class BaseOpenAILLM:
|
|||
"max_retries",
|
||||
"organization",
|
||||
"api_base",
|
||||
"workload_identity_config",
|
||||
)
|
||||
openai_client_fields: Final = (
|
||||
BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type=client_type)
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from .common_utils import (
|
|||
drop_params_from_unprocessable_entity_error,
|
||||
is_output_token_limit_error,
|
||||
)
|
||||
from .workload_identity import resolve_openai_workload_identity_config
|
||||
|
||||
openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
|
||||
openAIGPT5Config: Final = OpenAIGPT5Config()
|
||||
|
|
@ -349,6 +350,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
client: OpenAI | AsyncOpenAI | None = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> OpenAI | AsyncOpenAI | None:
|
||||
workload_identity_config: Final = resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base)
|
||||
client_initialization_params: Final[dict] = locals()
|
||||
if client is None:
|
||||
if not isinstance(max_retries, int):
|
||||
|
|
@ -364,28 +366,49 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
if cached_client:
|
||||
if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI):
|
||||
return cached_client
|
||||
http_client: Final[httpx.Client | httpx.AsyncClient | None] = (
|
||||
OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
|
||||
if is_async
|
||||
else OpenAIChatCompletion._get_sync_http_client()
|
||||
)
|
||||
if is_async:
|
||||
_new_client: OpenAI | AsyncOpenAI = AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
base_url=api_base,
|
||||
http_client=http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
async_http_client: Final = OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
|
||||
http_client: httpx.Client | httpx.AsyncClient | None = async_http_client
|
||||
_new_client: OpenAI | AsyncOpenAI = (
|
||||
AsyncOpenAI(
|
||||
workload_identity=workload_identity_config.to_sdk_workload_identity(),
|
||||
base_url=api_base,
|
||||
http_client=async_http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
)
|
||||
if workload_identity_config is not None
|
||||
else AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
base_url=api_base,
|
||||
http_client=async_http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
)
|
||||
)
|
||||
else:
|
||||
_new_client = OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=api_base,
|
||||
http_client=http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
sync_http_client: Final = OpenAIChatCompletion._get_sync_http_client()
|
||||
http_client = sync_http_client
|
||||
_new_client = (
|
||||
OpenAI(
|
||||
workload_identity=workload_identity_config.to_sdk_workload_identity(),
|
||||
base_url=api_base,
|
||||
http_client=sync_http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
)
|
||||
if workload_identity_config is not None
|
||||
else OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=api_base,
|
||||
http_client=sync_http_client,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
organization=organization,
|
||||
)
|
||||
)
|
||||
|
||||
## SAVE CACHE KEY
|
||||
|
|
|
|||
|
|
@ -4,7 +4,100 @@ OpenAI Responses API token counting transformation logic.
|
|||
This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
|
||||
"""
|
||||
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
||||
class ResponsesInputTextPart(TypedDict):
|
||||
type: ReadOnly[Literal["input_text"]]
|
||||
text: ReadOnly[str]
|
||||
|
||||
|
||||
class ResponsesInputImagePart(TypedDict):
|
||||
type: ReadOnly[Literal["input_image"]]
|
||||
image_url: ReadOnly[str]
|
||||
detail: ReadOnly[str]
|
||||
|
||||
|
||||
class ResponsesInputFilePart(TypedDict):
|
||||
type: ReadOnly[Literal["input_file"]]
|
||||
filename: ReadOnly[str]
|
||||
file_data: ReadOnly[str]
|
||||
|
||||
|
||||
ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart
|
||||
|
||||
ResponsesContentRole = Literal["user", "assistant"]
|
||||
|
||||
|
||||
def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
|
||||
url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
|
||||
if not isinstance(url, str) or not url:
|
||||
return None
|
||||
detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None
|
||||
part: Final[ResponsesInputImagePart] = {
|
||||
"type": "input_image",
|
||||
"image_url": url,
|
||||
"detail": detail if isinstance(detail, str) and detail else "auto",
|
||||
}
|
||||
return part
|
||||
|
||||
|
||||
def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None:
|
||||
"""Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
return None
|
||||
filename: Final = file_value.get("filename")
|
||||
file_data: Final = file_value.get("file_data")
|
||||
if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data:
|
||||
return None
|
||||
part: Final[ResponsesInputFilePart] = {
|
||||
"type": "input_file",
|
||||
"filename": filename,
|
||||
"file_data": file_data,
|
||||
}
|
||||
return part
|
||||
|
||||
|
||||
def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
|
||||
if isinstance(block, str):
|
||||
bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
|
||||
return bare
|
||||
if not isinstance(block, Mapping):
|
||||
return None
|
||||
match block.get("type"):
|
||||
case "text":
|
||||
text_value: Final = block.get("text")
|
||||
text: Final[ResponsesInputTextPart] = {
|
||||
"type": "input_text",
|
||||
"text": text_value if isinstance(text_value, str) else "",
|
||||
}
|
||||
return text
|
||||
case "image_url" if role == "user":
|
||||
return _chat_image_block_to_responses_part(block.get("image_url"))
|
||||
case "file" if role == "user":
|
||||
return _chat_file_block_to_responses_part(block.get("file"))
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def chat_content_blocks_to_responses_content(
|
||||
content: Sequence[object],
|
||||
role: ResponsesContentRole,
|
||||
) -> str | tuple[ResponsesInputPart, ...]:
|
||||
"""Text-only content collapses to a joined string, which every role accepts and counts identically.
|
||||
|
||||
Only a user turn may carry an image or file part: the Responses API rejects any part but
|
||||
output_text and refusal inside an assistant turn.
|
||||
"""
|
||||
parts: Final = tuple(
|
||||
part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None
|
||||
)
|
||||
if any(part["type"] != "input_text" for part in parts):
|
||||
return parts
|
||||
return "\n".join(part["text"] for part in parts if part["type"] == "input_text")
|
||||
|
||||
|
||||
class OpenAICountTokensConfig:
|
||||
|
|
@ -120,18 +213,13 @@ class OpenAICountTokensConfig:
|
|||
instructions_parts.append("\n".join(text_parts))
|
||||
elif role == "user":
|
||||
if isinstance(content, list):
|
||||
# Extract text from content blocks for Responses API
|
||||
text_parts = []
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
text_parts.append(block.get("text", ""))
|
||||
elif isinstance(block, str):
|
||||
text_parts.append(block)
|
||||
content = "\n".join(text_parts)
|
||||
content = chat_content_blocks_to_responses_content(content, "user")
|
||||
input_items.append({"role": "user", "content": content})
|
||||
elif role == "assistant":
|
||||
# Map tool_calls to Responses API function_call items
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if isinstance(content, list):
|
||||
content = chat_content_blocks_to_responses_content(content, "assistant")
|
||||
if content:
|
||||
input_items.append({"role": "assistant", "content": content})
|
||||
if tool_calls:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
||||
|
|
@ -392,6 +393,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
litellm_params = litellm_params or GenericLiteLLMParams()
|
||||
api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
|
||||
headers.setdefault("Content-Type", "application/json")
|
||||
workload_identity_config: Final = (
|
||||
resolve_openai_workload_identity_config(api_key=api_key, api_base=litellm_params.api_base)
|
||||
if self.custom_llm_provider is LlmProviders.OPENAI
|
||||
else None
|
||||
)
|
||||
if workload_identity_config is not None:
|
||||
headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}"
|
||||
return headers
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
|
|
|
|||
100
litellm/llms/openai/workload_identity.py
Normal file
100
litellm/llms/openai/workload_identity.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
|
||||
|
||||
from .common_utils import OpenAIError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
|
||||
|
||||
OPENAI_WIF_CLIENT_ID: Final = "litellm"
|
||||
_OPENAI_API_HOST: Final = "api.openai.com"
|
||||
_SDK_UPGRADE_MESSAGE: Final = (
|
||||
"OpenAI workload identity federation requires openai>=2.32.0. "
|
||||
"Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
|
||||
"OPENAI_SERVICE_ACCOUNT_ID / OPENAI_IDENTITY_TOKEN_FILE."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenAIWorkloadIdentityConfig:
|
||||
identity_provider_id: str
|
||||
service_account_id: str
|
||||
token_file: str
|
||||
|
||||
def to_sdk_workload_identity(self) -> WorkloadIdentity:
|
||||
k8s_token_provider: Final = _load_sdk_k8s_token_provider()
|
||||
workload_identity: Final[WorkloadIdentity] = {
|
||||
"client_id": OPENAI_WIF_CLIENT_ID,
|
||||
"identity_provider_id": self.identity_provider_id,
|
||||
"service_account_id": self.service_account_id,
|
||||
"provider": k8s_token_provider(self.token_file),
|
||||
}
|
||||
return workload_identity
|
||||
|
||||
|
||||
def resolve_openai_workload_identity_config(
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
) -> OpenAIWorkloadIdentityConfig | None:
|
||||
static_api_key: Final = normalize_nonempty_secret_str(api_key) or normalize_nonempty_secret_str(
|
||||
get_secret_str("OPENAI_API_KEY")
|
||||
)
|
||||
if static_api_key is not None:
|
||||
return None
|
||||
effective_api_base: Final = (
|
||||
api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
|
||||
)
|
||||
if not _targets_openai_api(effective_api_base):
|
||||
return None
|
||||
identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID")
|
||||
service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID")
|
||||
token_file: Final = get_secret_str("OPENAI_IDENTITY_TOKEN_FILE")
|
||||
if not identity_provider_id or not service_account_id or not token_file:
|
||||
return None
|
||||
return OpenAIWorkloadIdentityConfig(
|
||||
identity_provider_id=identity_provider_id,
|
||||
service_account_id=service_account_id,
|
||||
token_file=token_file,
|
||||
)
|
||||
|
||||
|
||||
def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) -> str:
|
||||
return _workload_identity_auth(config).get_token()
|
||||
|
||||
|
||||
def _targets_openai_api(api_base: str | None) -> bool:
|
||||
if api_base is None:
|
||||
return True
|
||||
parsed: Final = urlparse(api_base)
|
||||
return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
|
||||
sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
|
||||
return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())
|
||||
|
||||
|
||||
def _load_sdk_workload_identity_auth() -> type[WorkloadIdentityAuth]:
|
||||
try:
|
||||
from openai.auth import WorkloadIdentityAuth as sdk_workload_identity_auth
|
||||
except ImportError as e:
|
||||
raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
|
||||
return sdk_workload_identity_auth
|
||||
|
||||
|
||||
def _load_sdk_k8s_token_provider() -> Callable[[str], SubjectTokenProvider]:
|
||||
try:
|
||||
from openai.auth import k8s_service_account_token_provider
|
||||
except ImportError as e:
|
||||
raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
|
||||
return k8s_service_account_token_provider
|
||||
|
|
@ -27,6 +27,15 @@ from .common_utils import (
|
|||
get_vertex_base_url,
|
||||
)
|
||||
|
||||
|
||||
def _graft_default_vertex_path(api_base: str, default_url: str) -> str:
|
||||
parsed_api_base: Final = urlparse(api_base)
|
||||
default_segments: Final = urlparse(default_url).path.lstrip("/").split("/")
|
||||
graft_segments: Final = default_segments[1:] if default_segments[0] in ("v1", "v1beta1") else default_segments
|
||||
grafted_path: Final = parsed_api_base.path.rstrip("/") + "/" + "/".join(graft_segments)
|
||||
return parsed_api_base._replace(path=grafted_path).geturl()
|
||||
|
||||
|
||||
GOOGLE_IMPORT_ERROR_MESSAGE: Final = (
|
||||
"Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip install google-cloud-aiplatform"
|
||||
)
|
||||
|
|
@ -621,8 +630,9 @@ class VertexBase:
|
|||
|
||||
Handles custom api_base for:
|
||||
1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
|
||||
2. Vertex AI with standard proxies - constructs {api_base}:{endpoint};
|
||||
if api_base has no path (bare host), grafts the default vertex URL path onto it
|
||||
2. Vertex AI with standard proxies - grafts the default vertex URL path onto the
|
||||
api_base when its path is empty or only an API version (/v1, /v1beta1);
|
||||
otherwise constructs {api_base}:{endpoint}
|
||||
3. Vertex AI with PSC endpoints - constructs full path structure
|
||||
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
|
||||
(only when use_psc_endpoint_format=True)
|
||||
|
|
@ -669,10 +679,14 @@ class VertexBase:
|
|||
)
|
||||
elif urlparse(api_base).path in ("", "/"):
|
||||
url = api_base.rstrip("/") + urlparse(url).path
|
||||
elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path:
|
||||
url = _graft_default_vertex_path(api_base=api_base, default_url=url)
|
||||
else:
|
||||
url = f"{api_base}:{endpoint}"
|
||||
if stream is True:
|
||||
url = url + "?alt=sse"
|
||||
parsed_stream_url: Final = urlparse(url)
|
||||
stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse"
|
||||
url = parsed_stream_url._replace(query=stream_query).geturl()
|
||||
return auth_header, url
|
||||
|
||||
def _get_token_and_url(
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/responses/{response_id}/cancel",
|
||||
"/v1/responses/{response_id}/cancel",
|
||||
"/openai/v1/responses/{response_id}/cancel",
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
# vector stores
|
||||
"/vector_stores",
|
||||
"/v1/vector_stores",
|
||||
|
|
@ -3549,6 +3552,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
|
|||
)
|
||||
|
||||
|
||||
class SpendLogsRouterMetadata(TypedDict):
|
||||
"""
|
||||
Router provenance stamped on spend logs for deployments flagged with
|
||||
model_info.internal_router_model, correlating the requested model group
|
||||
with the provider deployment that served the call
|
||||
"""
|
||||
|
||||
requested_model: ReadOnly[str | None]
|
||||
selected_model: ReadOnly[str | None]
|
||||
selected_provider: ReadOnly[str | None]
|
||||
router_correlation_id: ReadOnly[str | None]
|
||||
|
||||
|
||||
class SpendLogsMetadata(TypedDict):
|
||||
"""
|
||||
Specific metadata k,v pairs logged to spendlogs for easier cost tracking
|
||||
|
|
@ -3591,6 +3607,7 @@ class SpendLogsMetadata(TypedDict):
|
|||
compression_savings: CompressionSavingsMetadata | None
|
||||
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
|
||||
litellm_gateway_injected_cache: ReadOnly[str | None]
|
||||
router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model
|
||||
|
||||
|
||||
class SpendLogsPayload(TypedDict):
|
||||
|
|
|
|||
|
|
@ -156,6 +156,31 @@ def _header_value(headers: Mapping[str, str], key: str, default: str) -> str:
|
|||
return headers.get(key, default)
|
||||
|
||||
|
||||
def _is_image_part(item: object) -> bool:
|
||||
"""Whether a structured-message content part carries an image rather than text."""
|
||||
|
||||
if not isinstance(item, Mapping):
|
||||
return False
|
||||
|
||||
part: Final[Mapping[object, object]] = item
|
||||
return part.get("type") == "image_url"
|
||||
|
||||
|
||||
def _scannable_text(content: object) -> str:
|
||||
"""Flatten a structured message's content into the single string the v1 detection endpoint takes.
|
||||
|
||||
Image parts are dropped: the endpoint accepts one string, so an image would only reach it as
|
||||
its stringified source (a base64 blob or a URL), which is not text the scanner can evaluate.
|
||||
"""
|
||||
|
||||
if not isinstance(content, list):
|
||||
return str(content or "")
|
||||
|
||||
parts: Final[Sequence[object]] = content
|
||||
text_parts: Final = [item for item in parts if not _is_image_part(item)] # mutable-ok: sent as a list repr
|
||||
return str(text_parts or "")
|
||||
|
||||
|
||||
def is_saas(host: str) -> bool:
|
||||
"""Checks whether the connection is to the SaaS platform"""
|
||||
|
||||
|
|
@ -270,7 +295,7 @@ class HiddenlayerGuardrail(CustomGuardrail):
|
|||
"messages": [
|
||||
{
|
||||
"role": last_msg.get("role", "user"),
|
||||
"content": str(last_msg.get("content", "")),
|
||||
"content": _scannable_text(last_msg.get("content")),
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
file_sanitization_fail_open=getattr(litellm_params, "file_sanitization_fail_open", None),
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback(_prompt_security_callback)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ import os
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.exceptions import Timeout as LiteLLMTimeout
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
|
|
@ -24,6 +26,9 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
|
||||
_SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS: Final = 30.0
|
||||
|
||||
|
||||
class PromptSecurityGuardrailMissingSecrets(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -63,6 +68,13 @@ class _SanitizeStatusResponse(TypedDict, total=False):
|
|||
metadata: ReadOnly[_SanitizeMetadata]
|
||||
|
||||
|
||||
class _SanitizeResult(TypedDict):
|
||||
action: ReadOnly[str]
|
||||
content: ReadOnly[str | None]
|
||||
metadata: ReadOnly[_SanitizeMetadata]
|
||||
violations: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class PromptSecurityGuardrail(CustomGuardrail):
|
||||
@classmethod
|
||||
def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
|
||||
|
|
@ -79,6 +91,8 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
user: str | None = None,
|
||||
system_prompt: str | None = None,
|
||||
check_tool_results: bool | None = None,
|
||||
file_sanitization_timeout: float = _SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS,
|
||||
file_sanitization_fail_open: bool | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
|
||||
|
|
@ -108,6 +122,8 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
# Configuration for file sanitization
|
||||
self.max_poll_attempts = 30 # Maximum number of polling attempts
|
||||
self.poll_interval = 2 # Seconds between polling attempts
|
||||
self.file_sanitization_timeout = file_sanitization_timeout
|
||||
self.file_sanitization_fail_open = file_sanitization_fail_open is not False
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
|
@ -397,6 +413,39 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
Sanitize file content using Prompt Security API.
|
||||
Returns: dict with keys 'action', 'content', 'metadata'
|
||||
"""
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self._sanitize_file_content(file_data, filename, user_api_key_alias),
|
||||
timeout=self.file_sanitization_timeout,
|
||||
)
|
||||
except (asyncio.TimeoutError, httpx.TimeoutException, LiteLLMTimeout) as exc:
|
||||
if not self.file_sanitization_fail_open:
|
||||
verbose_proxy_logger.error(
|
||||
"Prompt Security Guardrail: file sanitization for %s timed out with %s; failing closed",
|
||||
filename,
|
||||
type(exc).__name__,
|
||||
)
|
||||
raise HTTPException(status_code=408, detail="File sanitization timeout") from exc
|
||||
|
||||
verbose_proxy_logger.error(
|
||||
"Prompt Security Guardrail: file sanitization for %s timed out with %s; failing open",
|
||||
filename,
|
||||
type(exc).__name__,
|
||||
)
|
||||
fail_open_result: Final[_SanitizeResult] = {
|
||||
"action": "allow",
|
||||
"content": None,
|
||||
"metadata": {},
|
||||
"violations": (),
|
||||
}
|
||||
return fail_open_result
|
||||
|
||||
async def _sanitize_file_content(
|
||||
self,
|
||||
file_data: bytes,
|
||||
filename: str,
|
||||
user_api_key_alias: str | None,
|
||||
) -> _SanitizeResult:
|
||||
headers: Final = {"APP-ID": self.api_key}
|
||||
if user_api_key_alias:
|
||||
headers["X-LiteLLM-Key-Alias"] = user_api_key_alias
|
||||
|
|
|
|||
|
|
@ -626,11 +626,7 @@ async def update_end_user(
|
|||
# get non default values for key
|
||||
non_default_values: Final = dict[str, object]()
|
||||
for k, v in data_json.items():
|
||||
if v is not None and v not in (
|
||||
[],
|
||||
{},
|
||||
0,
|
||||
): # models default to [], spend defaults to 0, we should not reset these values
|
||||
if v is not None and ((isinstance(v, bool) and k in data.fields_set()) or v not in ([], {}, 0)):
|
||||
non_default_values[k] = v
|
||||
|
||||
## Get end user table data ##
|
||||
|
|
|
|||
|
|
@ -1027,6 +1027,14 @@ async def user_info_v2(
|
|||
This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem
|
||||
where the old endpoint loaded all keys and teams into memory.
|
||||
|
||||
Note on `spend`: this is the user's running budget counter, which the budget reset job
|
||||
resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default,
|
||||
or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT
|
||||
lifetime or per-period historical spend. For historical spend over a date range, use
|
||||
`/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend
|
||||
records that only ever accumulate and are never reset. The two values are expected to
|
||||
diverge once a budget reset has occurred within the queried period.
|
||||
|
||||
Access control:
|
||||
- Proxy admins can query any user
|
||||
- Team admins can query users within their teams
|
||||
|
|
@ -2726,6 +2734,11 @@ async def get_user_daily_activity(
|
|||
|
||||
Meant to optimize querying spend data for analytics for a user.
|
||||
|
||||
Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
resets. Their total can legitimately exceed the `spend` field returned by
|
||||
`/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
|
||||
Returns:
|
||||
(by date)
|
||||
- spend
|
||||
|
|
@ -2839,6 +2852,11 @@ async def get_user_daily_activity_aggregated(
|
|||
"""
|
||||
Aggregated analytics for a user's daily activity without pagination.
|
||||
Returns the same response shape as the paginated endpoint with page metadata set to single-page.
|
||||
|
||||
Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
resets. Their total can legitimately exceed the `spend` field returned by
|
||||
`/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ async def add_team_callbacks(
|
|||
Use this if if you want different teams to have different success/failure callbacks
|
||||
|
||||
Parameters:
|
||||
- callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
- callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
- callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
- "success": Callback for successful LLM calls
|
||||
- "failure": Callback for failed LLM calls
|
||||
|
|
@ -268,6 +268,8 @@ async def add_team_callbacks(
|
|||
- langsmith_api_key: The API key for the Langsmith callback
|
||||
- langsmith_project: The project for the Langsmith callback
|
||||
- langsmith_base_url: The base URL for the Langsmith callback
|
||||
- newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
- newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
|
||||
Example curl:
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from collections.abc import AsyncIterator, Awaitable, Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from starlette.websockets import WebSocket, WebSocketDisconnect
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
|
|
@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
_read_request_body,
|
||||
_safe_set_request_parsed_body,
|
||||
)
|
||||
from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse
|
||||
from litellm.types.llms.openai import (
|
||||
REASONING_EFFORT,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.responses.main import DeleteResponseResult
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
@ -35,7 +44,7 @@ if TYPE_CHECKING:
|
|||
router: Final = APIRouter()
|
||||
|
||||
_user_api_key_auth_dep: Final = Depends(user_api_key_auth)
|
||||
_RESPONSES_TAGS: Final = ["responses"] # mutable-ok: fastapi's route signature requires List[str] tags
|
||||
_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"] # mutable-ok: fastapi's route signature requires list tags
|
||||
|
||||
_TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -1017,6 +1026,152 @@ async def compact_response(
|
|||
)
|
||||
|
||||
|
||||
class _ResponsesApiErrorDetail(TypedDict):
|
||||
message: ReadOnly[str]
|
||||
type: ReadOnly[str]
|
||||
param: ReadOnly[str | None]
|
||||
code: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _ResponsesApiErrorBody(TypedDict):
|
||||
error: ReadOnly[_ResponsesApiErrorDetail]
|
||||
|
||||
|
||||
class _ResponsesInputTokensResult(TypedDict):
|
||||
object: ReadOnly[str]
|
||||
input_tokens: ReadOnly[int]
|
||||
|
||||
|
||||
class _TokenCountPayload(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
messages: ReadOnly[tuple[Mapping[str, object], ...]]
|
||||
tools: ReadOnly[object]
|
||||
|
||||
|
||||
class _TokenCounter(Protocol):
|
||||
def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ...
|
||||
|
||||
|
||||
def _proxy_token_counter() -> _TokenCounter:
|
||||
from litellm.proxy.proxy_server import token_counter
|
||||
|
||||
return token_counter
|
||||
|
||||
|
||||
_token_counter_dep: Final = Depends(_proxy_token_counter)
|
||||
|
||||
|
||||
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
|
||||
body: Final[_ResponsesApiErrorBody] = {
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": "invalid_request_error",
|
||||
"param": param,
|
||||
"code": code,
|
||||
}
|
||||
}
|
||||
return JSONResponse(status_code=400, content=body)
|
||||
|
||||
|
||||
def _missing_responses_param_response(param: str) -> JSONResponse:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Missing required parameter: '{param}'.",
|
||||
param=param,
|
||||
code="missing_required_parameter",
|
||||
)
|
||||
|
||||
|
||||
def _responses_input_as_token_count_messages(
|
||||
input_value: str | ResponseInputParam,
|
||||
instructions: str | None,
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
||||
request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions}
|
||||
transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=input_value,
|
||||
responses_api_request=request_params,
|
||||
)
|
||||
return tuple(
|
||||
message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
@router.post(
|
||||
"/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
@router.post(
|
||||
"/openai/v1/responses/input_tokens",
|
||||
dependencies=(_user_api_key_auth_dep,),
|
||||
tags=_RESPONSES_TAGS,
|
||||
)
|
||||
async def responses_input_tokens(
|
||||
request: Request,
|
||||
token_counter: _TokenCounter = _token_counter_dep,
|
||||
):
|
||||
"""
|
||||
Count the input tokens of a Responses API request without calling the model.
|
||||
|
||||
Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:4000/v1/responses/input_tokens \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"input": "Hello, how are you?"
|
||||
}'
|
||||
```
|
||||
|
||||
Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
"""
|
||||
data: Final = await _read_request_body(request=request)
|
||||
model_name: Final = data.get("model")
|
||||
input_value: Final = data.get("input")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
return _missing_responses_param_response("model")
|
||||
if input_value is None:
|
||||
return _missing_responses_param_response("input")
|
||||
if isinstance(input_value, (str, list)) and not input_value:
|
||||
return _responses_invalid_request_response(
|
||||
message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
param=None,
|
||||
code="missing_required_parameter",
|
||||
)
|
||||
|
||||
try:
|
||||
payload: Final[_TokenCountPayload] = {
|
||||
"model": model_name,
|
||||
"messages": _responses_input_as_token_count_messages(
|
||||
input_value=input_value,
|
||||
instructions=data.get("instructions"),
|
||||
),
|
||||
"tools": data.get("tools"),
|
||||
}
|
||||
token_request: Final = TokenCountRequest.model_validate(payload)
|
||||
except Exception as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Invalid request for token counting: {e}", param=None, code=None
|
||||
)
|
||||
|
||||
token_response: Final = await token_counter(request=token_request, call_endpoint=True)
|
||||
result: Final[_ResponsesInputTokensResult] = {
|
||||
"object": "response.input_tokens",
|
||||
"input_tokens": token_response.total_tokens,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/{response_id}/cancel",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
|
|||
|
|
@ -172,7 +172,14 @@ async def reserve_budget_for_request(
|
|||
) -> dict | None:
|
||||
if valid_token is None or not RouteChecks.is_llm_api_route(route=route):
|
||||
return None
|
||||
if route in {"/models", "/v1/models", "/utils/token_counter"}:
|
||||
if route in {
|
||||
"/models",
|
||||
"/v1/models",
|
||||
"/utils/token_counter",
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
}:
|
||||
return None
|
||||
if get_model_from_request(request_body, route, llm_router=llm_router) is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
request_model_access_groups_from_litellm_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
|
||||
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
|
||||
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsRouterMetadata
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.proxy.utils import PrismaClient, hash_token
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -93,6 +93,24 @@ def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False)
|
|||
return hash_token(stripped)
|
||||
|
||||
|
||||
def _get_router_metadata_for_spend_log(
|
||||
metadata: Mapping[str, object] | None,
|
||||
requested_model: str | None,
|
||||
selected_model: str | None,
|
||||
selected_provider: str | None,
|
||||
router_correlation_id: str | None,
|
||||
) -> SpendLogsRouterMetadata | None:
|
||||
model_info: Final = metadata.get("model_info") if metadata is not None else None
|
||||
if not isinstance(model_info, Mapping) or model_info.get("internal_router_model") is not True:
|
||||
return None
|
||||
return SpendLogsRouterMetadata(
|
||||
requested_model=requested_model or None,
|
||||
selected_model=selected_model or None,
|
||||
selected_provider=selected_provider or None,
|
||||
router_correlation_id=router_correlation_id,
|
||||
)
|
||||
|
||||
|
||||
def _get_spend_logs_metadata(
|
||||
metadata: dict | None,
|
||||
applied_guardrails: list[str] | None = None,
|
||||
|
|
@ -109,6 +127,7 @@ def _get_spend_logs_metadata(
|
|||
cost_breakdown: CostBreakdown | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
autorouter_savings: float | None = None,
|
||||
router_metadata: SpendLogsRouterMetadata | None = None,
|
||||
) -> SpendLogsMetadata:
|
||||
if metadata is None:
|
||||
return SpendLogsMetadata(
|
||||
|
|
@ -148,13 +167,17 @@ def _get_spend_logs_metadata(
|
|||
autorouter_savings=autorouter_savings,
|
||||
litellm_gateway_injected_cache=None,
|
||||
litellm_call_id=litellm_call_id,
|
||||
router_metadata=router_metadata,
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
"getting payload for SpendLogs, available keys in metadata: " + str(list(metadata.keys()))
|
||||
)
|
||||
|
||||
# Filter the metadata dictionary to include only the specified keys
|
||||
clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__})
|
||||
clean_metadata: Final = SpendLogsMetadata(
|
||||
**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"},
|
||||
router_metadata=router_metadata,
|
||||
)
|
||||
_raw_key: Final = clean_metadata.get("user_api_key")
|
||||
_trusted_hash: Final = metadata.get("user_api_key_hash")
|
||||
_already_redacted: Final = (
|
||||
|
|
@ -375,6 +398,20 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
hidden_params: Final = standard_logging_payload.get("hidden_params", {})
|
||||
litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms")
|
||||
|
||||
custom_llm_provider: Final = (
|
||||
kwargs.get("custom_llm_provider")
|
||||
or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider")
|
||||
or None
|
||||
)
|
||||
raw_model: Final = cast(str, kwargs.get("model") or "")
|
||||
model_name: Final = (
|
||||
standard_logging_payload.get("model") if standard_logging_payload is not None else None
|
||||
) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
|
||||
litellm_call_id: Final = cast(
|
||||
str | None,
|
||||
kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
|
||||
)
|
||||
|
||||
# clean up litellm metadata
|
||||
clean_metadata = _get_spend_logs_metadata(
|
||||
metadata,
|
||||
|
|
@ -433,9 +470,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
autorouter_savings=(
|
||||
standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None
|
||||
),
|
||||
litellm_call_id=cast(
|
||||
str | None,
|
||||
kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
|
||||
litellm_call_id=litellm_call_id,
|
||||
router_metadata=_get_router_metadata_for_spend_log(
|
||||
metadata=metadata,
|
||||
requested_model=_model_group,
|
||||
selected_model=model_name,
|
||||
selected_provider=custom_llm_provider,
|
||||
router_correlation_id=litellm_call_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -480,15 +521,6 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
|
||||
# Extract agent_id for A2A requests (set directly on model_call_details)
|
||||
agent_id: Final[str | None] = kwargs.get("agent_id") or metadata.get("agent_id")
|
||||
custom_llm_provider: Final = (
|
||||
kwargs.get("custom_llm_provider")
|
||||
or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider")
|
||||
or None
|
||||
)
|
||||
raw_model: Final = cast(str, kwargs.get("model") or "")
|
||||
model_name: Final = (
|
||||
standard_logging_payload.get("model") if standard_logging_payload is not None else None
|
||||
) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
|
||||
|
||||
try:
|
||||
payload: Final[SpendLogsPayload] = SpendLogsPayload(
|
||||
|
|
|
|||
|
|
@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
file_dict["file_id"] = file_id
|
||||
if item.get("file_data"):
|
||||
file_dict["file_data"] = item["file_data"]
|
||||
if item.get("filename"):
|
||||
file_dict["filename"] = item["filename"]
|
||||
|
||||
new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict}
|
||||
if "cache_control" in item:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ class PromptSecurityGuardrailConfigModel(GuardrailConfigModel):
|
|||
default=None,
|
||||
description="The API base for the Prompt Security guardrail. If not provided, the `PROMPT_SECURITY_API_BASE` environment variable is used.",
|
||||
)
|
||||
file_sanitization_fail_open: bool = Field(
|
||||
default=True,
|
||||
description="Whether file sanitization timeouts allow the original file through instead of blocking the request.",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
|
|
|
|||
|
|
@ -189,6 +189,11 @@ class ModelInfo(MirroredPricingParams):
|
|||
# router-wide default.
|
||||
enable_tag_filtering: bool | None = None
|
||||
|
||||
# when True, calls routed to this deployment persist a router_metadata block
|
||||
# (requested model group, selected model + provider, router correlation id)
|
||||
# in the spend log row's metadata. Set it on every deployment of the group.
|
||||
internal_router_model: bool | None = None
|
||||
|
||||
def __init__(self, id: str | int | None = None, **params) -> None:
|
||||
if id is None:
|
||||
id = str(uuid.uuid4()) # Generate a UUID if id is None or not provided
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
348
scripts/auto-close-duplicates.test.ts
Normal file
348
scripts/auto-close-duplicates.test.ts
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import {
|
||||
CLOSED_MARKER,
|
||||
REOPEN_COMMENT,
|
||||
candidateNumbers,
|
||||
duplicateTarget,
|
||||
normalizeTitle,
|
||||
pendingNotice,
|
||||
readConfig,
|
||||
reopenTarget,
|
||||
sweepClosedIssue,
|
||||
sweepIssue,
|
||||
type Comment,
|
||||
type GitHubApi,
|
||||
type Issue,
|
||||
type Reaction,
|
||||
type SweepConfig,
|
||||
} from "./auto-close-duplicates";
|
||||
|
||||
const NOW = new Date("2026-09-04T09:00:00Z");
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const daysAgo = (days: number): string => new Date(NOW.getTime() - days * DAY_MS).toISOString();
|
||||
|
||||
const issue = (number: number, title: string, overrides: Partial<Issue> = {}): Issue => ({
|
||||
number,
|
||||
title,
|
||||
state: "open",
|
||||
user: { login: "reporter" },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const notice = (candidates: readonly number[], createdAt: string, overrides: Partial<Comment> = {}): Comment => ({
|
||||
id: 900,
|
||||
body: `<!-- litellm:potential-duplicate candidates=${candidates.join(",")}, -->\n**Potential duplicate detected**`,
|
||||
created_at: createdAt,
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const humanComment = (createdAt: string, body = "It is not the same thing", login = "reporter"): Comment => ({
|
||||
id: 901,
|
||||
body,
|
||||
created_at: createdAt,
|
||||
user: { type: "User", login },
|
||||
});
|
||||
|
||||
const config: SweepConfig = { repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW };
|
||||
|
||||
describe("normalizeTitle", () => {
|
||||
test("drops the template prefix, case, and punctuation", () => {
|
||||
expect(normalizeTitle("[Bug]: Gemma 4-e4b fails on Vertex!")).toBe("gemma 4 e4b fails on vertex");
|
||||
expect(normalizeTitle("[Feature]: ")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("candidateNumbers", () => {
|
||||
test("reads only the marker field, keeps older issues, sorted ascending and deduplicated", () => {
|
||||
const body = "<!-- litellm:potential-duplicate candidates=40,10,30,10, -->\n- #1 - see #1 (100% similar)";
|
||||
expect(candidateNumbers(body, 35)).toEqual([10, 30]);
|
||||
});
|
||||
|
||||
test("returns nothing without the marker", () => {
|
||||
expect(candidateNumbers("- #1 - looks like #1", 35)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pendingNotice", () => {
|
||||
test("waits out the grace period from the latest notice", () => {
|
||||
const fresh = pendingNotice(issue(35, "t"), [notice([10], daysAgo(2.9))], config);
|
||||
expect(fresh.kind).toBe("skip");
|
||||
const aged = pendingNotice(issue(35, "t"), [notice([10], daysAgo(3.1))], config);
|
||||
expect(aged.kind).toBe("pending");
|
||||
const reposted = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(6)), notice([10], daysAgo(1), { id: 902 })],
|
||||
config,
|
||||
);
|
||||
expect(reposted.kind).toBe("skip");
|
||||
});
|
||||
|
||||
test("an objection posted before a re-posted notice still keeps the issue open", () => {
|
||||
const verdict = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(10)), humanComment(daysAgo(7)), notice([10], daysAgo(4), { id: 902 })],
|
||||
config,
|
||||
);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone replied after the notice" });
|
||||
});
|
||||
|
||||
test("a zero-day grace period acts on the notice at once", () => {
|
||||
const verdict = pendingNotice(issue(35, "t"), [notice([10], daysAgo(0.01))], { ...config, graceDays: 0 });
|
||||
expect(verdict.kind).toBe("pending");
|
||||
});
|
||||
|
||||
test("a human reply after the notice keeps the issue open, a bot reply does not", () => {
|
||||
const human = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5)), humanComment(daysAgo(4))], config);
|
||||
expect(human).toEqual({ kind: "skip", reason: "someone replied after the notice" });
|
||||
const bot = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(5)), { id: 903, body: "triage", created_at: daysAgo(4), user: { type: "Bot", login: "triage[bot]" } }],
|
||||
config,
|
||||
);
|
||||
expect(bot.kind).toBe("pending");
|
||||
});
|
||||
|
||||
test("a human quoting the marker is not a notice", () => {
|
||||
const quoted = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5), { user: { type: "User", login: "reporter" } })], config);
|
||||
expect(quoted).toEqual({ kind: "skip", reason: "carries no duplicate notice" });
|
||||
});
|
||||
|
||||
test("never closes an issue twice: a reopened issue is left alone", () => {
|
||||
const reopened = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(9)), { id: 904, body: `Closed automatically\n\n${CLOSED_MARKER}`, created_at: daysAgo(5), user: { type: "Bot", login: "github-actions[bot]" } }],
|
||||
config,
|
||||
);
|
||||
expect(reopened).toEqual({ kind: "skip", reason: "was reopened after an automatic close" });
|
||||
});
|
||||
|
||||
test("skips pull requests and issues whose only candidates are newer", () => {
|
||||
expect(pendingNotice(issue(35, "t", { pull_request: {} }), [notice([10], daysAgo(5))], config).kind).toBe("skip");
|
||||
expect(pendingNotice(issue(35, "t"), [notice([40], daysAgo(5))], config)).toEqual({
|
||||
kind: "skip",
|
||||
reason: "no candidate is older than this issue",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("duplicateTarget", () => {
|
||||
const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
|
||||
test("closes only against the earliest open issue with the identical normalized title", () => {
|
||||
const verdict = duplicateTarget(
|
||||
reporter,
|
||||
[issue(10, "[Bug]: Gemma 4-e4n fails on Vertex"), issue(20, "[bug]: gemma 4-e4b fails on vertex"), issue(30, "[Bug]: Gemma 4-e4b fails on Vertex")],
|
||||
[],
|
||||
);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 20 });
|
||||
});
|
||||
|
||||
test("a near miss in the title is not a duplicate", () => {
|
||||
const verdict = duplicateTarget(reporter, [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex")], []);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "no older open issue has the identical title" });
|
||||
});
|
||||
|
||||
test("bare template titles never match each other", () => {
|
||||
const verdict = duplicateTarget(issue(35, "[Bug]: "), [issue(10, "[Bug]: ")], []);
|
||||
expect(verdict.kind).toBe("skip");
|
||||
expect(verdict.kind === "skip" && verdict.reason).toContain("too short");
|
||||
});
|
||||
|
||||
test("a closed candidate or a pull request is never the target", () => {
|
||||
expect(duplicateTarget(reporter, [issue(10, reporter.title, { state: "closed" })], []).kind).toBe("skip");
|
||||
expect(duplicateTarget(reporter, [issue(10, reporter.title, { pull_request: {} })], []).kind).toBe("skip");
|
||||
});
|
||||
|
||||
test("a thumbs down on the notice keeps the issue open", () => {
|
||||
const verdict = duplicateTarget(reporter, [issue(10, reporter.title)], [{ content: "+1" }, { content: "-1" }]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepIssue", () => {
|
||||
const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
const original = issue(10, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
|
||||
function fakeApi(
|
||||
comments: readonly Comment[] = [notice([10], daysAgo(5))],
|
||||
reactionsByNotice: Readonly<Record<number, readonly Reaction[]>> = {},
|
||||
): { readonly api: GitHubApi; readonly writes: readonly string[] } {
|
||||
const writes: string[] = [];
|
||||
const api: GitHubApi = {
|
||||
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
|
||||
if (method !== "GET") {
|
||||
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
|
||||
return {} as T;
|
||||
}
|
||||
if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) {
|
||||
return comments as T;
|
||||
}
|
||||
const reactionsPath = path.match(/^\/repos\/BerriAI\/litellm\/issues\/comments\/(\d+)\/reactions/);
|
||||
if (reactionsPath) {
|
||||
return (reactionsByNotice[Number(reactionsPath[1])] ?? []) as T;
|
||||
}
|
||||
if (path === "/repos/BerriAI/litellm/issues/10") {
|
||||
return original as T;
|
||||
}
|
||||
throw new Error(`unexpected GET ${path}`);
|
||||
},
|
||||
};
|
||||
return { api, writes };
|
||||
}
|
||||
|
||||
test("a dry run reports the close and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await sweepIssue(api, { ...config, dryRun: true }, reporter);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 10 });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a thumbs down on an earlier notice still keeps the issue open", async () => {
|
||||
const { api, writes } = fakeApi([notice([10], daysAgo(9)), notice([10], daysAgo(5), { id: 902 })], { 900: [{ content: "-1" }] });
|
||||
const verdict = await sweepIssue(api, config, reporter);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a real run comments, labels, then closes with the duplicate reason", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await sweepIssue(api, config, reporter);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 10 });
|
||||
expect(writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([
|
||||
"POST /repos/BerriAI/litellm/issues/35/comments",
|
||||
"POST /repos/BerriAI/litellm/issues/35/labels",
|
||||
"PATCH /repos/BerriAI/litellm/issues/35",
|
||||
]);
|
||||
expect(writes[0]).toContain("duplicate of #10");
|
||||
expect(writes[0]).toContain("unanswered for 3 days");
|
||||
expect(writes[0]).toContain(CLOSED_MARKER);
|
||||
expect(writes[1]).toContain('{"labels":["duplicate"]}');
|
||||
expect(writes[2]).toContain('{"state":"closed","state_reason":"duplicate"}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("reopenTarget", () => {
|
||||
const closedByBot = (overrides: Partial<Issue> = {}): Issue =>
|
||||
issue(35, "t", { state: "closed", closed_by: { type: "Bot" }, ...overrides });
|
||||
const closeMarker = (createdAt: string): Comment => ({
|
||||
id: 905,
|
||||
body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`,
|
||||
created_at: createdAt,
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
});
|
||||
|
||||
test("a reporter reply after the automatic close reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
});
|
||||
|
||||
test("an issue closed by a person stays closed", () => {
|
||||
const verdict = reopenTarget(closedByBot({ closed_by: { type: "User" } }), [
|
||||
closeMarker(daysAgo(2)),
|
||||
humanComment(daysAgo(1)),
|
||||
]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "was closed by a person" });
|
||||
});
|
||||
|
||||
test("without the automatic-close marker nothing reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "carries no automatic-close marker" });
|
||||
});
|
||||
|
||||
test("a maintainer reply alone does not reopen", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [
|
||||
closeMarker(daysAgo(2)),
|
||||
humanComment(daysAgo(1), "Confirmed duplicate", "maintainer"),
|
||||
]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" });
|
||||
});
|
||||
|
||||
test("a reporter comment from before the close does not reopen", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(3)), closeMarker(daysAgo(2))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" });
|
||||
});
|
||||
|
||||
test("a pull request never reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot({ pull_request: {} }), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "is a pull request" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepClosedIssue", () => {
|
||||
function fakeApi(issueBody: Issue, comments: readonly Comment[]): { readonly api: GitHubApi; readonly writes: readonly string[] } {
|
||||
const writes: string[] = [];
|
||||
const api: GitHubApi = {
|
||||
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
|
||||
if (method !== "GET") {
|
||||
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
|
||||
return {} as T;
|
||||
}
|
||||
if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) {
|
||||
return comments as T;
|
||||
}
|
||||
if (path === "/repos/BerriAI/litellm/issues/35") {
|
||||
return issueBody as T;
|
||||
}
|
||||
throw new Error(`unexpected GET ${path}`);
|
||||
},
|
||||
};
|
||||
return { api, writes };
|
||||
}
|
||||
|
||||
const closedByBot = issue(35, "t", { state: "closed", closed_by: { type: "Bot" } });
|
||||
const closeMarker: Comment = {
|
||||
id: 905,
|
||||
body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`,
|
||||
created_at: daysAgo(2),
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
};
|
||||
|
||||
test("a real run unlabels, reopens, then explains", async () => {
|
||||
const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]);
|
||||
const verdict = await sweepClosedIssue(api, config, 35);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
expect(writes).toEqual([
|
||||
"DELETE /repos/BerriAI/litellm/issues/35/labels/duplicate undefined",
|
||||
'PATCH /repos/BerriAI/litellm/issues/35 {"state":"open"}',
|
||||
`POST /repos/BerriAI/litellm/issues/35/comments {"body":"${REOPEN_COMMENT}"}`,
|
||||
]);
|
||||
});
|
||||
|
||||
test("a dry run reports the reopen and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]);
|
||||
const verdict = await sweepClosedIssue(api, { ...config, dryRun: true }, 35);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("readConfig", () => {
|
||||
test("defaults to a real run with a 3-day grace period", () => {
|
||||
const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm" }, NOW);
|
||||
expect(parsed).toEqual({ token: "t", repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW });
|
||||
});
|
||||
|
||||
test("honors DRY_RUN and GRACE_PERIOD_DAYS overrides", () => {
|
||||
const parsed = readConfig(
|
||||
{ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", DRY_RUN: "true", GRACE_PERIOD_DAYS: "0" },
|
||||
NOW,
|
||||
);
|
||||
expect(parsed.dryRun).toBe(true);
|
||||
expect(parsed.graceDays).toBe(0);
|
||||
});
|
||||
|
||||
test("an empty GRACE_PERIOD_DAYS, as a schedule run renders it, means the default", () => {
|
||||
const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "" }, NOW);
|
||||
expect(parsed.graceDays).toBe(3);
|
||||
});
|
||||
|
||||
test("refuses a missing token, a malformed repository, or a bad grace period", () => {
|
||||
expect(() => readConfig({ GITHUB_REPOSITORY: "o/r" }, NOW)).toThrow("GITHUB_TOKEN");
|
||||
expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "litellm" }, NOW)).toThrow("owner/repo");
|
||||
expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "-1" }, NOW)).toThrow(
|
||||
"GRACE_PERIOD_DAYS",
|
||||
);
|
||||
});
|
||||
});
|
||||
300
scripts/auto-close-duplicates.ts
Normal file
300
scripts/auto-close-duplicates.ts
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
declare const process: { readonly env: Readonly<Record<string, string | undefined>> };
|
||||
|
||||
export interface Issue {
|
||||
readonly number: number;
|
||||
readonly title: string;
|
||||
readonly state: string;
|
||||
readonly user: { readonly login: string };
|
||||
readonly closed_by?: { readonly type: string } | null;
|
||||
readonly pull_request?: unknown;
|
||||
}
|
||||
|
||||
export interface Comment {
|
||||
readonly id: number;
|
||||
readonly body: string;
|
||||
readonly created_at: string;
|
||||
readonly user: { readonly type: string; readonly login: string };
|
||||
}
|
||||
|
||||
export interface Reaction {
|
||||
readonly content: string;
|
||||
}
|
||||
|
||||
export interface GitHubApi {
|
||||
readonly request: <T>(method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface SweepConfig {
|
||||
readonly repo: string;
|
||||
readonly graceDays: number;
|
||||
readonly dryRun: boolean;
|
||||
readonly now: Date;
|
||||
}
|
||||
|
||||
export type NoticeVerdict =
|
||||
| { readonly kind: "pending"; readonly notices: readonly Comment[]; readonly candidates: readonly number[] }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export type CloseVerdict =
|
||||
| { readonly kind: "close"; readonly duplicateOf: number }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export type ReopenVerdict =
|
||||
| { readonly kind: "reopen" }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export const FLAG_LABEL = "potential-duplicate";
|
||||
export const CLOSED_MARKER = "<!-- litellm:closed-as-duplicate -->";
|
||||
export const DEFAULT_GRACE_DAYS = 3;
|
||||
export const REOPEN_COMMENT =
|
||||
"Reopened automatically: the reporter replied after the duplicate close, so this needs a human look.";
|
||||
const NOTICE_MARKER = /<!-- litellm:potential-duplicate candidates=([\d,]*) -->/;
|
||||
const MIN_TITLE_WORDS = 3;
|
||||
const PAGE_SIZE = 100;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const REOPEN_LOOKBACK_DAYS = 30;
|
||||
|
||||
const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason });
|
||||
|
||||
export function normalizeTitle(title: string): string {
|
||||
return title
|
||||
.toLowerCase()
|
||||
.replace(/^\s*\[[^\]]*\]\s*:?/, "")
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function candidateNumbers(noticeBody: string, issueNumber: number): readonly number[] {
|
||||
const field = noticeBody.match(NOTICE_MARKER);
|
||||
if (!field) {
|
||||
return [];
|
||||
}
|
||||
const older = field[1]
|
||||
.split(",")
|
||||
.filter((value) => value !== "")
|
||||
.map(Number)
|
||||
.filter((candidate) => candidate < issueNumber);
|
||||
return [...new Set(older)].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
export function pendingNotice(
|
||||
issue: Issue,
|
||||
comments: readonly Comment[],
|
||||
config: Pick<SweepConfig, "graceDays" | "now">,
|
||||
): NoticeVerdict {
|
||||
if (issue.pull_request !== undefined) {
|
||||
return skip("is a pull request");
|
||||
}
|
||||
if (comments.some((comment) => comment.body.includes(CLOSED_MARKER))) {
|
||||
return skip("was reopened after an automatic close");
|
||||
}
|
||||
const notices = comments.filter((comment) => comment.user.type === "Bot" && NOTICE_MARKER.test(comment.body));
|
||||
const first = notices[0];
|
||||
const latest = notices[notices.length - 1];
|
||||
if (first === undefined || latest === undefined) {
|
||||
return skip("carries no duplicate notice");
|
||||
}
|
||||
const ageDays = (config.now.getTime() - new Date(latest.created_at).getTime()) / DAY_MS;
|
||||
if (ageDays < config.graceDays) {
|
||||
return skip(`notice is ${ageDays.toFixed(1)} days old, grace period is ${config.graceDays}`);
|
||||
}
|
||||
const firstNoticeAt = new Date(first.created_at);
|
||||
if (comments.some((comment) => comment.user.type !== "Bot" && new Date(comment.created_at) > firstNoticeAt)) {
|
||||
return skip("someone replied after the notice");
|
||||
}
|
||||
const candidates = candidateNumbers(latest.body, issue.number);
|
||||
if (candidates.length === 0) {
|
||||
return skip("no candidate is older than this issue");
|
||||
}
|
||||
return { kind: "pending", notices, candidates };
|
||||
}
|
||||
|
||||
export function duplicateTarget(
|
||||
issue: Issue,
|
||||
candidates: readonly Issue[],
|
||||
reactions: readonly Reaction[],
|
||||
): CloseVerdict {
|
||||
if (reactions.some((reaction) => reaction.content === "-1")) {
|
||||
return skip("someone gave the notice a thumbs down");
|
||||
}
|
||||
const title = normalizeTitle(issue.title);
|
||||
if (title.split(" ").length < MIN_TITLE_WORDS) {
|
||||
return skip(`title "${issue.title}" is too short to match on`);
|
||||
}
|
||||
const original = candidates.find(
|
||||
(candidate) =>
|
||||
candidate.state === "open" && candidate.pull_request === undefined && normalizeTitle(candidate.title) === title,
|
||||
);
|
||||
if (original === undefined) {
|
||||
return skip("no older open issue has the identical title");
|
||||
}
|
||||
return { kind: "close", duplicateOf: original.number };
|
||||
}
|
||||
|
||||
export function reopenTarget(issue: Issue, comments: readonly Comment[]): ReopenVerdict {
|
||||
if (issue.pull_request !== undefined) {
|
||||
return skip("is a pull request");
|
||||
}
|
||||
if (issue.closed_by?.type !== "Bot") {
|
||||
return skip("was closed by a person");
|
||||
}
|
||||
const marker = comments.find((comment) => comment.body.includes(CLOSED_MARKER));
|
||||
if (marker === undefined) {
|
||||
return skip("carries no automatic-close marker");
|
||||
}
|
||||
const markerAt = new Date(marker.created_at);
|
||||
if (!comments.some((comment) => comment.user.login === issue.user.login && new Date(comment.created_at) > markerAt)) {
|
||||
return skip("the reporter has not replied since the close");
|
||||
}
|
||||
return { kind: "reopen" };
|
||||
}
|
||||
|
||||
export function closingComment(duplicateOf: number, graceDays: number): string {
|
||||
return `Closed automatically as a duplicate of #${duplicateOf}. Its title is identical to that older open issue and the duplicate notice above went unanswered for ${graceDays} days. If this is wrong, comment here with how it differs from #${duplicateOf} and this issue will be reopened automatically within a day.
|
||||
|
||||
${CLOSED_MARKER}`;
|
||||
}
|
||||
|
||||
async function listAll<T>(api: GitHubApi, path: string, page = 1): Promise<readonly T[]> {
|
||||
const separator = path.includes("?") ? "&" : "?";
|
||||
const batch = await api.request<readonly T[]>("GET", `${path}${separator}per_page=${PAGE_SIZE}&page=${page}`);
|
||||
return batch.length < PAGE_SIZE ? batch : [...batch, ...(await listAll<T>(api, path, page + 1))];
|
||||
}
|
||||
|
||||
async function closeAsDuplicate(
|
||||
api: GitHubApi,
|
||||
config: SweepConfig,
|
||||
issueNumber: number,
|
||||
duplicateOf: number,
|
||||
): Promise<void> {
|
||||
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
|
||||
await api.request("POST", `${issuePath}/comments`, { body: closingComment(duplicateOf, config.graceDays) });
|
||||
await api.request("POST", `${issuePath}/labels`, { labels: ["duplicate"] });
|
||||
await api.request("PATCH", issuePath, { state: "closed", state_reason: "duplicate" });
|
||||
}
|
||||
|
||||
async function reopenForReporter(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise<void> {
|
||||
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
|
||||
await api.request("DELETE", `${issuePath}/labels/duplicate`);
|
||||
await api.request("PATCH", issuePath, { state: "open" });
|
||||
await api.request("POST", `${issuePath}/comments`, { body: REOPEN_COMMENT });
|
||||
}
|
||||
|
||||
export async function sweepClosedIssue(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise<ReopenVerdict> {
|
||||
const issue = await api.request<Issue>("GET", `/repos/${config.repo}/issues/${issueNumber}`);
|
||||
const comments = await listAll<Comment>(api, `/repos/${config.repo}/issues/${issueNumber}/comments`);
|
||||
const verdict = reopenTarget(issue, comments);
|
||||
if (verdict.kind === "reopen" && !config.dryRun) {
|
||||
await reopenForReporter(api, config, issueNumber);
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
export async function sweepIssue(api: GitHubApi, config: SweepConfig, issue: Issue): Promise<CloseVerdict> {
|
||||
const comments = await listAll<Comment>(api, `/repos/${config.repo}/issues/${issue.number}/comments`);
|
||||
const pending = pendingNotice(issue, comments, config);
|
||||
if (pending.kind === "skip") {
|
||||
return pending;
|
||||
}
|
||||
const reactions = (
|
||||
await Promise.all(
|
||||
pending.notices.map((notice) => listAll<Reaction>(api, `/repos/${config.repo}/issues/comments/${notice.id}/reactions`)),
|
||||
)
|
||||
).flat();
|
||||
const candidates = await Promise.all(
|
||||
pending.candidates.map((candidate) => api.request<Issue>("GET", `/repos/${config.repo}/issues/${candidate}`)),
|
||||
);
|
||||
const verdict = duplicateTarget(issue, candidates, reactions);
|
||||
if (verdict.kind === "close" && !config.dryRun) {
|
||||
await closeAsDuplicate(api, config, issue.number, verdict.duplicateOf);
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function describe(issue: Issue, verdict: CloseVerdict, dryRun: boolean): string {
|
||||
if (verdict.kind === "skip") {
|
||||
return `#${issue.number}: skipped, ${verdict.reason}`;
|
||||
}
|
||||
return `#${issue.number}: ${dryRun ? "would close" : "closed"} as a duplicate of #${verdict.duplicateOf}`;
|
||||
}
|
||||
|
||||
export async function sweep(api: GitHubApi, config: SweepConfig): Promise<readonly CloseVerdict[]> {
|
||||
const issues = await listAll<Issue>(api, `/repos/${config.repo}/issues?state=open&labels=${FLAG_LABEL}`);
|
||||
console.log(`${issues.length} open issues carry the ${FLAG_LABEL} label in ${config.repo}${config.dryRun ? " (dry run)" : ""}`);
|
||||
return issues.reduce<Promise<readonly CloseVerdict[]>>(async (previous, issue) => {
|
||||
const verdicts = await previous;
|
||||
const verdict = await sweepIssue(api, config, issue);
|
||||
console.log(describe(issue, verdict, config.dryRun));
|
||||
return [...verdicts, verdict];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
function describeReopen(issueNumber: number, verdict: ReopenVerdict, dryRun: boolean): string {
|
||||
if (verdict.kind === "skip") {
|
||||
return `#${issueNumber}: skipped, ${verdict.reason}`;
|
||||
}
|
||||
return `#${issueNumber}: ${dryRun ? "would reopen" : "reopened"} for the reporter's reply`;
|
||||
}
|
||||
|
||||
export async function reopenSweep(api: GitHubApi, config: SweepConfig): Promise<readonly ReopenVerdict[]> {
|
||||
const since = new Date(config.now.getTime() - REOPEN_LOOKBACK_DAYS * DAY_MS).toISOString();
|
||||
const closedPath = `/repos/${config.repo}/issues?state=closed&labels=duplicate,${FLAG_LABEL}&since=${encodeURIComponent(since)}`;
|
||||
const issues = await listAll<Issue>(api, closedPath);
|
||||
console.log(`${issues.length} recently closed issues carry the duplicate and ${FLAG_LABEL} labels in ${config.repo}${config.dryRun ? " (dry run)" : ""}`);
|
||||
return issues.reduce<Promise<readonly ReopenVerdict[]>>(async (previous, issue) => {
|
||||
const verdicts = await previous;
|
||||
const verdict = await sweepClosedIssue(api, config, issue.number);
|
||||
console.log(describeReopen(issue.number, verdict, config.dryRun));
|
||||
return [...verdicts, verdict];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
export function readConfig(env: Readonly<Record<string, string | undefined>>, now: Date): SweepConfig & { readonly token: string } {
|
||||
const token = env.GITHUB_TOKEN;
|
||||
const repo = env.GITHUB_REPOSITORY;
|
||||
if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) {
|
||||
throw new Error("GITHUB_TOKEN and GITHUB_REPOSITORY (owner/repo) are required");
|
||||
}
|
||||
const rawGraceDays = env.GRACE_PERIOD_DAYS?.trim();
|
||||
const graceDays = rawGraceDays === undefined || rawGraceDays === "" ? DEFAULT_GRACE_DAYS : Number(rawGraceDays);
|
||||
if (!Number.isFinite(graceDays) || graceDays < 0) {
|
||||
throw new Error(`GRACE_PERIOD_DAYS must be a non-negative number, got "${env.GRACE_PERIOD_DAYS}"`);
|
||||
}
|
||||
return { token, repo, graceDays, dryRun: env.DRY_RUN === "true", now };
|
||||
}
|
||||
|
||||
export function githubApi(token: string): GitHubApi {
|
||||
return {
|
||||
request: async <T>(method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object): Promise<T> => {
|
||||
const response = await fetch(`https://api.github.com${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "litellm-auto-close-duplicates",
|
||||
...(body === undefined ? {} : { "Content-Type": "application/json" }),
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`${method} ${path} failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const { token, ...config } = readConfig(process.env, new Date());
|
||||
const api = githubApi(token);
|
||||
const closeVerdicts = await sweep(api, config);
|
||||
const reopenVerdicts = await reopenSweep(api, config);
|
||||
const closed = closeVerdicts.filter((verdict) => verdict.kind === "close").length;
|
||||
const reopened = reopenVerdicts.filter((verdict) => verdict.kind === "reopen").length;
|
||||
console.log(
|
||||
`${config.dryRun ? "Would close" : "Closed"} ${closed} of ${closeVerdicts.length} flagged issues, ${config.dryRun ? "would reopen" : "reopened"} ${reopened}`,
|
||||
);
|
||||
}
|
||||
|
|
@ -1317,6 +1317,62 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch):
|
|||
assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_team_settings_newrelic_resolves_traces_and_metrics():
|
||||
"""Static `default_team_settings` is the config-file twin of POST /team/callback.
|
||||
|
||||
A team pinned to New Relic through `default_team_settings` must reach the
|
||||
same two loggers the dynamic path does: the per-team metrics logger (cost
|
||||
and usage) and the trace logger (LLM/agent spans). This proves the static
|
||||
path resolves both, not just one, so the config-file customer gets the
|
||||
same per-team routing as the API customer.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
pc = ProxyConfig()
|
||||
pc.config = {
|
||||
"litellm_settings": {
|
||||
"default_team_settings": [
|
||||
{
|
||||
"team_id": "team-a",
|
||||
"success_callback": ["newrelic"],
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config(
|
||||
team_id="team-a",
|
||||
proxy_config=pc,
|
||||
)
|
||||
|
||||
assert callback_metadata is not None
|
||||
assert callback_metadata.success_callback == ["newrelic"]
|
||||
assert callback_metadata.callback_vars == {
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
|
||||
logging_obj = Logging(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="static-nr-1",
|
||||
function_id="static-nr-1",
|
||||
)
|
||||
logging_obj._trusted_callback_vars = tuple(callback_metadata.callback_vars.items())
|
||||
|
||||
resolved = logging_obj._resolve_dynamic_callback_string("newrelic")
|
||||
resolved_names = {type(logger).__name__ for logger in resolved}
|
||||
assert resolved_names == {"NewRelicMetricsLogger", "NewRelicLogger"}
|
||||
|
||||
|
||||
def test_proxy_config_state_get_config_state_error():
|
||||
"""
|
||||
Ensures that get_config_state does not raise an error when the config is not a valid dictionary
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
|
|||
builders, and the registry validator's failure paths. Needs the OTel SDK."""
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -215,6 +216,27 @@ def test_genai_mapper_all_request_params():
|
|||
assert attrs["server.port"] == 443
|
||||
|
||||
|
||||
def test_genai_mapper_cache_token_attrs():
|
||||
cached = replace(
|
||||
_full_llm_call(),
|
||||
usage=LLMUsage(
|
||||
input_tokens=10,
|
||||
output_tokens=5,
|
||||
total_tokens=15,
|
||||
cache_creation_input_tokens=7,
|
||||
cache_read_input_tokens=3,
|
||||
),
|
||||
)
|
||||
attrs = GenAIMapper().map(cached)
|
||||
assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7
|
||||
assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3
|
||||
|
||||
# No cache usage keeps the span sparse: neither key present.
|
||||
uncached = GenAIMapper().map(_full_llm_call())
|
||||
assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached
|
||||
assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached
|
||||
|
||||
|
||||
def test_genai_mapper_stamps_input_output_messages():
|
||||
data = LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
|
|
|
|||
|
|
@ -525,6 +525,28 @@ def test_llm_call_adapter_extracts_all_fields():
|
|||
assert data.identity.key_hash == "hsh"
|
||||
|
||||
|
||||
def test_llm_call_adapter_extracts_cache_tokens_from_usage_object():
|
||||
payload = _sample_payload()
|
||||
payload["metadata"] = {
|
||||
**payload["metadata"],
|
||||
"usage_object": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"cache_creation_input_tokens": 7,
|
||||
"cache_read_input_tokens": 3,
|
||||
},
|
||||
}
|
||||
data = LLMCallSpanData.from_standard_logging_payload(payload)
|
||||
assert data.usage.cache_creation_input_tokens == 7
|
||||
assert data.usage.cache_read_input_tokens == 3
|
||||
|
||||
|
||||
def test_llm_call_adapter_cache_tokens_none_without_usage_object():
|
||||
data = LLMCallSpanData.from_standard_logging_payload(_sample_payload())
|
||||
assert data.usage.cache_creation_input_tokens is None
|
||||
assert data.usage.cache_read_input_tokens is None
|
||||
|
||||
|
||||
def test_llm_call_adapter_failure_path():
|
||||
payload = _sample_payload(
|
||||
status="failure",
|
||||
|
|
|
|||
|
|
@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens():
|
|||
{"type": "document", "source": source},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_openai_file_block_prices_like_the_equivalent_anthropic_document():
|
||||
"""An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise.
|
||||
|
||||
Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject`
|
||||
is in the union this counter accepts, so every local count of a Responses `input_file` raised
|
||||
`Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens.
|
||||
"""
|
||||
prompt = {"type": "text", "text": "Summarize this file."}
|
||||
inline_file = {
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
|
||||
}
|
||||
document = {
|
||||
"type": "document",
|
||||
"title": "report.pdf",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"},
|
||||
}
|
||||
|
||||
assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document])
|
||||
assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt])
|
||||
|
||||
|
||||
def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
|
||||
"""A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens."""
|
||||
prompt = {"type": "text", "text": "Summarize this file."}
|
||||
|
||||
by_id = {"type": "file", "file": {"file_id": "file-abc123"}}
|
||||
assert _count_user_content([prompt, by_id]) == _count_user_content([prompt])
|
||||
|
||||
named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}}
|
||||
assert _count_user_content([prompt, named]) == _count_user_content(
|
||||
[prompt, {"type": "text", "text": "report.pdf"}]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -957,6 +957,28 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools():
|
|||
assert fields["tools"][0]["type"] == "computer_20250124"
|
||||
|
||||
|
||||
def test_config_blocks_do_not_leak_into_inference_config():
|
||||
"""Regression: inferenceConfig was built before the config blocks were popped, so a dead
|
||||
nested copy of each block (guardrailConfig, performanceConfig, serviceTier) rode inside
|
||||
inferenceConfig alongside the real top-level one."""
|
||||
data = AmazonConverseConfig()._transform_request_helper(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
system_content_blocks=[],
|
||||
optional_params={
|
||||
"maxTokens": 100,
|
||||
"guardrailConfig": {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"},
|
||||
"performanceConfig": {"latency": "optimized"},
|
||||
"serviceTier": {"type": "priority"},
|
||||
},
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert data["inferenceConfig"] == {"maxTokens": 100}
|
||||
assert data["guardrailConfig"] == {"guardrailIdentifier": "gr-id", "guardrailVersion": "DRAFT"}
|
||||
assert data["performanceConfig"] == {"latency": "optimized"}
|
||||
assert data["serviceTier"] == {"type": "priority"}
|
||||
|
||||
|
||||
def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch):
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
|
|
@ -2853,17 +2875,11 @@ def test_guarded_text_guardrail_config_preserved():
|
|||
headers={},
|
||||
)
|
||||
|
||||
# GuardrailConfig should be present at top level
|
||||
assert "guardrailConfig" in result
|
||||
assert result["guardrailConfig"]["guardrailIdentifier"] == "gr-abc123"
|
||||
|
||||
# GuardrailConfig should also be in inferenceConfig
|
||||
assert "inferenceConfig" in result
|
||||
assert "guardrailConfig" in result["inferenceConfig"]
|
||||
assert (
|
||||
result["inferenceConfig"]["guardrailConfig"]["guardrailIdentifier"]
|
||||
== "gr-abc123"
|
||||
)
|
||||
assert "guardrailConfig" not in result["inferenceConfig"]
|
||||
|
||||
|
||||
def test_auto_convert_last_user_message_to_guarded_text():
|
||||
|
|
|
|||
|
|
@ -163,6 +163,240 @@ def test_messages_to_responses_input_with_tool():
|
|||
}
|
||||
|
||||
|
||||
def test_messages_to_responses_input_preserves_images():
|
||||
"""An image block must survive the round trip, or OpenAI counts only the text.
|
||||
|
||||
A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it
|
||||
turned a 268-token request into a 13-token one.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert instructions is None
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,iVBORw0KGgo=",
|
||||
"detail": "high",
|
||||
},
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_image_without_detail_defaults_to_auto():
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_bare_string_image_url_is_preserved():
|
||||
messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string():
|
||||
"""Text-only content must keep collapsing to a string so existing counts do not shift."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "user", "content": "first\nsecond"}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_drops_unmappable_blocks():
|
||||
"""A block with no Responses API equivalent is skipped, never forwarded verbatim."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "hi"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items[0]["content"] == (
|
||||
{"type": "input_text", "text": "hi"},
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
)
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string():
|
||||
"""An assistant turn must never forward chat `text` blocks.
|
||||
|
||||
The Responses API only accepts output_text and refusal inside an assistant turn, so
|
||||
forwarding them 400s the whole request and silently drops the count back to the local
|
||||
tokenizer, which is exactly what defeats the image fix above.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "Paris."}]},
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
{"role": "assistant", "content": "Paris."},
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_image_block_is_dropped():
|
||||
"""An image part is illegal inside an assistant turn, so it must not reach the provider."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here it is"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "assistant", "content": "Here it is"}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn():
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
],
|
||||
},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "What is in this image?"},
|
||||
{"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
|
||||
),
|
||||
},
|
||||
{"role": "assistant", "content": "A cat."},
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_preserves_inline_files():
|
||||
"""An inline file must survive the round trip, or the count silently drops the file.
|
||||
|
||||
A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same
|
||||
request counting 13, the text-only total.
|
||||
"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this file."},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
{"type": "input_text", "text": "Summarize this file."},
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "report.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
},
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_drops_a_file_with_no_inline_data():
|
||||
"""OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this file."},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}},
|
||||
{"type": "file", "file": {"file_id": "file-abc123"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "user", "content": "Summarize this file."}]
|
||||
|
||||
|
||||
def test_messages_to_responses_input_assistant_file_block_is_dropped():
|
||||
"""A file part is illegal inside an assistant turn, so it must not reach the provider."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here it is"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
|
||||
|
||||
assert input_items == [{"role": "assistant", "content": "Here it is"}]
|
||||
|
||||
|
||||
def test_validate_request_valid():
|
||||
"""Test that valid requests pass validation."""
|
||||
config = OpenAICountTokensConfig()
|
||||
|
|
|
|||
238
tests/test_litellm/llms/openai/test_openai_workload_identity.py
Normal file
238
tests/test_litellm/llms/openai/test_openai_workload_identity.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig
|
||||
from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError
|
||||
from litellm.llms.openai.openai import OpenAIChatCompletion
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.openai.workload_identity import (
|
||||
OpenAIWorkloadIdentityConfig,
|
||||
_workload_identity_auth,
|
||||
get_workload_identity_bearer_token,
|
||||
resolve_openai_workload_identity_config,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
TOKEN_EXCHANGE_URL: Final = "https://auth.openai.com/oauth/token"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> OpenAIWorkloadIdentityConfig:
|
||||
token_file: Final = tmp_path / "subject_token.jwt"
|
||||
token_file.write_text("subject-token-from-file")
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123")
|
||||
monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456")
|
||||
monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file))
|
||||
_workload_identity_auth.cache_clear()
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
return OpenAIWorkloadIdentityConfig(
|
||||
identity_provider_id="idp_test123",
|
||||
service_account_id="user-test456",
|
||||
token_file=str(token_file),
|
||||
)
|
||||
|
||||
|
||||
def mock_token_exchange(access_token: str = "exchanged-bearer-token") -> respx.Route:
|
||||
return respx.post(TOKEN_EXCHANGE_URL).mock(
|
||||
return_value=httpx.Response(200, json={"access_token": access_token, "expires_in": 3600})
|
||||
)
|
||||
|
||||
|
||||
class TestResolveConfig:
|
||||
def test_resolves_from_env(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
|
||||
|
||||
def test_static_api_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key="sk-static", api_base=None) is None
|
||||
|
||||
def test_env_openai_api_key_wins(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
|
||||
|
||||
@pytest.mark.parametrize("empty_key", ["", " "])
|
||||
def test_empty_api_key_arg_does_not_disable_wif(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, empty_key: str
|
||||
) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key=empty_key, api_base=None) == wif_env
|
||||
|
||||
@pytest.mark.parametrize("empty_key", ["", " "])
|
||||
def test_empty_env_openai_api_key_does_not_disable_wif(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, empty_key: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", empty_key)
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
|
||||
|
||||
def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None
|
||||
|
||||
def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env
|
||||
|
||||
def test_plaintext_http_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base="http://api.openai.com/v1") is None
|
||||
|
||||
def test_foreign_env_base_url_disables(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://my-vllm.internal/v1")
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
|
||||
|
||||
def test_openai_env_base_url_allows(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
|
||||
|
||||
def test_foreign_litellm_api_base_disables(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "api_base", "https://my-vllm.internal/v1")
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing_var",
|
||||
["OPENAI_IDENTITY_PROVIDER_ID", "OPENAI_SERVICE_ACCOUNT_ID", "OPENAI_IDENTITY_TOKEN_FILE"],
|
||||
)
|
||||
def test_partial_env_disables(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, missing_var: str
|
||||
) -> None:
|
||||
monkeypatch.delenv(missing_var)
|
||||
assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
|
||||
|
||||
|
||||
class TestTokenExchange:
|
||||
@respx.mock
|
||||
def test_exchanges_subject_token_for_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
route: Final = mock_token_exchange()
|
||||
assert get_workload_identity_bearer_token(wif_env) == "exchanged-bearer-token"
|
||||
request_body: Final = json.loads(route.calls.last.request.content)
|
||||
assert request_body["grant_type"] == "urn:ietf:params:oauth:grant-type:token-exchange"
|
||||
assert request_body["subject_token"] == "subject-token-from-file"
|
||||
assert request_body["subject_token_type"] == "urn:ietf:params:oauth:token-type:jwt"
|
||||
assert request_body["identity_provider_id"] == "idp_test123"
|
||||
assert request_body["service_account_id"] == "user-test456"
|
||||
|
||||
@respx.mock
|
||||
def test_token_cached_across_mints(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
route: Final = mock_token_exchange()
|
||||
first: Final = get_workload_identity_bearer_token(wif_env)
|
||||
second: Final = get_workload_identity_bearer_token(wif_env)
|
||||
assert first == second == "exchanged-bearer-token"
|
||||
assert route.call_count == 1
|
||||
|
||||
def test_old_sdk_raises_upgrade_error(
|
||||
self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import openai as openai_module
|
||||
|
||||
monkeypatch.delattr(openai_module, "auth", raising=False)
|
||||
monkeypatch.setitem(sys.modules, "openai.auth", None)
|
||||
with pytest.raises(OpenAIError, match=r"openai>=2\.32\.0"):
|
||||
wif_env.to_sdk_workload_identity()
|
||||
|
||||
|
||||
class TestClientConstruction:
|
||||
def test_sync_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None)
|
||||
assert isinstance(client, OpenAI)
|
||||
assert client.api_key == "workload-identity-auth"
|
||||
assert client._workload_identity_auth is not None
|
||||
|
||||
def test_async_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
client: Final = OpenAIChatCompletion()._get_openai_client(is_async=True, api_key=None, api_base=None)
|
||||
assert isinstance(client, AsyncOpenAI)
|
||||
assert client.api_key == "workload-identity-auth"
|
||||
assert client._workload_identity_auth is not None
|
||||
|
||||
def test_static_key_client_unaffected(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key="sk-static", api_base=None)
|
||||
assert isinstance(client, OpenAI)
|
||||
assert client.api_key == "sk-static"
|
||||
assert client._workload_identity_auth is None
|
||||
|
||||
def test_cache_key_separates_wif_identities(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
other_config: Final = OpenAIWorkloadIdentityConfig(
|
||||
identity_provider_id="idp_other",
|
||||
service_account_id="user-other",
|
||||
token_file=wif_env.token_file,
|
||||
)
|
||||
keys: Final = tuple(
|
||||
BaseOpenAILLM.get_openai_client_cache_key(
|
||||
client_initialization_params={"api_key": None, "is_async": False, "workload_identity_config": config},
|
||||
client_type="openai",
|
||||
)
|
||||
for config in (wif_env, other_config, None)
|
||||
)
|
||||
assert len(set(keys)) == 3
|
||||
|
||||
@respx.mock
|
||||
def test_request_carries_exchanged_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
mock_token_exchange()
|
||||
completion_route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-wif",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
client = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None)
|
||||
assert isinstance(client, OpenAI)
|
||||
client.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}])
|
||||
auth_header: Final = completion_route.calls.last.request.headers["Authorization"]
|
||||
assert auth_header == "Bearer exchanged-bearer-token"
|
||||
|
||||
|
||||
class TestResponsesValidateEnvironment:
|
||||
@respx.mock
|
||||
def test_mints_bearer_when_wif_configured(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
mock_token_exchange()
|
||||
headers: Final = OpenAIResponsesAPIConfig().validate_environment(
|
||||
headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams()
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer exchanged-bearer-token"
|
||||
|
||||
def test_static_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
headers: Final = OpenAIResponsesAPIConfig().validate_environment(
|
||||
headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams(api_key="sk-responses")
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer sk-responses"
|
||||
|
||||
def test_foreign_api_base_skips_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
headers: Final = OpenAIResponsesAPIConfig().validate_environment(
|
||||
headers={},
|
||||
model="gpt-4o-mini",
|
||||
litellm_params=GenericLiteLLMParams(api_base="https://my-vllm.internal/v1"),
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer None"
|
||||
|
||||
def test_litellm_proxy_subclass_never_mints_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
|
||||
headers: Final = LiteLLMProxyResponsesAPIConfig().validate_environment(
|
||||
headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams()
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer None"
|
||||
|
|
@ -982,6 +982,116 @@ class TestVertexBase:
|
|||
|
||||
assert result_url == f"{gateway_api_base}:embedContent"
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_with_version_path_grafts_default_path(self):
|
||||
vertex_base = VertexBase()
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base="https://aiplatform.googleapis.com/v1beta1",
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="generateContent",
|
||||
stream=None,
|
||||
auth_header="Bearer token123",
|
||||
url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent",
|
||||
model="gemini-3.5-flash-lite",
|
||||
)
|
||||
|
||||
assert (
|
||||
result_url
|
||||
== "https://aiplatform.googleapis.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent"
|
||||
)
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_with_version_path_trailing_slash_grafts_default_path(self):
|
||||
vertex_base = VertexBase()
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base="https://internal-gateway.example.com/v1/",
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="generateContent",
|
||||
stream=None,
|
||||
auth_header="Bearer token123",
|
||||
url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent",
|
||||
model="gemini-3.5-flash-lite",
|
||||
)
|
||||
|
||||
assert (
|
||||
result_url
|
||||
== "https://internal-gateway.example.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent"
|
||||
)
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_grafts_before_query(self):
|
||||
vertex_base = VertexBase()
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base="https://internal-gateway.example.com/v1beta1?key=abc",
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="generateContent",
|
||||
stream=None,
|
||||
auth_header="Bearer token123",
|
||||
url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent",
|
||||
model="gemini-3.5-flash-lite",
|
||||
)
|
||||
|
||||
assert (
|
||||
result_url
|
||||
== "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent?key=abc"
|
||||
)
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_with_version_path_and_query_streaming_appends_alt_sse(self):
|
||||
vertex_base = VertexBase()
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base="https://internal-gateway.example.com/v1beta1?key=abc",
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="streamGenerateContent",
|
||||
stream=True,
|
||||
auth_header="Bearer token123",
|
||||
url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent",
|
||||
model="gemini-3.5-flash-lite",
|
||||
)
|
||||
|
||||
assert (
|
||||
result_url
|
||||
== "https://internal-gateway.example.com/v1beta1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:streamGenerateContent?key=abc&alt=sse"
|
||||
)
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_with_non_version_path_keeps_endpoint_append(self):
|
||||
vertex_base = VertexBase()
|
||||
gateway_api_base = "https://gateway.example.com/vertex-proxy"
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base=gateway_api_base,
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="generateContent",
|
||||
stream=None,
|
||||
auth_header="Bearer token123",
|
||||
url="https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent",
|
||||
model="gemini-3.5-flash-lite",
|
||||
)
|
||||
|
||||
assert result_url == f"{gateway_api_base}:generateContent"
|
||||
|
||||
def test_check_custom_proxy_vertex_api_base_without_projects_in_default_url_keeps_endpoint_append(self):
|
||||
vertex_base = VertexBase()
|
||||
gemma_api_base = "https://example.com/custom/gemma-deployment"
|
||||
|
||||
_, result_url = vertex_base._check_custom_proxy(
|
||||
api_base=gemma_api_base,
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="predict",
|
||||
stream=False,
|
||||
auth_header=None,
|
||||
url=gemma_api_base,
|
||||
model="gemma-3-27b-it",
|
||||
)
|
||||
|
||||
assert result_url == f"{gemma_api_base}:predict"
|
||||
|
||||
def test_check_custom_proxy_vertex_bare_host_streaming_keeps_single_alt_sse(self):
|
||||
vertex_base = VertexBase()
|
||||
|
||||
|
|
|
|||
|
|
@ -432,7 +432,7 @@ class TestHiddenlayerGuardrail:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_request_with_image(self, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test apply_guardrail sends multimodal content (image) to HiddenLayer v1."""
|
||||
"""Test apply_guardrail strips images from multimodal content before sending to HiddenLayer v1."""
|
||||
monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer")
|
||||
|
||||
guardrail = HiddenlayerGuardrail(
|
||||
|
|
@ -485,12 +485,13 @@ class TestHiddenlayerGuardrail:
|
|||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
# v1 API requires string content — multimodal list is stringified
|
||||
# v1 API requires string content — image_url items are stripped and the
|
||||
# remaining (text-only) content is stringified before being sent.
|
||||
mock_post.assert_called_once()
|
||||
call_kwargs = mock_post.call_args.kwargs
|
||||
sent_content = call_kwargs["json"]["input"]["messages"][0]["content"]
|
||||
assert isinstance(sent_content, str)
|
||||
assert sent_content == str(multimodal_content)
|
||||
assert sent_content == str([{"type": "text", "text": "how much is on this receipt?"}])
|
||||
|
||||
# Result should be returned without error
|
||||
assert result is not None
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
from fastapi.exceptions import HTTPException
|
||||
from unittest.mock import patch, AsyncMock
|
||||
from httpx import Response, Request
|
||||
import asyncio
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import (
|
||||
PromptSecurityGuardrailMissingSecrets,
|
||||
PromptSecurityGuardrail,
|
||||
)
|
||||
from fastapi.exceptions import HTTPException
|
||||
from httpx import ReadTimeout, Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import (
|
||||
PromptSecurityGuardrail,
|
||||
PromptSecurityGuardrailMissingSecrets,
|
||||
)
|
||||
from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2
|
||||
|
||||
|
||||
|
|
@ -30,6 +30,7 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch):
|
|||
"guardrail": "prompt_security",
|
||||
"mode": "during_call",
|
||||
"default_on": True,
|
||||
"file_sanitization_fail_open": False,
|
||||
},
|
||||
}
|
||||
],
|
||||
|
|
@ -41,6 +42,10 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch):
|
|||
assert registered[0].guardrail_name == "prompt_security"
|
||||
assert registered[0].default_on is True
|
||||
assert registered[0].event_hook == "during_call"
|
||||
assert registered[0].file_sanitization_fail_open is False
|
||||
config_model = registered[0].get_config_model()
|
||||
assert config_model is not None
|
||||
assert config_model().file_sanitization_fail_open is True
|
||||
|
||||
|
||||
def test_prompt_security_guard_config_no_api_key(monkeypatch: pytest.MonkeyPatch):
|
||||
|
|
@ -374,6 +379,86 @@ async def test_file_sanitization(monkeypatch: pytest.MonkeyPatch):
|
|||
assert result is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"timeout",
|
||||
(
|
||||
litellm.Timeout(
|
||||
message="Prompt Security upload timed out",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
),
|
||||
ReadTimeout(
|
||||
"Prompt Security poll timed out",
|
||||
request=Request(method="GET", url="https://test.prompt.security/api/sanitizeFile"),
|
||||
),
|
||||
),
|
||||
ids=("litellm", "httpx"),
|
||||
)
|
||||
@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed"))
|
||||
async def test_file_sanitization_request_timeout_policy(
|
||||
monkeypatch: pytest.MonkeyPatch, timeout: Exception, fail_open: bool
|
||||
):
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key")
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security")
|
||||
|
||||
guardrail = PromptSecurityGuardrail(
|
||||
guardrail_name="test-guard",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
file_sanitization_fail_open=fail_open,
|
||||
)
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", AsyncMock(side_effect=timeout)):
|
||||
if not fail_open:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
assert exc_info.value.status_code == 408
|
||||
assert exc_info.value.detail == "File sanitization timeout"
|
||||
return
|
||||
|
||||
result = await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
|
||||
assert result == {
|
||||
"action": "allow",
|
||||
"content": None,
|
||||
"metadata": {},
|
||||
"violations": (),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed"))
|
||||
async def test_file_sanitization_overall_timeout_policy(monkeypatch: pytest.MonkeyPatch, fail_open: bool):
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key")
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security")
|
||||
|
||||
guardrail = PromptSecurityGuardrail(
|
||||
guardrail_name="test-guard",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
file_sanitization_timeout=0.01,
|
||||
file_sanitization_fail_open=fail_open,
|
||||
)
|
||||
|
||||
async def hanging_post(*_args: object, **_kwargs: object) -> None:
|
||||
await asyncio.sleep(60)
|
||||
raise AssertionError("sanitization request should have been cancelled")
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", side_effect=hanging_post):
|
||||
if not fail_open:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
assert exc_info.value.status_code == 408
|
||||
assert exc_info.value.detail == "File sanitization timeout"
|
||||
return
|
||||
|
||||
result = await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
|
||||
assert result["action"] == "allow"
|
||||
assert result["content"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_sanitization_block(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that file sanitization blocks malicious files"""
|
||||
|
|
@ -544,7 +629,7 @@ async def test_role_filtering(monkeypatch: pytest.MonkeyPatch):
|
|||
return mock_response
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", side_effect=mock_post):
|
||||
result = await guardrail.apply_guardrail(
|
||||
await guardrail.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
|
|
|
|||
|
|
@ -83,6 +83,50 @@ def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth):
|
|||
assert response.json()["alias"] == "Updated Test User"
|
||||
|
||||
|
||||
def test_update_customer_unblock(mock_prisma_client, mock_user_api_key_auth):
|
||||
mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
|
||||
updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=False)
|
||||
|
||||
mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user)
|
||||
mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user)
|
||||
|
||||
response = client.post(
|
||||
"/customer/update",
|
||||
json={"user_id": "test-user-1", "blocked": False},
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["blocked"] is False
|
||||
update_mock = mock_prisma_client.db.litellm_endusertable.update
|
||||
update_mock.assert_called_once()
|
||||
assert update_mock.call_args.kwargs["data"]["blocked"] is False
|
||||
|
||||
|
||||
def test_update_customer_keeps_blocked_when_omitted(mock_prisma_client, mock_user_api_key_auth):
|
||||
"""
|
||||
Regression test: updating a blocked customer without supplying `blocked`
|
||||
must NOT reset it to unblocked. `blocked=False` is the model default and
|
||||
should only be applied when explicitly provided by the caller.
|
||||
"""
|
||||
mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
|
||||
updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
|
||||
|
||||
mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user)
|
||||
mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user)
|
||||
|
||||
response = client.post(
|
||||
"/customer/update",
|
||||
json={"user_id": "test-user-1", "alias": "Updated Test User"},
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
update_mock = mock_prisma_client.db.litellm_endusertable.update
|
||||
update_mock.assert_called_once()
|
||||
assert "blocked" not in update_mock.call_args.kwargs["data"]
|
||||
|
||||
|
||||
def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
|
||||
"""
|
||||
Test that update_end_user raises a 404 ProxyException when user_id does not exist.
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py
|
|||
"""
|
||||
|
||||
import unittest
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import Response
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
|
@ -82,11 +84,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
ResponseOutputMessage(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
type="output_text", text="Hello from Cursor!"
|
||||
)
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -121,9 +119,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
@pytest.mark.asyncio
|
||||
@patch("litellm.proxy.proxy_server.llm_router")
|
||||
@patch("litellm.proxy.proxy_server.user_api_key_auth")
|
||||
async def test_responses_api_key_spend_header_includes_response_cost(
|
||||
self, mock_auth, mock_router
|
||||
):
|
||||
async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router):
|
||||
"""
|
||||
Test that x-litellm-key-spend header includes the current request's response_cost
|
||||
for /v1/responses endpoint.
|
||||
|
|
@ -159,9 +155,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
|
|||
ResponseOutputMessage(
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[
|
||||
ResponseOutputText(type="output_text", text="Test response")
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="Test response")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -356,6 +350,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "model": "gpt-4o", "input": "hello"}
|
||||
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
|
||||
|
||||
|
|
@ -363,6 +358,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}}
|
||||
assert _extract_model_from_first_ws_event(event) == "gpt-4o"
|
||||
|
||||
|
|
@ -370,6 +366,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {
|
||||
"type": "response.create",
|
||||
"model": "flat-model",
|
||||
|
|
@ -381,6 +378,7 @@ class TestWSModelExtraction:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
_extract_model_from_first_ws_event,
|
||||
)
|
||||
|
||||
event = {"type": "response.create", "input": "hello"}
|
||||
assert _extract_model_from_first_ws_event(event) is None
|
||||
|
||||
|
|
@ -400,9 +398,7 @@ class TestResponsesWSFirstFrameValidation:
|
|||
)
|
||||
|
||||
ws = MagicMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})
|
||||
)
|
||||
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}))
|
||||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
||||
|
|
@ -412,10 +408,7 @@ class TestResponsesWSFirstFrameValidation:
|
|||
ws.send_text.assert_awaited_once()
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message")
|
||||
error_payload = json.loads(ws.send_text.await_args.args[0])
|
||||
assert (
|
||||
error_payload["error"]["message"]
|
||||
== "First message must be a response.create JSON object."
|
||||
)
|
||||
assert error_payload["error"]["message"] == "First message must be a response.create JSON object."
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_non_object_json_first_frame(self):
|
||||
|
|
@ -484,16 +477,12 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
ws.url = "ws://testserver/v1/responses"
|
||||
ws.accept = AsyncMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps(
|
||||
{"type": "response.create", "model": "gpt-4o-mini", "input": []}
|
||||
)
|
||||
return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
|
||||
)
|
||||
ws.close = AsyncMock()
|
||||
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini"}, MagicMock())
|
||||
)
|
||||
processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock()))
|
||||
|
||||
async def fake_llm_call():
|
||||
return None
|
||||
|
|
@ -529,9 +518,7 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
_enforce_responses_ws_first_frame_model_auth,
|
||||
)
|
||||
|
||||
request = Request(
|
||||
{"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}
|
||||
)
|
||||
request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []})
|
||||
user_api_key_dict = MagicMock()
|
||||
llm_router = MagicMock()
|
||||
|
||||
|
|
@ -593,9 +580,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
|
||||
assert result is None
|
||||
ws.send_text.assert_not_awaited()
|
||||
ws.close.assert_awaited_once_with(
|
||||
code=1008, reason="Timed out waiting for first message"
|
||||
)
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_json_sends_error_and_closes(self):
|
||||
|
|
@ -613,9 +598,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
assert result is None
|
||||
payload = json.loads(ws.send_text.await_args.args[0])
|
||||
assert payload["error"]["message"] == "First message is not valid JSON."
|
||||
ws.close.assert_awaited_once_with(
|
||||
code=1008, reason="Invalid JSON in first message"
|
||||
)
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_model_sends_error_and_closes(self):
|
||||
|
|
@ -624,9 +607,7 @@ class TestReadWSModelFromFirstFrameErrors:
|
|||
)
|
||||
|
||||
ws = MagicMock()
|
||||
ws.receive_text = AsyncMock(
|
||||
return_value=json.dumps({"type": "response.create", "input": []})
|
||||
)
|
||||
ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []}))
|
||||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
||||
|
|
@ -679,10 +660,7 @@ class TestManagedResponsesSameProvider:
|
|||
assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True
|
||||
|
||||
def test_different_provider_is_not_same(self):
|
||||
assert (
|
||||
self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash")
|
||||
is False
|
||||
)
|
||||
assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False
|
||||
|
||||
def test_inject_credentials_keeps_provider_for_same_provider_model(self):
|
||||
handler = self._handler("gpt-4o", custom_llm_provider="openai")
|
||||
|
|
@ -697,18 +675,14 @@ class TestManagedResponsesSameProvider:
|
|||
assert "custom_llm_provider" not in call_kwargs
|
||||
|
||||
def test_unresolvable_connection_model_falls_back_to_custom_provider(self):
|
||||
handler = self._handler(
|
||||
"my-custom-deployment", custom_llm_provider="openai"
|
||||
)
|
||||
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
|
||||
assert handler._same_provider("gpt-4o-mini") is True
|
||||
call_kwargs: dict = {}
|
||||
handler._inject_credentials(call_kwargs, model="gpt-4o-mini")
|
||||
assert call_kwargs["custom_llm_provider"] == "openai"
|
||||
|
||||
def test_unresolvable_connection_model_still_drops_cross_provider(self):
|
||||
handler = self._handler(
|
||||
"my-custom-deployment", custom_llm_provider="openai"
|
||||
)
|
||||
handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
|
||||
call_kwargs: dict = {}
|
||||
handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash")
|
||||
assert "custom_llm_provider" not in call_kwargs
|
||||
|
|
@ -840,9 +814,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
|
|||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[
|
||||
ResponseOutputText(type="output_text", text="agent reply", annotations=[])
|
||||
],
|
||||
content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -851,9 +823,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
|
|||
|
||||
app.dependency_overrides[user_api_key_auth] = _auth_override
|
||||
try:
|
||||
with patch.object(ps, "llm_router", mock_router), patch(
|
||||
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
|
||||
side_effect=capturing_read_request_body,
|
||||
with (
|
||||
patch.object(ps, "llm_router", mock_router),
|
||||
patch(
|
||||
"litellm.proxy.response_api_endpoints.endpoints._read_request_body",
|
||||
side_effect=capturing_read_request_body,
|
||||
),
|
||||
):
|
||||
client = TestClient(app)
|
||||
response = client.post(
|
||||
|
|
@ -1488,8 +1463,8 @@ def _router_serving_only(base_model: str) -> MagicMock:
|
|||
mock_router.router_general_settings.pass_through_all_models = False
|
||||
mock_router.default_deployment = None
|
||||
mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]}
|
||||
mock_router.pattern_router.get_pattern.side_effect = (
|
||||
lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None
|
||||
mock_router.pattern_router.get_pattern.side_effect = lambda model: (
|
||||
[{"model_name": "anthropic/*"}] if model == base_model else None
|
||||
)
|
||||
return mock_router
|
||||
|
||||
|
|
@ -1739,9 +1714,7 @@ class TestCursorGateRecognizesRoutingGroups:
|
|||
from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}
|
||||
],
|
||||
model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}],
|
||||
routing_groups=[
|
||||
{"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"}
|
||||
],
|
||||
|
|
@ -1836,3 +1809,153 @@ class TestGuardrailBlockedResponsesUsage:
|
|||
assert usage["input_tokens"] == 0
|
||||
assert usage["output_tokens"] == 0
|
||||
assert usage["total_tokens"] == 0
|
||||
|
||||
|
||||
class TestResponsesInputTokens:
|
||||
"""Regression tests for POST /v1/responses/input_tokens.
|
||||
|
||||
The docs promise OpenAI-format token counting on the proxy, but the route was
|
||||
never registered, so the POST fell through to the GET/DELETE-only
|
||||
/v1/responses/{response_id} route and returned 405."""
|
||||
|
||||
def _post_input_tokens(
|
||||
self,
|
||||
body: dict[str, Any],
|
||||
path: str = "/v1/responses/input_tokens",
|
||||
counter: AsyncMock | None = None,
|
||||
) -> tuple[Response, AsyncMock]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
token_counter_mock = (
|
||||
counter
|
||||
if counter is not None
|
||||
else AsyncMock(
|
||||
return_value=TokenCountResponse(
|
||||
total_tokens=13,
|
||||
request_model=body.get("model", ""),
|
||||
model_used=body.get("model", ""),
|
||||
tokenizer_type="openai_api",
|
||||
)
|
||||
)
|
||||
)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
|
||||
app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"})
|
||||
return response, token_counter_mock
|
||||
finally:
|
||||
app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
app.dependency_overrides.pop(_proxy_token_counter, None)
|
||||
|
||||
def test_string_input_returns_openai_input_tokens_shape(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"object": "response.input_tokens", "input_tokens": 13}
|
||||
counter.assert_awaited_once()
|
||||
assert counter.call_args.kwargs["call_endpoint"] is True
|
||||
token_request = counter.call_args.kwargs["request"]
|
||||
assert token_request.model == "gpt-4o"
|
||||
assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}]
|
||||
|
||||
def test_every_route_alias_is_registered(self):
|
||||
for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"):
|
||||
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path)
|
||||
assert response.status_code == 200, f"{path}: {response.status_code} {response.text}"
|
||||
|
||||
def test_input_items_instructions_and_tools_are_forwarded(self):
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
}
|
||||
]
|
||||
response, counter = self._post_input_tokens(
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"input": [{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
"instructions": "You are terse.",
|
||||
"tools": tools,
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
token_request = counter.call_args.kwargs["request"]
|
||||
assert token_request.messages == [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": "What is the weather in Paris?"},
|
||||
]
|
||||
assert token_request.tools == tools
|
||||
|
||||
def test_missing_model_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"input": "Hello"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": "Missing required parameter: 'model'.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "model",
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_missing_input_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": "Missing required parameter: 'input'.",
|
||||
"type": "invalid_request_error",
|
||||
"param": "input",
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
@pytest.mark.parametrize("empty_input", ["", []])
|
||||
def test_empty_input_returns_openai_400(self, empty_input):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "missing_required_parameter",
|
||||
}
|
||||
}
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_invalid_tools_returns_openai_400(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"})
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_provider_error_maps_status_code(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
failing_counter = AsyncMock(
|
||||
side_effect=ProxyException(
|
||||
message="rate limited",
|
||||
type="token_counting_error",
|
||||
param="model",
|
||||
code="429",
|
||||
)
|
||||
)
|
||||
response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter)
|
||||
|
||||
assert response.status_code == 429, response.text
|
||||
assert response.json()["error"]["message"] == "rate limited"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
TOKEN_COUNTING_ROUTES: Final = (
|
||||
"/responses/input_tokens",
|
||||
"/v1/responses/input_tokens",
|
||||
"/openai/v1/responses/input_tokens",
|
||||
"/utils/token_counter",
|
||||
)
|
||||
|
||||
|
||||
def _budgeted_token() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0)
|
||||
|
||||
|
||||
async def _reserve(route: str) -> dict | None:
|
||||
return await reserve_budget_for_request(
|
||||
request_body={"model": "gpt-4o", "input": "hello"},
|
||||
route=route,
|
||||
llm_router=None,
|
||||
valid_token=_budgeted_token(),
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=UserApiKeyCache(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES)
|
||||
async def test_token_counting_routes_are_exempt_from_budget_reservation(route):
|
||||
assert await _reserve(route) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_exempt_llm_route_still_reserves_budget():
|
||||
reservation: Final = await _reserve("/v1/responses")
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation["reserved_cost"] > 0
|
||||
|
|
@ -2865,7 +2865,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
@ -2961,7 +2961,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
@ -3055,7 +3055,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import datetime
|
||||
import json
|
||||
from datetime import timezone
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -3956,3 +3957,71 @@ def test_passthrough_caching_carries_no_injection_marker():
|
|||
)
|
||||
metadata = json.loads(payload["metadata"])
|
||||
assert metadata["litellm_gateway_injected_cache"] is None
|
||||
|
||||
|
||||
def _routed_call_kwargs(model_info: Mapping[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model": "claude-haiku-4-5",
|
||||
"custom_llm_provider": "azure_ai",
|
||||
"litellm_call_id": "router-corr-123",
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": "test-key",
|
||||
"model_group": "internal-router/gpt-5.4",
|
||||
"deployment": "azure_ai/claude-haiku-4-5",
|
||||
"model_info": model_info,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_router_metadata_stamped_for_internal_router_model_deployment():
|
||||
"""A deployment flagged model_info.internal_router_model gets a router_metadata
|
||||
block correlating the requested model group with the selected deployment."""
|
||||
payload = get_logging_payload(
|
||||
kwargs=_routed_call_kwargs({"id": "mi-1", "internal_router_model": True}),
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-router-meta", choices=[], usage=litellm.Usage()),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
metadata = json.loads(payload["metadata"])
|
||||
assert metadata["router_metadata"] == {
|
||||
"requested_model": "internal-router/gpt-5.4",
|
||||
"selected_model": "azure_ai/claude-haiku-4-5",
|
||||
"selected_provider": "azure_ai",
|
||||
"router_correlation_id": "router-corr-123",
|
||||
}
|
||||
|
||||
|
||||
def test_router_metadata_absent_without_internal_router_model_flag():
|
||||
payload = get_logging_payload(
|
||||
kwargs=_routed_call_kwargs({"id": "mi-1"}),
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-unflagged", choices=[], usage=litellm.Usage()),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
metadata = json.loads(payload["metadata"])
|
||||
assert metadata["router_metadata"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
|
||||
def test_caller_forged_router_metadata_is_discarded(bucket):
|
||||
"""The raw request bucket is client-writable and _get_spend_logs_metadata projects
|
||||
every SpendLogsMetadata key from it, so the server-derived value must overwrite
|
||||
unconditionally or a caller could plant router provenance the router never produced."""
|
||||
payload = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "gpt-4o-mini",
|
||||
"litellm_params": {
|
||||
bucket: {
|
||||
"user_api_key": "test-key",
|
||||
"router_metadata": {"requested_model": "forged", "router_correlation_id": "forged-id"},
|
||||
}
|
||||
},
|
||||
},
|
||||
response_obj=litellm.ModelResponse(id="chatcmpl-forged-router-meta", choices=[], usage=litellm.Usage()),
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
metadata = json.loads(payload["metadata"])
|
||||
assert metadata["router_metadata"] is None
|
||||
|
|
|
|||
|
|
@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig:
|
|||
assert "extra_field" not in result["file"]
|
||||
assert "another_field" not in result["file"]
|
||||
|
||||
def test_transform_input_file_item_to_file_item_keeps_filename(self):
|
||||
"""OpenAI rejects file_data with no filename beside it, so dropping it 400s the request"""
|
||||
result = (
|
||||
LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item(
|
||||
{
|
||||
"type": "input_file",
|
||||
"filename": "report.pdf",
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
}
|
||||
)
|
||||
)
|
||||
assert result == {
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_data": "data:application/pdf;base64,JVBERi0=",
|
||||
"filename": "report.pdf",
|
||||
},
|
||||
}
|
||||
|
||||
def test_transform_input_file_item_to_file_item_with_file_url(self):
|
||||
"""file_url should be mapped to file_id for downstream URL handling"""
|
||||
result = (
|
||||
|
|
|
|||
177
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
177
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -9700,6 +9700,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/openai/v1/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/openai/v1/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -12619,6 +12650,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -15505,7 +15567,7 @@ export interface paths {
|
|||
* Use this if if you want different teams to have different success/failure callbacks
|
||||
*
|
||||
* Parameters:
|
||||
* - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
* - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
* - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
* - "success": Callback for successful LLM calls
|
||||
* - "failure": Callback for failed LLM calls
|
||||
|
|
@ -15521,6 +15583,8 @@ export interface paths {
|
|||
* - langsmith_api_key: The API key for the Langsmith callback
|
||||
* - langsmith_project: The project for the Langsmith callback
|
||||
* - langsmith_base_url: The base URL for the Langsmith callback
|
||||
* - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
* - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
*
|
||||
* Example curl:
|
||||
* ```
|
||||
|
|
@ -16210,6 +16274,11 @@ export interface paths {
|
|||
*
|
||||
* Meant to optimize querying spend data for analytics for a user.
|
||||
*
|
||||
* Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
* resets. Their total can legitimately exceed the `spend` field returned by
|
||||
* `/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
* to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
*
|
||||
* Returns:
|
||||
* (by date)
|
||||
* - spend
|
||||
|
|
@ -16241,6 +16310,11 @@ export interface paths {
|
|||
* Get User Daily Activity Aggregated
|
||||
* @description Aggregated analytics for a user's daily activity without pagination.
|
||||
* Returns the same response shape as the paginated endpoint with page metadata set to single-page.
|
||||
*
|
||||
* Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
* resets. Their total can legitimately exceed the `spend` field returned by
|
||||
* `/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
* to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
*/
|
||||
get: operations["get_user_daily_activity_aggregated_user_daily_activity_aggregated_get"];
|
||||
put?: never;
|
||||
|
|
@ -19182,6 +19256,37 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/responses/input_tokens": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Responses Input Tokens
|
||||
* @description Count the input tokens of a Responses API request without calling the model.
|
||||
*
|
||||
* Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
|
||||
*
|
||||
* ```bash
|
||||
* curl -X POST http://localhost:4000/v1/responses/input_tokens -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{
|
||||
* "model": "gpt-4o",
|
||||
* "input": "Hello, how are you?"
|
||||
* }'
|
||||
* ```
|
||||
*
|
||||
* Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
|
||||
*/
|
||||
post: operations["responses_input_tokens_v1_responses_input_tokens_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/responses/{response_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -21004,6 +21109,14 @@ export interface paths {
|
|||
* This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem
|
||||
* where the old endpoint loaded all keys and teams into memory.
|
||||
*
|
||||
* Note on `spend`: this is the user's running budget counter, which the budget reset job
|
||||
* resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default,
|
||||
* or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT
|
||||
* lifetime or per-period historical spend. For historical spend over a date range, use
|
||||
* `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend
|
||||
* records that only ever accumulate and are never reset. The two values are expected to
|
||||
* diverge once a budget reset has occurred within the queried period.
|
||||
*
|
||||
* Access control:
|
||||
* - Proxy admins can query any user
|
||||
* - Team admins can query users within their teams
|
||||
|
|
@ -38418,6 +38531,8 @@ export interface components {
|
|||
input_cost_per_character?: number | null;
|
||||
/** Input Cost Per Token */
|
||||
input_cost_per_token?: number | null;
|
||||
/** Internal Router Model */
|
||||
internal_router_model?: boolean | null;
|
||||
/** Output Cost Per Character */
|
||||
output_cost_per_character?: number | null;
|
||||
/** Output Cost Per Token */
|
||||
|
|
@ -51474,6 +51589,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_openai_v1_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_openai_v1_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -54438,6 +54573,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -62860,6 +63015,26 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
responses_input_tokens_v1_responses_input_tokens_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_response_v1_responses__response_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue