mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
chore: merge litellm_internal_staging and resolve prompt security conflicts
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI
commit
d0897caf45
35 changed files with 1416 additions and 542 deletions
230
.github/scripts/close_duplicate_issues.py
vendored
230
.github/scripts/close_duplicate_issues.py
vendored
|
|
@ -1,230 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Detect and close duplicate GitHub issues using title similarity.
|
||||
|
||||
Modes:
|
||||
--scan Compare all open issues against each other (batch)
|
||||
--issue-number N Check a single issue against older open issues
|
||||
|
||||
Requires the `gh` CLI to be authenticated.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
"""Strip common prefixes, lowercase, and collapse whitespace."""
|
||||
title = re.sub(
|
||||
r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
|
||||
"",
|
||||
title,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
return " ".join(title.lower().split())
|
||||
|
||||
|
||||
def gh(*args: str) -> str:
|
||||
"""Run a gh CLI command and return stdout."""
|
||||
result = subprocess.run(
|
||||
["gh", *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def fetch_open_issues(repo: str | None) -> list[dict]:
|
||||
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
|
||||
if repo:
|
||||
endpoint = (
|
||||
f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
|
||||
)
|
||||
else:
|
||||
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
|
||||
cmd = ["api", "--paginate", endpoint]
|
||||
|
||||
raw = gh(*cmd)
|
||||
# gh --paginate concatenates JSON arrays, so we may get multiple arrays
|
||||
issues = []
|
||||
for line in raw.strip().splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parsed = json.loads(line)
|
||||
if isinstance(parsed, list):
|
||||
issues.extend(parsed)
|
||||
else:
|
||||
issues.append(parsed)
|
||||
|
||||
# Filter out pull requests (they also appear in the issues endpoint)
|
||||
return [i for i in issues if "pull_request" not in i]
|
||||
|
||||
|
||||
def close_as_duplicate(
|
||||
issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
|
||||
) -> None:
|
||||
"""Close an issue as duplicate of another, adding a comment and label."""
|
||||
repo_args = ["--repo", repo] if repo else []
|
||||
|
||||
if dry_run:
|
||||
print(
|
||||
f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
|
||||
)
|
||||
return
|
||||
|
||||
# Add comment
|
||||
comment_body = (
|
||||
f"Closing as duplicate of #{duplicate_of}.\n\n"
|
||||
"If you believe this is not a duplicate, please reopen and add context "
|
||||
"explaining how this differs."
|
||||
)
|
||||
gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
|
||||
|
||||
# Add label
|
||||
gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
|
||||
|
||||
# Close with not_planned reason
|
||||
gh(
|
||||
"api",
|
||||
f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
|
||||
"-X",
|
||||
"PATCH",
|
||||
"-f",
|
||||
"state=closed",
|
||||
"-f",
|
||||
"state_reason=not_planned",
|
||||
)
|
||||
|
||||
print(f" Closed #{issue_number} as duplicate of #{duplicate_of}")
|
||||
|
||||
|
||||
def find_duplicate(
|
||||
issue: dict, candidates: list[dict], threshold: float
|
||||
) -> dict | None:
|
||||
"""Return the first candidate whose normalized title is above threshold."""
|
||||
norm = normalize_title(issue["title"])
|
||||
for candidate in candidates:
|
||||
if candidate["number"] == issue["number"]:
|
||||
continue
|
||||
cand_norm = normalize_title(candidate["title"])
|
||||
ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
|
||||
if ratio >= threshold:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def scan_all(
|
||||
issues: list[dict], threshold: float, repo: str | None, dry_run: bool
|
||||
) -> int:
|
||||
"""Compare every issue against all older issues. Returns count of duplicates found."""
|
||||
# Sort oldest first
|
||||
issues.sort(key=lambda i: i["number"])
|
||||
closed_count = 0
|
||||
|
||||
for idx, issue in enumerate(issues):
|
||||
older = issues[:idx]
|
||||
if not older:
|
||||
continue
|
||||
dup = find_duplicate(issue, older, threshold)
|
||||
if dup:
|
||||
ratio = difflib.SequenceMatcher(
|
||||
None,
|
||||
normalize_title(issue["title"]),
|
||||
normalize_title(dup["title"]),
|
||||
).ratio()
|
||||
print(
|
||||
f"#{issue['number']}: \"{issue['title']}\"\n"
|
||||
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
|
||||
f"({ratio:.0%} similar)"
|
||||
)
|
||||
close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
|
||||
closed_count += 1
|
||||
|
||||
return closed_count
|
||||
|
||||
|
||||
def check_single(
|
||||
issue_number: int,
|
||||
issues: list[dict],
|
||||
threshold: float,
|
||||
repo: str | None,
|
||||
dry_run: bool,
|
||||
) -> bool:
|
||||
"""Check a single issue against all older open issues. Returns True if duplicate found."""
|
||||
target = None
|
||||
for i in issues:
|
||||
if i["number"] == issue_number:
|
||||
target = i
|
||||
break
|
||||
|
||||
if target is None:
|
||||
print(f"Issue #{issue_number} not found among open issues.")
|
||||
return False
|
||||
|
||||
older = [i for i in issues if i["number"] < issue_number]
|
||||
dup = find_duplicate(target, older, threshold)
|
||||
if dup:
|
||||
ratio = difflib.SequenceMatcher(
|
||||
None,
|
||||
normalize_title(target["title"]),
|
||||
normalize_title(dup["title"]),
|
||||
).ratio()
|
||||
print(
|
||||
f"#{target['number']}: \"{target['title']}\"\n"
|
||||
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
|
||||
f"({ratio:.0%} similar)"
|
||||
)
|
||||
close_as_duplicate(issue_number, dup["number"], repo, dry_run)
|
||||
return True
|
||||
|
||||
print(f"#{issue_number}: no duplicate found above threshold {threshold}")
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Detect and close duplicate GitHub issues"
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
|
||||
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
|
||||
parser.add_argument(
|
||||
"--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--close",
|
||||
action="store_true",
|
||||
help="Actually close duplicates (default is dry-run)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
dry_run = not args.close
|
||||
|
||||
if dry_run:
|
||||
print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
|
||||
|
||||
print("Fetching open issues...")
|
||||
issues = fetch_open_issues(args.repo)
|
||||
print(f"Found {len(issues)} open issues.\n")
|
||||
|
||||
if args.scan:
|
||||
count = scan_all(issues, args.threshold, args.repo, dry_run)
|
||||
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
|
||||
else:
|
||||
found = check_single(
|
||||
args.issue_number, issues, args.threshold, args.repo, dry_run
|
||||
)
|
||||
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
69
.github/workflows/auto-close-duplicates.yml
vendored
Normal file
69
.github/workflows/auto-close-duplicates.yml
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
name: Auto-close duplicate issues
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 9 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Log which issues would close without closing anything
|
||||
type: boolean
|
||||
default: true
|
||||
grace_period_days:
|
||||
description: Days a duplicate notice must go unanswered before the close
|
||||
type: number
|
||||
default: 3
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/auto-close-duplicates.yml
|
||||
- scripts/auto-close-duplicates.ts
|
||||
- scripts/auto-close-duplicates.test.ts
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Test the sweep
|
||||
run: bun test scripts/auto-close-duplicates.test.ts
|
||||
|
||||
sweep:
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
# Exact version, never latest: the next step holds an issues: write token
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Close unanswered duplicates, reopen ones the reporter answered
|
||||
run: bun run scripts/auto-close-duplicates.ts
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DRY_RUN: ${{ inputs.dry_run == true }}
|
||||
GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }}
|
||||
40
.github/workflows/check_duplicate_issues.yml
vendored
40
.github/workflows/check_duplicate_issues.yml
vendored
|
|
@ -1,12 +1,19 @@
|
|||
name: Check Duplicate Issues
|
||||
|
||||
# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later,
|
||||
# and only when its title is identical to an older open issue and nobody replied.
|
||||
# The HTML marker below is the handshake between the two, so keep it in the template.
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-duplicate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
|
@ -19,35 +26,12 @@ jobs:
|
|||
threshold: 0.6
|
||||
reaction: eyes
|
||||
comment: |
|
||||
**⚠️ Potential duplicate detected**
|
||||
<!-- litellm:potential-duplicate candidates={{#issues}}{{number}},{{/issues}} -->
|
||||
**Potential duplicate detected**
|
||||
|
||||
This issue appears similar to existing issue(s):
|
||||
This looks similar to:
|
||||
{{#issues}}
|
||||
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
|
||||
- #{{number}} - {{title}}
|
||||
{{/issues}}
|
||||
|
||||
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
|
||||
|
||||
- name: Checkout close script
|
||||
if: github.event.action == 'opened'
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
if: github.event.action == 'opened'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Auto-close if high-confidence duplicate
|
||||
if: github.event.action == 'opened'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python3 .github/scripts/close_duplicate_issues.py \
|
||||
--issue-number ${{ github.event.issue.number }} \
|
||||
--repo ${{ github.repository }} \
|
||||
--threshold 0.85 \
|
||||
--close
|
||||
If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ When adding new features, add meaningful tests. Don't add tests that don't check
|
|||
|
||||
Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
|
||||
|
||||
Never test structure of code only function of it
|
||||
|
||||
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
|
||||
|
|
|
|||
|
|
@ -1728,6 +1728,7 @@ SENTRY_DENYLIST: Final = [
|
|||
"jwt_token",
|
||||
"private_key",
|
||||
"SLACK_WEBHOOK_URL",
|
||||
"ALERTING_WEBHOOK_URL",
|
||||
"webhook_url",
|
||||
"LANGFUSE_SECRET_KEY",
|
||||
# Email Configuration
|
||||
|
|
|
|||
|
|
@ -1485,9 +1485,9 @@ Model Info:
|
|||
elif self.default_webhook_url is not None:
|
||||
_digest_webhook = self.default_webhook_url
|
||||
else:
|
||||
_digest_webhook = os.getenv("SLACK_WEBHOOK_URL", None)
|
||||
_digest_webhook = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
if _digest_webhook is None:
|
||||
raise ValueError("Missing SLACK_WEBHOOK_URL from environment")
|
||||
raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment")
|
||||
|
||||
digest_key: Final = f"{alert_type_name_str}:{request_model or ''}:{api_base or ''}"
|
||||
|
||||
|
|
@ -1516,10 +1516,10 @@ Model Info:
|
|||
elif self.default_webhook_url is not None:
|
||||
slack_webhook_url = self.default_webhook_url
|
||||
else:
|
||||
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None)
|
||||
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL")
|
||||
|
||||
if slack_webhook_url is None:
|
||||
raise ValueError("Missing SLACK_WEBHOOK_URL from environment")
|
||||
raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment")
|
||||
payload: Final = {"text": formatted_message}
|
||||
headers: Final = {"Content-type": "application/json"}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2541,7 +2541,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
)
|
||||
alerting: list | None = Field(
|
||||
None,
|
||||
description="List of alerting integrations. Today, just slack - `alerting: ['slack']`",
|
||||
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",
|
||||
)
|
||||
alert_types: list[AlertType] | None = Field(
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -489,7 +489,7 @@ lite codex exec "summarize the repo"
|
|||
|
||||
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
|
||||
|
||||
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol).
|
||||
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol).
|
||||
|
||||
Options (these belong to the wrapper, so put them before the agent's own flags):
|
||||
|
||||
|
|
@ -505,7 +505,7 @@ The credential is short-lived by design (default 24h, configurable via `LITELLM_
|
|||
|
||||
### Route Every Claude Code Session Through the Proxy
|
||||
|
||||
`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it.
|
||||
`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL, `env.ENABLE_TOOL_SEARCH` to `true` when that key is missing, and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it.
|
||||
|
||||
Two things need to already be true: you've run `lite login` (or `lite login --pkce`, whose key the helper renews on its own), since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you.
|
||||
|
||||
|
|
@ -529,7 +529,7 @@ Cursor is not supported: it has no equivalent file-based config to hot-patch thi
|
|||
lite --base-url https://your-proxy.example.com login --config-claude
|
||||
```
|
||||
|
||||
It writes the same two settings `lite up` does, `env.ANTHROPIC_BASE_URL` and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag.
|
||||
It writes the same settings `lite up` does, `env.ANTHROPIC_BASE_URL`, `env.ENABLE_TOOL_SEARCH`, and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag.
|
||||
|
||||
Because the credential is reached through `apiKeyHelper` rather than copied into the file, a later `lite login` refreshes it with no further action: Claude Code re-runs the helper on every request and picks up whatever token the most recent login stored. Nothing secret is written to `settings.json`.
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ from .auth import context_secret_vault, get_stored_api_key, login
|
|||
ANTHROPIC_BASE_URL_ENV: Final = "ANTHROPIC_BASE_URL"
|
||||
ANTHROPIC_AUTH_TOKEN_ENV: Final = "ANTHROPIC_AUTH_TOKEN"
|
||||
ANTHROPIC_API_KEY_ENV: Final = "ANTHROPIC_API_KEY"
|
||||
ENABLE_TOOL_SEARCH_ENV: Final = "ENABLE_TOOL_SEARCH"
|
||||
ENABLE_TOOL_SEARCH_VALUE: Final = "true"
|
||||
OPENAI_BASE_URL_ENV: Final = "OPENAI_BASE_URL"
|
||||
OPENAI_API_KEY_ENV: Final = "OPENAI_API_KEY"
|
||||
|
||||
|
|
@ -61,7 +63,10 @@ def build_agent_env(
|
|||
Anthropic clients (Claude Code) append /v1/messages to ANTHROPIC_BASE_URL,
|
||||
so it stays the bare proxy root; OpenAI clients (Codex, OpenCode) expect the
|
||||
/v1 suffix on OPENAI_BASE_URL. ANTHROPIC_API_KEY is dropped so a stray
|
||||
Anthropic key cannot win over the bearer token we set.
|
||||
Anthropic key cannot win over the bearer token we set. ENABLE_TOOL_SEARCH
|
||||
defaults to true because Claude Code turns tool search off when
|
||||
ANTHROPIC_BASE_URL is not a first-party Anthropic host; a value already in
|
||||
the environment is left alone.
|
||||
"""
|
||||
env: Final = dict(base_env)
|
||||
root: Final = base_url.rstrip("/")
|
||||
|
|
@ -69,6 +74,8 @@ def build_agent_env(
|
|||
env[ANTHROPIC_BASE_URL_ENV] = root
|
||||
env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key
|
||||
env.pop(ANTHROPIC_API_KEY_ENV, None)
|
||||
if ENABLE_TOOL_SEARCH_ENV not in env:
|
||||
env[ENABLE_TOOL_SEARCH_ENV] = ENABLE_TOOL_SEARCH_VALUE
|
||||
if PROFILE_OPENAI in profiles:
|
||||
env[OPENAI_BASE_URL_ENV] = root + "/v1"
|
||||
env[OPENAI_API_KEY_ENV] = api_key
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ API_KEY_HELPER_KEY: Final = "apiKeyHelper"
|
|||
ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY"
|
||||
ANTHROPIC_AUTH_TOKEN_KEY: Final = "ANTHROPIC_AUTH_TOKEN"
|
||||
ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL"
|
||||
ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH"
|
||||
ENABLE_TOOL_SEARCH_VALUE: Final = "true"
|
||||
# Force every one of Claude Code's own model tiers to request the auto-router by name.
|
||||
# Router's auto-router registry is keyed by the literal requested model string
|
||||
# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*"
|
||||
|
|
@ -34,6 +36,7 @@ def merge_claude_settings_static_token(
|
|||
raw_env: Final = settings.get(ENV_KEY, {})
|
||||
base_env: Final = raw_env if isinstance(raw_env, dict) else {}
|
||||
env: Final[dict[str, JsonValue]] = {
|
||||
ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE,
|
||||
**base_env,
|
||||
ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"),
|
||||
ANTHROPIC_AUTH_TOKEN_KEY: auth_token,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ ENV_KEY: Final = "env"
|
|||
API_KEY_HELPER_KEY: Final = "apiKeyHelper"
|
||||
ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL"
|
||||
ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY"
|
||||
ENABLE_TOOL_SEARCH_KEY: Final = "ENABLE_TOOL_SEARCH"
|
||||
ENABLE_TOOL_SEARCH_VALUE: Final = "true"
|
||||
|
||||
CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json"
|
||||
BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json"
|
||||
|
|
@ -70,12 +72,15 @@ def merge_claude_settings(
|
|||
|
||||
Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a
|
||||
stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued
|
||||
token (same reasoning as build_agent_env in agents.py). Every other key is
|
||||
preserved untouched.
|
||||
token (same reasoning as build_agent_env in agents.py). ENABLE_TOOL_SEARCH
|
||||
defaults to true because Claude Code turns tool search off when
|
||||
ANTHROPIC_BASE_URL is not a first-party Anthropic host; an existing value is
|
||||
left alone. Every other key is preserved untouched.
|
||||
"""
|
||||
raw_env: Final = settings.get(ENV_KEY, {})
|
||||
base_env: Final = raw_env if isinstance(raw_env, dict) else {}
|
||||
env: Final = {
|
||||
ENABLE_TOOL_SEARCH_KEY: ENABLE_TOOL_SEARCH_VALUE,
|
||||
**{key: value for key, value in base_env.items() if key != ANTHROPIC_API_KEY_KEY},
|
||||
ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"),
|
||||
}
|
||||
|
|
@ -144,6 +149,8 @@ __all__ = (
|
|||
"AUTOROUTE_BACKUP_PATH",
|
||||
"BACKUP_PATH",
|
||||
"CLAUDE_SETTINGS_PATH",
|
||||
"ENABLE_TOOL_SEARCH_KEY",
|
||||
"ENABLE_TOOL_SEARCH_VALUE",
|
||||
"ENV_KEY",
|
||||
"SETTINGS_FILE_OWNERS",
|
||||
"ClaudeSettingsError",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
streaming_transform_mode=getattr(litellm_params, "streaming_transform_mode", None),
|
||||
file_sanitization_fail_open=getattr(litellm_params, "file_sanitization_fail_open", None),
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback(_prompt_security_callback)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ import os
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.exceptions import Timeout as LiteLLMTimeout
|
||||
from litellm.integrations.custom_guardrail import (
|
||||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
|
|
@ -24,6 +26,9 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
|
||||
_SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS: Final = 30.0
|
||||
|
||||
|
||||
class PromptSecurityGuardrailMissingSecrets(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -63,6 +68,13 @@ class _SanitizeStatusResponse(TypedDict, total=False):
|
|||
metadata: ReadOnly[_SanitizeMetadata]
|
||||
|
||||
|
||||
class _SanitizeResult(TypedDict):
|
||||
action: ReadOnly[str]
|
||||
content: ReadOnly[str | None]
|
||||
metadata: ReadOnly[_SanitizeMetadata]
|
||||
violations: ReadOnly[Sequence[str]]
|
||||
|
||||
|
||||
class PromptSecurityGuardrail(CustomGuardrail):
|
||||
@classmethod
|
||||
def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
|
||||
|
|
@ -80,6 +92,8 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
system_prompt: str | None = None,
|
||||
check_tool_results: bool | None = None,
|
||||
streaming_transform_mode: Literal["block_only", "incremental_diff"] | None = None,
|
||||
file_sanitization_timeout: float = _SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS,
|
||||
file_sanitization_fail_open: bool | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
|
||||
|
|
@ -113,6 +127,8 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
# Configuration for file sanitization
|
||||
self.max_poll_attempts = 30 # Maximum number of polling attempts
|
||||
self.poll_interval = 2 # Seconds between polling attempts
|
||||
self.file_sanitization_timeout = file_sanitization_timeout
|
||||
self.file_sanitization_fail_open = file_sanitization_fail_open is not False
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
|
@ -402,6 +418,39 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
Sanitize file content using Prompt Security API.
|
||||
Returns: dict with keys 'action', 'content', 'metadata'
|
||||
"""
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self._sanitize_file_content(file_data, filename, user_api_key_alias),
|
||||
timeout=self.file_sanitization_timeout,
|
||||
)
|
||||
except (asyncio.TimeoutError, httpx.TimeoutException, LiteLLMTimeout) as exc:
|
||||
if not self.file_sanitization_fail_open:
|
||||
verbose_proxy_logger.error(
|
||||
"Prompt Security Guardrail: file sanitization for %s timed out with %s; failing closed",
|
||||
filename,
|
||||
type(exc).__name__,
|
||||
)
|
||||
raise HTTPException(status_code=408, detail="File sanitization timeout") from exc
|
||||
|
||||
verbose_proxy_logger.error(
|
||||
"Prompt Security Guardrail: file sanitization for %s timed out with %s; failing open",
|
||||
filename,
|
||||
type(exc).__name__,
|
||||
)
|
||||
fail_open_result: Final[_SanitizeResult] = {
|
||||
"action": "allow",
|
||||
"content": None,
|
||||
"metadata": {},
|
||||
"violations": (),
|
||||
}
|
||||
return fail_open_result
|
||||
|
||||
async def _sanitize_file_content(
|
||||
self,
|
||||
file_data: bytes,
|
||||
filename: str,
|
||||
user_api_key_alias: str | None,
|
||||
) -> _SanitizeResult:
|
||||
headers: Final = {"APP-ID": self.api_key}
|
||||
if user_api_key_alias:
|
||||
headers["X-LiteLLM-Key-Alias"] = user_api_key_alias
|
||||
|
|
|
|||
|
|
@ -1027,6 +1027,14 @@ async def user_info_v2(
|
|||
This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem
|
||||
where the old endpoint loaded all keys and teams into memory.
|
||||
|
||||
Note on `spend`: this is the user's running budget counter, which the budget reset job
|
||||
resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default,
|
||||
or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT
|
||||
lifetime or per-period historical spend. For historical spend over a date range, use
|
||||
`/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend
|
||||
records that only ever accumulate and are never reset. The two values are expected to
|
||||
diverge once a budget reset has occurred within the queried period.
|
||||
|
||||
Access control:
|
||||
- Proxy admins can query any user
|
||||
- Team admins can query users within their teams
|
||||
|
|
@ -2726,6 +2734,11 @@ async def get_user_daily_activity(
|
|||
|
||||
Meant to optimize querying spend data for analytics for a user.
|
||||
|
||||
Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
resets. Their total can legitimately exceed the `spend` field returned by
|
||||
`/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
|
||||
Returns:
|
||||
(by date)
|
||||
- spend
|
||||
|
|
@ -2839,6 +2852,11 @@ async def get_user_daily_activity_aggregated(
|
|||
"""
|
||||
Aggregated analytics for a user's daily activity without pagination.
|
||||
Returns the same response shape as the paginated endpoint with page metadata set to single-page.
|
||||
|
||||
Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
resets. Their total can legitimately exceed the `spend` field returned by
|
||||
`/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ async def add_team_callbacks(
|
|||
Use this if if you want different teams to have different success/failure callbacks
|
||||
|
||||
Parameters:
|
||||
- callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
- callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
- callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
- "success": Callback for successful LLM calls
|
||||
- "failure": Callback for failed LLM calls
|
||||
|
|
@ -268,6 +268,8 @@ async def add_team_callbacks(
|
|||
- langsmith_api_key: The API key for the Langsmith callback
|
||||
- langsmith_project: The project for the Langsmith callback
|
||||
- langsmith_base_url: The base URL for the Langsmith callback
|
||||
- newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
- newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
|
||||
Example curl:
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1363,7 +1363,7 @@ _OPENAPI_HTTP_METHODS: Final = {
|
|||
# the UI. Kept here at module scope to match the analogous descriptor
|
||||
# `is_secret` flags in litellm.proxy.config_resolvers and the
|
||||
# `_CACHE_SENSITIVE_FIELDS` constant in the cache endpoint file.
|
||||
_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
|
||||
_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"ALERTING_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
|
||||
|
||||
|
||||
def _strip_operation_id_method_suffix(operation_id: str) -> str:
|
||||
|
|
@ -16566,6 +16566,7 @@ async def create_config_audit_log(
|
|||
|
||||
_EXTRA_SECRET_CALLBACK_ENV_VARS: Final = frozenset(
|
||||
{
|
||||
"ALERTING_WEBHOOK_URL",
|
||||
"GALILEO_USERNAME",
|
||||
"GENERIC_LOGGER_HEADERS",
|
||||
"OTEL_HEADERS",
|
||||
|
|
|
|||
|
|
@ -645,7 +645,7 @@ class ProxyLogging:
|
|||
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(self.internal_usage_cache)
|
||||
self.max_budget_limiter = _PROXY_MaxBudgetLimiter()
|
||||
self.cache_control_check = _PROXY_CacheControlCheck()
|
||||
self.alerting: list | None = None
|
||||
self.alerting: list[str] | None = None
|
||||
self.alerting_threshold: float = 300 # default to 5 min. threshold
|
||||
self.alert_types: list[AlertType] = DEFAULT_ALERT_TYPES
|
||||
self.alert_to_webhook_url: dict | None = None
|
||||
|
|
@ -2364,7 +2364,9 @@ class ProxyLogging:
|
|||
# do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails)
|
||||
return
|
||||
|
||||
if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
|
||||
if self.alerting is not None and (
|
||||
"slack" in self.alerting or "ms_teams" in self.alerting or "webhook" in self.alerting
|
||||
):
|
||||
if self.slack_alerting_instance is not None:
|
||||
await self.slack_alerting_instance.budget_alerts(
|
||||
type=type,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ class PromptSecurityGuardrailConfigModel(GuardrailConfigModel):
|
|||
default=None,
|
||||
description="The API base for the Prompt Security guardrail. If not provided, the `PROMPT_SECURITY_API_BASE` environment variable is used.",
|
||||
)
|
||||
file_sanitization_fail_open: bool = Field(
|
||||
default=True,
|
||||
description="Whether file sanitization timeouts allow the original file through instead of blocking the request.",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
348
scripts/auto-close-duplicates.test.ts
Normal file
348
scripts/auto-close-duplicates.test.ts
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import {
|
||||
CLOSED_MARKER,
|
||||
REOPEN_COMMENT,
|
||||
candidateNumbers,
|
||||
duplicateTarget,
|
||||
normalizeTitle,
|
||||
pendingNotice,
|
||||
readConfig,
|
||||
reopenTarget,
|
||||
sweepClosedIssue,
|
||||
sweepIssue,
|
||||
type Comment,
|
||||
type GitHubApi,
|
||||
type Issue,
|
||||
type Reaction,
|
||||
type SweepConfig,
|
||||
} from "./auto-close-duplicates";
|
||||
|
||||
const NOW = new Date("2026-09-04T09:00:00Z");
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const daysAgo = (days: number): string => new Date(NOW.getTime() - days * DAY_MS).toISOString();
|
||||
|
||||
const issue = (number: number, title: string, overrides: Partial<Issue> = {}): Issue => ({
|
||||
number,
|
||||
title,
|
||||
state: "open",
|
||||
user: { login: "reporter" },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const notice = (candidates: readonly number[], createdAt: string, overrides: Partial<Comment> = {}): Comment => ({
|
||||
id: 900,
|
||||
body: `<!-- litellm:potential-duplicate candidates=${candidates.join(",")}, -->\n**Potential duplicate detected**`,
|
||||
created_at: createdAt,
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const humanComment = (createdAt: string, body = "It is not the same thing", login = "reporter"): Comment => ({
|
||||
id: 901,
|
||||
body,
|
||||
created_at: createdAt,
|
||||
user: { type: "User", login },
|
||||
});
|
||||
|
||||
const config: SweepConfig = { repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW };
|
||||
|
||||
describe("normalizeTitle", () => {
|
||||
test("drops the template prefix, case, and punctuation", () => {
|
||||
expect(normalizeTitle("[Bug]: Gemma 4-e4b fails on Vertex!")).toBe("gemma 4 e4b fails on vertex");
|
||||
expect(normalizeTitle("[Feature]: ")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("candidateNumbers", () => {
|
||||
test("reads only the marker field, keeps older issues, sorted ascending and deduplicated", () => {
|
||||
const body = "<!-- litellm:potential-duplicate candidates=40,10,30,10, -->\n- #1 - see #1 (100% similar)";
|
||||
expect(candidateNumbers(body, 35)).toEqual([10, 30]);
|
||||
});
|
||||
|
||||
test("returns nothing without the marker", () => {
|
||||
expect(candidateNumbers("- #1 - looks like #1", 35)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pendingNotice", () => {
|
||||
test("waits out the grace period from the latest notice", () => {
|
||||
const fresh = pendingNotice(issue(35, "t"), [notice([10], daysAgo(2.9))], config);
|
||||
expect(fresh.kind).toBe("skip");
|
||||
const aged = pendingNotice(issue(35, "t"), [notice([10], daysAgo(3.1))], config);
|
||||
expect(aged.kind).toBe("pending");
|
||||
const reposted = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(6)), notice([10], daysAgo(1), { id: 902 })],
|
||||
config,
|
||||
);
|
||||
expect(reposted.kind).toBe("skip");
|
||||
});
|
||||
|
||||
test("an objection posted before a re-posted notice still keeps the issue open", () => {
|
||||
const verdict = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(10)), humanComment(daysAgo(7)), notice([10], daysAgo(4), { id: 902 })],
|
||||
config,
|
||||
);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone replied after the notice" });
|
||||
});
|
||||
|
||||
test("a zero-day grace period acts on the notice at once", () => {
|
||||
const verdict = pendingNotice(issue(35, "t"), [notice([10], daysAgo(0.01))], { ...config, graceDays: 0 });
|
||||
expect(verdict.kind).toBe("pending");
|
||||
});
|
||||
|
||||
test("a human reply after the notice keeps the issue open, a bot reply does not", () => {
|
||||
const human = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5)), humanComment(daysAgo(4))], config);
|
||||
expect(human).toEqual({ kind: "skip", reason: "someone replied after the notice" });
|
||||
const bot = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(5)), { id: 903, body: "triage", created_at: daysAgo(4), user: { type: "Bot", login: "triage[bot]" } }],
|
||||
config,
|
||||
);
|
||||
expect(bot.kind).toBe("pending");
|
||||
});
|
||||
|
||||
test("a human quoting the marker is not a notice", () => {
|
||||
const quoted = pendingNotice(issue(35, "t"), [notice([10], daysAgo(5), { user: { type: "User", login: "reporter" } })], config);
|
||||
expect(quoted).toEqual({ kind: "skip", reason: "carries no duplicate notice" });
|
||||
});
|
||||
|
||||
test("never closes an issue twice: a reopened issue is left alone", () => {
|
||||
const reopened = pendingNotice(
|
||||
issue(35, "t"),
|
||||
[notice([10], daysAgo(9)), { id: 904, body: `Closed automatically\n\n${CLOSED_MARKER}`, created_at: daysAgo(5), user: { type: "Bot", login: "github-actions[bot]" } }],
|
||||
config,
|
||||
);
|
||||
expect(reopened).toEqual({ kind: "skip", reason: "was reopened after an automatic close" });
|
||||
});
|
||||
|
||||
test("skips pull requests and issues whose only candidates are newer", () => {
|
||||
expect(pendingNotice(issue(35, "t", { pull_request: {} }), [notice([10], daysAgo(5))], config).kind).toBe("skip");
|
||||
expect(pendingNotice(issue(35, "t"), [notice([40], daysAgo(5))], config)).toEqual({
|
||||
kind: "skip",
|
||||
reason: "no candidate is older than this issue",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("duplicateTarget", () => {
|
||||
const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
|
||||
test("closes only against the earliest open issue with the identical normalized title", () => {
|
||||
const verdict = duplicateTarget(
|
||||
reporter,
|
||||
[issue(10, "[Bug]: Gemma 4-e4n fails on Vertex"), issue(20, "[bug]: gemma 4-e4b fails on vertex"), issue(30, "[Bug]: Gemma 4-e4b fails on Vertex")],
|
||||
[],
|
||||
);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 20 });
|
||||
});
|
||||
|
||||
test("a near miss in the title is not a duplicate", () => {
|
||||
const verdict = duplicateTarget(reporter, [issue(10, "[Bug]: Gemma 4-e4n fails on Vertex")], []);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "no older open issue has the identical title" });
|
||||
});
|
||||
|
||||
test("bare template titles never match each other", () => {
|
||||
const verdict = duplicateTarget(issue(35, "[Bug]: "), [issue(10, "[Bug]: ")], []);
|
||||
expect(verdict.kind).toBe("skip");
|
||||
expect(verdict.kind === "skip" && verdict.reason).toContain("too short");
|
||||
});
|
||||
|
||||
test("a closed candidate or a pull request is never the target", () => {
|
||||
expect(duplicateTarget(reporter, [issue(10, reporter.title, { state: "closed" })], []).kind).toBe("skip");
|
||||
expect(duplicateTarget(reporter, [issue(10, reporter.title, { pull_request: {} })], []).kind).toBe("skip");
|
||||
});
|
||||
|
||||
test("a thumbs down on the notice keeps the issue open", () => {
|
||||
const verdict = duplicateTarget(reporter, [issue(10, reporter.title)], [{ content: "+1" }, { content: "-1" }]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepIssue", () => {
|
||||
const reporter = issue(35, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
const original = issue(10, "[Bug]: Gemma 4-e4b fails on Vertex");
|
||||
|
||||
function fakeApi(
|
||||
comments: readonly Comment[] = [notice([10], daysAgo(5))],
|
||||
reactionsByNotice: Readonly<Record<number, readonly Reaction[]>> = {},
|
||||
): { readonly api: GitHubApi; readonly writes: readonly string[] } {
|
||||
const writes: string[] = [];
|
||||
const api: GitHubApi = {
|
||||
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
|
||||
if (method !== "GET") {
|
||||
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
|
||||
return {} as T;
|
||||
}
|
||||
if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) {
|
||||
return comments as T;
|
||||
}
|
||||
const reactionsPath = path.match(/^\/repos\/BerriAI\/litellm\/issues\/comments\/(\d+)\/reactions/);
|
||||
if (reactionsPath) {
|
||||
return (reactionsByNotice[Number(reactionsPath[1])] ?? []) as T;
|
||||
}
|
||||
if (path === "/repos/BerriAI/litellm/issues/10") {
|
||||
return original as T;
|
||||
}
|
||||
throw new Error(`unexpected GET ${path}`);
|
||||
},
|
||||
};
|
||||
return { api, writes };
|
||||
}
|
||||
|
||||
test("a dry run reports the close and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await sweepIssue(api, { ...config, dryRun: true }, reporter);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 10 });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a thumbs down on an earlier notice still keeps the issue open", async () => {
|
||||
const { api, writes } = fakeApi([notice([10], daysAgo(9)), notice([10], daysAgo(5), { id: 902 })], { 900: [{ content: "-1" }] });
|
||||
const verdict = await sweepIssue(api, config, reporter);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "someone gave the notice a thumbs down" });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a real run comments, labels, then closes with the duplicate reason", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await sweepIssue(api, config, reporter);
|
||||
expect(verdict).toEqual({ kind: "close", duplicateOf: 10 });
|
||||
expect(writes.map((write) => write.split(" ").slice(0, 2).join(" "))).toEqual([
|
||||
"POST /repos/BerriAI/litellm/issues/35/comments",
|
||||
"POST /repos/BerriAI/litellm/issues/35/labels",
|
||||
"PATCH /repos/BerriAI/litellm/issues/35",
|
||||
]);
|
||||
expect(writes[0]).toContain("duplicate of #10");
|
||||
expect(writes[0]).toContain("unanswered for 3 days");
|
||||
expect(writes[0]).toContain(CLOSED_MARKER);
|
||||
expect(writes[1]).toContain('{"labels":["duplicate"]}');
|
||||
expect(writes[2]).toContain('{"state":"closed","state_reason":"duplicate"}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("reopenTarget", () => {
|
||||
const closedByBot = (overrides: Partial<Issue> = {}): Issue =>
|
||||
issue(35, "t", { state: "closed", closed_by: { type: "Bot" }, ...overrides });
|
||||
const closeMarker = (createdAt: string): Comment => ({
|
||||
id: 905,
|
||||
body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`,
|
||||
created_at: createdAt,
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
});
|
||||
|
||||
test("a reporter reply after the automatic close reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
});
|
||||
|
||||
test("an issue closed by a person stays closed", () => {
|
||||
const verdict = reopenTarget(closedByBot({ closed_by: { type: "User" } }), [
|
||||
closeMarker(daysAgo(2)),
|
||||
humanComment(daysAgo(1)),
|
||||
]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "was closed by a person" });
|
||||
});
|
||||
|
||||
test("without the automatic-close marker nothing reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "carries no automatic-close marker" });
|
||||
});
|
||||
|
||||
test("a maintainer reply alone does not reopen", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [
|
||||
closeMarker(daysAgo(2)),
|
||||
humanComment(daysAgo(1), "Confirmed duplicate", "maintainer"),
|
||||
]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" });
|
||||
});
|
||||
|
||||
test("a reporter comment from before the close does not reopen", () => {
|
||||
const verdict = reopenTarget(closedByBot(), [humanComment(daysAgo(3)), closeMarker(daysAgo(2))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "the reporter has not replied since the close" });
|
||||
});
|
||||
|
||||
test("a pull request never reopens", () => {
|
||||
const verdict = reopenTarget(closedByBot({ pull_request: {} }), [closeMarker(daysAgo(2)), humanComment(daysAgo(1))]);
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "is a pull request" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepClosedIssue", () => {
|
||||
function fakeApi(issueBody: Issue, comments: readonly Comment[]): { readonly api: GitHubApi; readonly writes: readonly string[] } {
|
||||
const writes: string[] = [];
|
||||
const api: GitHubApi = {
|
||||
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
|
||||
if (method !== "GET") {
|
||||
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
|
||||
return {} as T;
|
||||
}
|
||||
if (path.startsWith("/repos/BerriAI/litellm/issues/35/comments")) {
|
||||
return comments as T;
|
||||
}
|
||||
if (path === "/repos/BerriAI/litellm/issues/35") {
|
||||
return issueBody as T;
|
||||
}
|
||||
throw new Error(`unexpected GET ${path}`);
|
||||
},
|
||||
};
|
||||
return { api, writes };
|
||||
}
|
||||
|
||||
const closedByBot = issue(35, "t", { state: "closed", closed_by: { type: "Bot" } });
|
||||
const closeMarker: Comment = {
|
||||
id: 905,
|
||||
body: `Closed automatically as a duplicate of #10.\n\n${CLOSED_MARKER}`,
|
||||
created_at: daysAgo(2),
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
};
|
||||
|
||||
test("a real run unlabels, reopens, then explains", async () => {
|
||||
const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]);
|
||||
const verdict = await sweepClosedIssue(api, config, 35);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
expect(writes).toEqual([
|
||||
"DELETE /repos/BerriAI/litellm/issues/35/labels/duplicate undefined",
|
||||
'PATCH /repos/BerriAI/litellm/issues/35 {"state":"open"}',
|
||||
`POST /repos/BerriAI/litellm/issues/35/comments {"body":"${REOPEN_COMMENT}"}`,
|
||||
]);
|
||||
});
|
||||
|
||||
test("a dry run reports the reopen and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi(closedByBot, [closeMarker, humanComment(daysAgo(1))]);
|
||||
const verdict = await sweepClosedIssue(api, { ...config, dryRun: true }, 35);
|
||||
expect(verdict).toEqual({ kind: "reopen" });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("readConfig", () => {
|
||||
test("defaults to a real run with a 3-day grace period", () => {
|
||||
const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm" }, NOW);
|
||||
expect(parsed).toEqual({ token: "t", repo: "BerriAI/litellm", graceDays: 3, dryRun: false, now: NOW });
|
||||
});
|
||||
|
||||
test("honors DRY_RUN and GRACE_PERIOD_DAYS overrides", () => {
|
||||
const parsed = readConfig(
|
||||
{ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", DRY_RUN: "true", GRACE_PERIOD_DAYS: "0" },
|
||||
NOW,
|
||||
);
|
||||
expect(parsed.dryRun).toBe(true);
|
||||
expect(parsed.graceDays).toBe(0);
|
||||
});
|
||||
|
||||
test("an empty GRACE_PERIOD_DAYS, as a schedule run renders it, means the default", () => {
|
||||
const parsed = readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "" }, NOW);
|
||||
expect(parsed.graceDays).toBe(3);
|
||||
});
|
||||
|
||||
test("refuses a missing token, a malformed repository, or a bad grace period", () => {
|
||||
expect(() => readConfig({ GITHUB_REPOSITORY: "o/r" }, NOW)).toThrow("GITHUB_TOKEN");
|
||||
expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "litellm" }, NOW)).toThrow("owner/repo");
|
||||
expect(() => readConfig({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GRACE_PERIOD_DAYS: "-1" }, NOW)).toThrow(
|
||||
"GRACE_PERIOD_DAYS",
|
||||
);
|
||||
});
|
||||
});
|
||||
300
scripts/auto-close-duplicates.ts
Normal file
300
scripts/auto-close-duplicates.ts
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
declare const process: { readonly env: Readonly<Record<string, string | undefined>> };
|
||||
|
||||
export interface Issue {
|
||||
readonly number: number;
|
||||
readonly title: string;
|
||||
readonly state: string;
|
||||
readonly user: { readonly login: string };
|
||||
readonly closed_by?: { readonly type: string } | null;
|
||||
readonly pull_request?: unknown;
|
||||
}
|
||||
|
||||
export interface Comment {
|
||||
readonly id: number;
|
||||
readonly body: string;
|
||||
readonly created_at: string;
|
||||
readonly user: { readonly type: string; readonly login: string };
|
||||
}
|
||||
|
||||
export interface Reaction {
|
||||
readonly content: string;
|
||||
}
|
||||
|
||||
export interface GitHubApi {
|
||||
readonly request: <T>(method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object) => Promise<T>;
|
||||
}
|
||||
|
||||
export interface SweepConfig {
|
||||
readonly repo: string;
|
||||
readonly graceDays: number;
|
||||
readonly dryRun: boolean;
|
||||
readonly now: Date;
|
||||
}
|
||||
|
||||
export type NoticeVerdict =
|
||||
| { readonly kind: "pending"; readonly notices: readonly Comment[]; readonly candidates: readonly number[] }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export type CloseVerdict =
|
||||
| { readonly kind: "close"; readonly duplicateOf: number }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export type ReopenVerdict =
|
||||
| { readonly kind: "reopen" }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export const FLAG_LABEL = "potential-duplicate";
|
||||
export const CLOSED_MARKER = "<!-- litellm:closed-as-duplicate -->";
|
||||
export const DEFAULT_GRACE_DAYS = 3;
|
||||
export const REOPEN_COMMENT =
|
||||
"Reopened automatically: the reporter replied after the duplicate close, so this needs a human look.";
|
||||
const NOTICE_MARKER = /<!-- litellm:potential-duplicate candidates=([\d,]*) -->/;
|
||||
const MIN_TITLE_WORDS = 3;
|
||||
const PAGE_SIZE = 100;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const REOPEN_LOOKBACK_DAYS = 30;
|
||||
|
||||
const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason });
|
||||
|
||||
export function normalizeTitle(title: string): string {
|
||||
return title
|
||||
.toLowerCase()
|
||||
.replace(/^\s*\[[^\]]*\]\s*:?/, "")
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function candidateNumbers(noticeBody: string, issueNumber: number): readonly number[] {
|
||||
const field = noticeBody.match(NOTICE_MARKER);
|
||||
if (!field) {
|
||||
return [];
|
||||
}
|
||||
const older = field[1]
|
||||
.split(",")
|
||||
.filter((value) => value !== "")
|
||||
.map(Number)
|
||||
.filter((candidate) => candidate < issueNumber);
|
||||
return [...new Set(older)].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
export function pendingNotice(
|
||||
issue: Issue,
|
||||
comments: readonly Comment[],
|
||||
config: Pick<SweepConfig, "graceDays" | "now">,
|
||||
): NoticeVerdict {
|
||||
if (issue.pull_request !== undefined) {
|
||||
return skip("is a pull request");
|
||||
}
|
||||
if (comments.some((comment) => comment.body.includes(CLOSED_MARKER))) {
|
||||
return skip("was reopened after an automatic close");
|
||||
}
|
||||
const notices = comments.filter((comment) => comment.user.type === "Bot" && NOTICE_MARKER.test(comment.body));
|
||||
const first = notices[0];
|
||||
const latest = notices[notices.length - 1];
|
||||
if (first === undefined || latest === undefined) {
|
||||
return skip("carries no duplicate notice");
|
||||
}
|
||||
const ageDays = (config.now.getTime() - new Date(latest.created_at).getTime()) / DAY_MS;
|
||||
if (ageDays < config.graceDays) {
|
||||
return skip(`notice is ${ageDays.toFixed(1)} days old, grace period is ${config.graceDays}`);
|
||||
}
|
||||
const firstNoticeAt = new Date(first.created_at);
|
||||
if (comments.some((comment) => comment.user.type !== "Bot" && new Date(comment.created_at) > firstNoticeAt)) {
|
||||
return skip("someone replied after the notice");
|
||||
}
|
||||
const candidates = candidateNumbers(latest.body, issue.number);
|
||||
if (candidates.length === 0) {
|
||||
return skip("no candidate is older than this issue");
|
||||
}
|
||||
return { kind: "pending", notices, candidates };
|
||||
}
|
||||
|
||||
export function duplicateTarget(
|
||||
issue: Issue,
|
||||
candidates: readonly Issue[],
|
||||
reactions: readonly Reaction[],
|
||||
): CloseVerdict {
|
||||
if (reactions.some((reaction) => reaction.content === "-1")) {
|
||||
return skip("someone gave the notice a thumbs down");
|
||||
}
|
||||
const title = normalizeTitle(issue.title);
|
||||
if (title.split(" ").length < MIN_TITLE_WORDS) {
|
||||
return skip(`title "${issue.title}" is too short to match on`);
|
||||
}
|
||||
const original = candidates.find(
|
||||
(candidate) =>
|
||||
candidate.state === "open" && candidate.pull_request === undefined && normalizeTitle(candidate.title) === title,
|
||||
);
|
||||
if (original === undefined) {
|
||||
return skip("no older open issue has the identical title");
|
||||
}
|
||||
return { kind: "close", duplicateOf: original.number };
|
||||
}
|
||||
|
||||
export function reopenTarget(issue: Issue, comments: readonly Comment[]): ReopenVerdict {
|
||||
if (issue.pull_request !== undefined) {
|
||||
return skip("is a pull request");
|
||||
}
|
||||
if (issue.closed_by?.type !== "Bot") {
|
||||
return skip("was closed by a person");
|
||||
}
|
||||
const marker = comments.find((comment) => comment.body.includes(CLOSED_MARKER));
|
||||
if (marker === undefined) {
|
||||
return skip("carries no automatic-close marker");
|
||||
}
|
||||
const markerAt = new Date(marker.created_at);
|
||||
if (!comments.some((comment) => comment.user.login === issue.user.login && new Date(comment.created_at) > markerAt)) {
|
||||
return skip("the reporter has not replied since the close");
|
||||
}
|
||||
return { kind: "reopen" };
|
||||
}
|
||||
|
||||
export function closingComment(duplicateOf: number, graceDays: number): string {
|
||||
return `Closed automatically as a duplicate of #${duplicateOf}. Its title is identical to that older open issue and the duplicate notice above went unanswered for ${graceDays} days. If this is wrong, comment here with how it differs from #${duplicateOf} and this issue will be reopened automatically within a day.
|
||||
|
||||
${CLOSED_MARKER}`;
|
||||
}
|
||||
|
||||
async function listAll<T>(api: GitHubApi, path: string, page = 1): Promise<readonly T[]> {
|
||||
const separator = path.includes("?") ? "&" : "?";
|
||||
const batch = await api.request<readonly T[]>("GET", `${path}${separator}per_page=${PAGE_SIZE}&page=${page}`);
|
||||
return batch.length < PAGE_SIZE ? batch : [...batch, ...(await listAll<T>(api, path, page + 1))];
|
||||
}
|
||||
|
||||
async function closeAsDuplicate(
|
||||
api: GitHubApi,
|
||||
config: SweepConfig,
|
||||
issueNumber: number,
|
||||
duplicateOf: number,
|
||||
): Promise<void> {
|
||||
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
|
||||
await api.request("POST", `${issuePath}/comments`, { body: closingComment(duplicateOf, config.graceDays) });
|
||||
await api.request("POST", `${issuePath}/labels`, { labels: ["duplicate"] });
|
||||
await api.request("PATCH", issuePath, { state: "closed", state_reason: "duplicate" });
|
||||
}
|
||||
|
||||
async function reopenForReporter(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise<void> {
|
||||
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
|
||||
await api.request("DELETE", `${issuePath}/labels/duplicate`);
|
||||
await api.request("PATCH", issuePath, { state: "open" });
|
||||
await api.request("POST", `${issuePath}/comments`, { body: REOPEN_COMMENT });
|
||||
}
|
||||
|
||||
export async function sweepClosedIssue(api: GitHubApi, config: SweepConfig, issueNumber: number): Promise<ReopenVerdict> {
|
||||
const issue = await api.request<Issue>("GET", `/repos/${config.repo}/issues/${issueNumber}`);
|
||||
const comments = await listAll<Comment>(api, `/repos/${config.repo}/issues/${issueNumber}/comments`);
|
||||
const verdict = reopenTarget(issue, comments);
|
||||
if (verdict.kind === "reopen" && !config.dryRun) {
|
||||
await reopenForReporter(api, config, issueNumber);
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
export async function sweepIssue(api: GitHubApi, config: SweepConfig, issue: Issue): Promise<CloseVerdict> {
|
||||
const comments = await listAll<Comment>(api, `/repos/${config.repo}/issues/${issue.number}/comments`);
|
||||
const pending = pendingNotice(issue, comments, config);
|
||||
if (pending.kind === "skip") {
|
||||
return pending;
|
||||
}
|
||||
const reactions = (
|
||||
await Promise.all(
|
||||
pending.notices.map((notice) => listAll<Reaction>(api, `/repos/${config.repo}/issues/comments/${notice.id}/reactions`)),
|
||||
)
|
||||
).flat();
|
||||
const candidates = await Promise.all(
|
||||
pending.candidates.map((candidate) => api.request<Issue>("GET", `/repos/${config.repo}/issues/${candidate}`)),
|
||||
);
|
||||
const verdict = duplicateTarget(issue, candidates, reactions);
|
||||
if (verdict.kind === "close" && !config.dryRun) {
|
||||
await closeAsDuplicate(api, config, issue.number, verdict.duplicateOf);
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function describe(issue: Issue, verdict: CloseVerdict, dryRun: boolean): string {
|
||||
if (verdict.kind === "skip") {
|
||||
return `#${issue.number}: skipped, ${verdict.reason}`;
|
||||
}
|
||||
return `#${issue.number}: ${dryRun ? "would close" : "closed"} as a duplicate of #${verdict.duplicateOf}`;
|
||||
}
|
||||
|
||||
export async function sweep(api: GitHubApi, config: SweepConfig): Promise<readonly CloseVerdict[]> {
|
||||
const issues = await listAll<Issue>(api, `/repos/${config.repo}/issues?state=open&labels=${FLAG_LABEL}`);
|
||||
console.log(`${issues.length} open issues carry the ${FLAG_LABEL} label in ${config.repo}${config.dryRun ? " (dry run)" : ""}`);
|
||||
return issues.reduce<Promise<readonly CloseVerdict[]>>(async (previous, issue) => {
|
||||
const verdicts = await previous;
|
||||
const verdict = await sweepIssue(api, config, issue);
|
||||
console.log(describe(issue, verdict, config.dryRun));
|
||||
return [...verdicts, verdict];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
function describeReopen(issueNumber: number, verdict: ReopenVerdict, dryRun: boolean): string {
|
||||
if (verdict.kind === "skip") {
|
||||
return `#${issueNumber}: skipped, ${verdict.reason}`;
|
||||
}
|
||||
return `#${issueNumber}: ${dryRun ? "would reopen" : "reopened"} for the reporter's reply`;
|
||||
}
|
||||
|
||||
export async function reopenSweep(api: GitHubApi, config: SweepConfig): Promise<readonly ReopenVerdict[]> {
|
||||
const since = new Date(config.now.getTime() - REOPEN_LOOKBACK_DAYS * DAY_MS).toISOString();
|
||||
const closedPath = `/repos/${config.repo}/issues?state=closed&labels=duplicate,${FLAG_LABEL}&since=${encodeURIComponent(since)}`;
|
||||
const issues = await listAll<Issue>(api, closedPath);
|
||||
console.log(`${issues.length} recently closed issues carry the duplicate and ${FLAG_LABEL} labels in ${config.repo}${config.dryRun ? " (dry run)" : ""}`);
|
||||
return issues.reduce<Promise<readonly ReopenVerdict[]>>(async (previous, issue) => {
|
||||
const verdicts = await previous;
|
||||
const verdict = await sweepClosedIssue(api, config, issue.number);
|
||||
console.log(describeReopen(issue.number, verdict, config.dryRun));
|
||||
return [...verdicts, verdict];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
export function readConfig(env: Readonly<Record<string, string | undefined>>, now: Date): SweepConfig & { readonly token: string } {
|
||||
const token = env.GITHUB_TOKEN;
|
||||
const repo = env.GITHUB_REPOSITORY;
|
||||
if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) {
|
||||
throw new Error("GITHUB_TOKEN and GITHUB_REPOSITORY (owner/repo) are required");
|
||||
}
|
||||
const rawGraceDays = env.GRACE_PERIOD_DAYS?.trim();
|
||||
const graceDays = rawGraceDays === undefined || rawGraceDays === "" ? DEFAULT_GRACE_DAYS : Number(rawGraceDays);
|
||||
if (!Number.isFinite(graceDays) || graceDays < 0) {
|
||||
throw new Error(`GRACE_PERIOD_DAYS must be a non-negative number, got "${env.GRACE_PERIOD_DAYS}"`);
|
||||
}
|
||||
return { token, repo, graceDays, dryRun: env.DRY_RUN === "true", now };
|
||||
}
|
||||
|
||||
export function githubApi(token: string): GitHubApi {
|
||||
return {
|
||||
request: async <T>(method: "GET" | "POST" | "PATCH" | "DELETE", path: string, body?: object): Promise<T> => {
|
||||
const response = await fetch(`https://api.github.com${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "litellm-auto-close-duplicates",
|
||||
...(body === undefined ? {} : { "Content-Type": "application/json" }),
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`${method} ${path} failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const { token, ...config } = readConfig(process.env, new Date());
|
||||
const api = githubApi(token);
|
||||
const closeVerdicts = await sweep(api, config);
|
||||
const reopenVerdicts = await reopenSweep(api, config);
|
||||
const closed = closeVerdicts.filter((verdict) => verdict.kind === "close").length;
|
||||
const reopened = reopenVerdicts.filter((verdict) => verdict.kind === "reopen").length;
|
||||
console.log(
|
||||
`${config.dryRun ? "Would close" : "Closed"} ${closed} of ${closeVerdicts.length} flagged issues, ${config.dryRun ? "would reopen" : "reopened"} ${reopened}`,
|
||||
);
|
||||
}
|
||||
|
|
@ -1317,6 +1317,62 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch):
|
|||
assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_team_settings_newrelic_resolves_traces_and_metrics():
|
||||
"""Static `default_team_settings` is the config-file twin of POST /team/callback.
|
||||
|
||||
A team pinned to New Relic through `default_team_settings` must reach the
|
||||
same two loggers the dynamic path does: the per-team metrics logger (cost
|
||||
and usage) and the trace logger (LLM/agent spans). This proves the static
|
||||
path resolves both, not just one, so the config-file customer gets the
|
||||
same per-team routing as the API customer.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
pc = ProxyConfig()
|
||||
pc.config = {
|
||||
"litellm_settings": {
|
||||
"default_team_settings": [
|
||||
{
|
||||
"team_id": "team-a",
|
||||
"success_callback": ["newrelic"],
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config(
|
||||
team_id="team-a",
|
||||
proxy_config=pc,
|
||||
)
|
||||
|
||||
assert callback_metadata is not None
|
||||
assert callback_metadata.success_callback == ["newrelic"]
|
||||
assert callback_metadata.callback_vars == {
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
|
||||
logging_obj = Logging(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="static-nr-1",
|
||||
function_id="static-nr-1",
|
||||
)
|
||||
logging_obj._trusted_callback_vars = tuple(callback_metadata.callback_vars.items())
|
||||
|
||||
resolved = logging_obj._resolve_dynamic_callback_string("newrelic")
|
||||
resolved_names = {type(logger).__name__ for logger in resolved}
|
||||
assert resolved_names == {"NewRelicMetricsLogger", "NewRelicLogger"}
|
||||
|
||||
|
||||
def test_proxy_config_state_get_config_state_error():
|
||||
"""
|
||||
Ensures that get_config_state does not raise an error when the config is not a valid dictionary
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import litellm
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy._types import CallInfo, Litellm_EntityType
|
||||
from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys
|
||||
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys
|
||||
|
||||
|
||||
class TestSlackAlerting(unittest.TestCase):
|
||||
|
|
@ -366,3 +366,56 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through():
|
|||
|
||||
_, kwargs = slack_alerting._run_scheduler_helper.await_args
|
||||
assert kwargs["pod_lock_manager"] is pod_lock_manager
|
||||
|
||||
|
||||
def _slack_alerting_with_env_resolution() -> SlackAlerting:
|
||||
slack_alerting: Final = SlackAlerting(alerting=["slack"], internal_usage_cache=DualCache())
|
||||
slack_alerting.periodic_started = True
|
||||
return slack_alerting
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_falls_back_to_alerting_webhook_url_env(monkeypatch):
|
||||
monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False)
|
||||
monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc")
|
||||
slack_alerting: Final = _slack_alerting_with_env_resolution()
|
||||
|
||||
await slack_alerting.send_alert(
|
||||
message="budget crossed",
|
||||
level="High",
|
||||
alert_type=AlertType.budget_alerts,
|
||||
alerting_metadata={},
|
||||
)
|
||||
|
||||
assert slack_alerting.log_queue[0]["url"] == "https://chat.example.com/hooks/abc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_prefers_slack_webhook_url_over_fallback(monkeypatch):
|
||||
monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T0/B0/X0")
|
||||
monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc")
|
||||
slack_alerting: Final = _slack_alerting_with_env_resolution()
|
||||
|
||||
await slack_alerting.send_alert(
|
||||
message="budget crossed",
|
||||
level="High",
|
||||
alert_type=AlertType.budget_alerts,
|
||||
alerting_metadata={},
|
||||
)
|
||||
|
||||
assert slack_alerting.log_queue[0]["url"] == "https://hooks.slack.com/services/T0/B0/X0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_alert_raises_when_no_webhook_url_configured(monkeypatch):
|
||||
monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False)
|
||||
monkeypatch.delenv("ALERTING_WEBHOOK_URL", raising=False)
|
||||
slack_alerting: Final = _slack_alerting_with_env_resolution()
|
||||
|
||||
with pytest.raises(ValueError, match="SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL"):
|
||||
await slack_alerting.send_alert(
|
||||
message="budget crossed",
|
||||
level="High",
|
||||
alert_type=AlertType.budget_alerts,
|
||||
alerting_metadata={},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -79,6 +79,23 @@ class TestDigestMode(unittest.IsolatedAsyncioTestCase):
|
|||
|
||||
self.assertEqual(len(self.slack_alerting.digest_buckets), 2)
|
||||
|
||||
async def test_digest_falls_back_to_alerting_webhook_url_env(self):
|
||||
"""With SLACK_WEBHOOK_URL unset, the digest entry resolves ALERTING_WEBHOOK_URL instead."""
|
||||
env = {k: v for k, v in os.environ.items() if k != "SLACK_WEBHOOK_URL"}
|
||||
env["ALERTING_WEBHOOK_URL"] = "https://chat.example.com/hooks/abc"
|
||||
with unittest.mock.patch.dict(os.environ, env, clear=True):
|
||||
await self.slack_alerting.send_alert(
|
||||
message="`Requests are hanging`",
|
||||
level="Medium",
|
||||
alert_type=AlertType.llm_requests_hanging,
|
||||
alerting_metadata={},
|
||||
request_model="gemini-2.5-flash",
|
||||
api_base="None",
|
||||
)
|
||||
|
||||
bucket = list(self.slack_alerting.digest_buckets.values())[0]
|
||||
self.assertEqual(bucket["webhook_url"], "https://chat.example.com/hooks/abc")
|
||||
|
||||
async def test_non_digest_alert_goes_to_queue(self):
|
||||
"""Alert types without digest enabled should go straight to the log queue."""
|
||||
message = "Budget exceeded"
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ class TestUpCommand:
|
|||
assert captured["settings"]["theme"] == "dark"
|
||||
assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:5483"
|
||||
assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key"
|
||||
assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert "apiKeyHelper" not in captured["settings"]
|
||||
assert captured["settings_mode"] == 0o600
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,13 @@ def test_sets_base_url_and_auth_token():
|
|||
merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000/", "token-abc")
|
||||
assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000"
|
||||
assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc"
|
||||
assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
|
||||
|
||||
def test_preserves_existing_tool_search():
|
||||
settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}}
|
||||
merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc")
|
||||
assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false"
|
||||
|
||||
|
||||
def test_drops_stray_api_key():
|
||||
|
|
|
|||
|
|
@ -77,9 +77,19 @@ class TestBuildAgentEnv:
|
|||
)
|
||||
assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
|
||||
assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key"
|
||||
assert env["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert "OPENAI_BASE_URL" not in env
|
||||
assert "OPENAI_API_KEY" not in env
|
||||
|
||||
def test_anthropic_profile_preserves_existing_tool_search(self):
|
||||
env = build_agent_env(
|
||||
{"ENABLE_TOOL_SEARCH": "false"},
|
||||
"http://localhost:4000",
|
||||
"sk-key",
|
||||
frozenset({"anthropic"}),
|
||||
)
|
||||
assert env["ENABLE_TOOL_SEARCH"] == "false"
|
||||
|
||||
def test_anthropic_profile_drops_existing_api_key(self):
|
||||
env = build_agent_env(
|
||||
{"ANTHROPIC_API_KEY": "real-key"},
|
||||
|
|
@ -96,6 +106,7 @@ class TestBuildAgentEnv:
|
|||
assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1"
|
||||
assert env["OPENAI_API_KEY"] == "sk-key"
|
||||
assert "ANTHROPIC_BASE_URL" not in env
|
||||
assert "ENABLE_TOOL_SEARCH" not in env
|
||||
|
||||
def test_both_profiles_set_everything(self):
|
||||
env = build_agent_env(
|
||||
|
|
@ -105,6 +116,7 @@ class TestBuildAgentEnv:
|
|||
assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1"
|
||||
assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key"
|
||||
assert env["OPENAI_API_KEY"] == "sk-key"
|
||||
assert env["ENABLE_TOOL_SEARCH"] == "true"
|
||||
|
||||
def test_preserves_unrelated_env_and_does_not_mutate_input(self):
|
||||
base = {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"}
|
||||
|
|
@ -201,6 +213,7 @@ class TestRunAgent:
|
|||
env = calls["env"]
|
||||
assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
|
||||
assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key"
|
||||
assert env["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert "ANTHROPIC_API_KEY" not in env
|
||||
assert "OPENAI_BASE_URL" not in env
|
||||
|
||||
|
|
@ -218,6 +231,7 @@ class TestRunAgent:
|
|||
assert calls["env"]["OPENAI_BASE_URL"] == "http://localhost:4000/v1"
|
||||
assert calls["env"]["OPENAI_API_KEY"] == "sk-key"
|
||||
assert "ANTHROPIC_BASE_URL" not in calls["env"]
|
||||
assert "ENABLE_TOOL_SEARCH" not in calls["env"]
|
||||
|
||||
def test_codex_injects_proxy_provider_args_before_user_args(self):
|
||||
calls = {}
|
||||
|
|
|
|||
|
|
@ -1373,6 +1373,7 @@ class TestLoginConfigClaude:
|
|||
assert result.exit_code == 0
|
||||
written = json.loads(settings_path.read_text())
|
||||
assert written["env"]["ANTHROPIC_BASE_URL"] == "https://test.example.com"
|
||||
assert written["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token"
|
||||
assert "Configured Claude Code" in result.output
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ class TestWriteClaudeSettings:
|
|||
|
||||
written = json.loads(settings_path.read_text())
|
||||
assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com"
|
||||
assert written["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token"
|
||||
|
||||
def test_updates_an_existing_file_preserving_unrelated_settings(self, paths, lite_on_path):
|
||||
|
|
|
|||
|
|
@ -55,8 +55,14 @@ class TestMergeClaudeSettings:
|
|||
}
|
||||
merged = merge_claude_settings(settings, "http://localhost:4000/", "new-helper")
|
||||
assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
|
||||
assert merged["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert merged["apiKeyHelper"] == "new-helper"
|
||||
|
||||
def test_preserves_existing_tool_search(self):
|
||||
settings = {"env": {"ENABLE_TOOL_SEARCH": "false"}}
|
||||
merged = merge_claude_settings(settings, "http://localhost:4000", "helper")
|
||||
assert merged["env"]["ENABLE_TOOL_SEARCH"] == "false"
|
||||
|
||||
def test_drops_stray_api_key(self):
|
||||
settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}}
|
||||
merged = merge_claude_settings(settings, "http://localhost:4000", "helper")
|
||||
|
|
@ -64,7 +70,10 @@ class TestMergeClaudeSettings:
|
|||
|
||||
def test_works_from_empty_settings(self):
|
||||
merged = merge_claude_settings({}, "http://localhost:4000", "helper")
|
||||
assert merged["env"] == {"ANTHROPIC_BASE_URL": "http://localhost:4000"}
|
||||
assert merged["env"] == {
|
||||
"ANTHROPIC_BASE_URL": "http://localhost:4000",
|
||||
"ENABLE_TOOL_SEARCH": "true",
|
||||
}
|
||||
assert merged["apiKeyHelper"] == "helper"
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
|
|
@ -486,6 +495,7 @@ class TestUpCommand:
|
|||
assert captured["backup_existed"] is True
|
||||
assert captured["settings"]["theme"] == "dark"
|
||||
assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
|
||||
assert captured["settings"]["env"]["ENABLE_TOOL_SEARCH"] == "true"
|
||||
assert captured["settings"]["apiKeyHelper"] == "/usr/local/bin/lite auth print-token"
|
||||
assert json.loads(settings_path.read_text()) == original
|
||||
assert not backup_path.exists()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import asyncio
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.exceptions import HTTPException
|
||||
from httpx import Request, Response
|
||||
from httpx import ReadTimeout, Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.chat.guardrail_translation.handler import (
|
||||
|
|
@ -40,6 +41,7 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch):
|
|||
"guardrail": "prompt_security",
|
||||
"mode": "during_call",
|
||||
"default_on": True,
|
||||
"file_sanitization_fail_open": False,
|
||||
},
|
||||
}
|
||||
],
|
||||
|
|
@ -51,6 +53,10 @@ def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch):
|
|||
assert registered[0].guardrail_name == "prompt_security"
|
||||
assert registered[0].default_on is True
|
||||
assert registered[0].event_hook == "during_call"
|
||||
assert registered[0].file_sanitization_fail_open is False
|
||||
config_model = registered[0].get_config_model()
|
||||
assert config_model is not None
|
||||
assert config_model().file_sanitization_fail_open is True
|
||||
|
||||
|
||||
def test_prompt_security_guard_config_no_api_key(monkeypatch: pytest.MonkeyPatch):
|
||||
|
|
@ -384,6 +390,86 @@ async def test_file_sanitization(monkeypatch: pytest.MonkeyPatch):
|
|||
assert result is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"timeout",
|
||||
(
|
||||
litellm.Timeout(
|
||||
message="Prompt Security upload timed out",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
),
|
||||
ReadTimeout(
|
||||
"Prompt Security poll timed out",
|
||||
request=Request(method="GET", url="https://test.prompt.security/api/sanitizeFile"),
|
||||
),
|
||||
),
|
||||
ids=("litellm", "httpx"),
|
||||
)
|
||||
@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed"))
|
||||
async def test_file_sanitization_request_timeout_policy(
|
||||
monkeypatch: pytest.MonkeyPatch, timeout: Exception, fail_open: bool
|
||||
):
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key")
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security")
|
||||
|
||||
guardrail = PromptSecurityGuardrail(
|
||||
guardrail_name="test-guard",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
file_sanitization_fail_open=fail_open,
|
||||
)
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", AsyncMock(side_effect=timeout)):
|
||||
if not fail_open:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
assert exc_info.value.status_code == 408
|
||||
assert exc_info.value.detail == "File sanitization timeout"
|
||||
return
|
||||
|
||||
result = await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
|
||||
assert result == {
|
||||
"action": "allow",
|
||||
"content": None,
|
||||
"metadata": {},
|
||||
"violations": (),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fail_open", (True, False), ids=("fail-open", "fail-closed"))
|
||||
async def test_file_sanitization_overall_timeout_policy(monkeypatch: pytest.MonkeyPatch, fail_open: bool):
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key")
|
||||
monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security")
|
||||
|
||||
guardrail = PromptSecurityGuardrail(
|
||||
guardrail_name="test-guard",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
file_sanitization_timeout=0.01,
|
||||
file_sanitization_fail_open=fail_open,
|
||||
)
|
||||
|
||||
async def hanging_post(*_args: object, **_kwargs: object) -> None:
|
||||
await asyncio.sleep(60)
|
||||
raise AssertionError("sanitization request should have been cancelled")
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", side_effect=hanging_post):
|
||||
if not fail_open:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
assert exc_info.value.status_code == 408
|
||||
assert exc_info.value.detail == "File sanitization timeout"
|
||||
return
|
||||
|
||||
result = await guardrail.sanitize_file_content(b"file-content", "document.pdf")
|
||||
|
||||
assert result["action"] == "allow"
|
||||
assert result["content"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_sanitization_block(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that file sanitization blocks malicious files"""
|
||||
|
|
|
|||
|
|
@ -115,6 +115,35 @@ async def test_budget_alerts_slack_when_slack_alerting(proxy_logging):
|
|||
assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_alerts_webhook_only_forwards_to_slack_alerting_instance(proxy_logging):
|
||||
proxy_logging.alerting = ["webhook"]
|
||||
captured: Dict[str, Any] = {}
|
||||
|
||||
async def fake_alert(**kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=fake_alert)
|
||||
proxy_logging.email_logging_instance = None
|
||||
await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info())
|
||||
snapshot = {
|
||||
"type": captured["type"],
|
||||
"user_info_is_callinfo": isinstance(captured["user_info"], CallInfo),
|
||||
"user_id": captured["user_info"].user_id,
|
||||
}
|
||||
assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_alerts_email_only_skips_slack_alerting_instance(proxy_logging):
|
||||
proxy_logging.alerting = ["email"]
|
||||
proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=AsyncMock())
|
||||
proxy_logging.email_logging_instance = MagicMock(budget_alerts=AsyncMock())
|
||||
await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info())
|
||||
proxy_logging.slack_alerting_instance.budget_alerts.assert_not_called()
|
||||
proxy_logging.email_logging_instance.budget_alerts.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_global(proxy_logging):
|
||||
proxy_logging.alerting = None
|
||||
|
|
|
|||
|
|
@ -522,7 +522,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
<TabsContent value="alerting-types" keepMounted>
|
||||
<Card className="p-6">
|
||||
<p className="my-2">
|
||||
Alerts are only supported for Slack Webhook URLs. Get your webhook urls from{" "}
|
||||
Alerts are sent to any Slack-compatible incoming webhook URL (Slack, Rocket.Chat, Mattermost, etc.). Get
|
||||
Slack webhook urls from{" "}
|
||||
<a href="https://api.slack.com/messaging/webhooks" target="_blank" style={{ color: "blue" }}>
|
||||
here
|
||||
</a>
|
||||
|
|
@ -532,7 +533,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
<TableRow>
|
||||
<TableHead></TableHead>
|
||||
<TableHead></TableHead>
|
||||
<TableHead>Slack Webhook URL</TableHead>
|
||||
<TableHead>Webhook URL (Slack-compatible)</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
|
||||
|
|
|
|||
24
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
24
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -15505,7 +15505,7 @@ export interface paths {
|
|||
* Use this if if you want different teams to have different success/failure callbacks
|
||||
*
|
||||
* Parameters:
|
||||
* - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
* - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
* - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
* - "success": Callback for successful LLM calls
|
||||
* - "failure": Callback for failed LLM calls
|
||||
|
|
@ -15521,6 +15521,8 @@ export interface paths {
|
|||
* - langsmith_api_key: The API key for the Langsmith callback
|
||||
* - langsmith_project: The project for the Langsmith callback
|
||||
* - langsmith_base_url: The base URL for the Langsmith callback
|
||||
* - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
* - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
*
|
||||
* Example curl:
|
||||
* ```
|
||||
|
|
@ -16210,6 +16212,11 @@ export interface paths {
|
|||
*
|
||||
* Meant to optimize querying spend data for analytics for a user.
|
||||
*
|
||||
* Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
* resets. Their total can legitimately exceed the `spend` field returned by
|
||||
* `/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
* to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
*
|
||||
* Returns:
|
||||
* (by date)
|
||||
* - spend
|
||||
|
|
@ -16241,6 +16248,11 @@ export interface paths {
|
|||
* Get User Daily Activity Aggregated
|
||||
* @description Aggregated analytics for a user's daily activity without pagination.
|
||||
* Returns the same response shape as the paginated endpoint with page metadata set to single-page.
|
||||
*
|
||||
* Reads daily spend records that only ever accumulate and are never affected by budget
|
||||
* resets. Their total can legitimately exceed the `spend` field returned by
|
||||
* `/v2/user/info`, which is a running budget counter that every budget reset sets back
|
||||
* to zero (or to the overage above `max_budget` when `budget_rollover` is enabled).
|
||||
*/
|
||||
get: operations["get_user_daily_activity_aggregated_user_daily_activity_aggregated_get"];
|
||||
put?: never;
|
||||
|
|
@ -21004,6 +21016,14 @@ export interface paths {
|
|||
* This is the v2 replacement for /user/info, designed to avoid the "god endpoint" problem
|
||||
* where the old endpoint loaded all keys and teams into memory.
|
||||
*
|
||||
* Note on `spend`: this is the user's running budget counter, which the budget reset job
|
||||
* resets whenever `budget_reset_at` elapses (see `budget_duration`): to zero by default,
|
||||
* or to the overage above `max_budget` when `budget_rollover` is enabled. It is NOT
|
||||
* lifetime or per-period historical spend. For historical spend over a date range, use
|
||||
* `/user/daily/activity` or `/user/daily/activity/aggregated`, which read daily spend
|
||||
* records that only ever accumulate and are never reset. The two values are expected to
|
||||
* diverge once a budget reset has occurred within the queried period.
|
||||
*
|
||||
* Access control:
|
||||
* - Proxy admins can query any user
|
||||
* - Team admins can query users within their teams
|
||||
|
|
@ -25182,7 +25202,7 @@ export interface components {
|
|||
alert_types?: components["schemas"]["AlertType"][] | null;
|
||||
/**
|
||||
* Alerting
|
||||
* @description List of alerting integrations. Today, just slack - `alerting: ['slack']`
|
||||
* @description List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL
|
||||
*/
|
||||
alerting?: unknown[] | null;
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue