mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_same_provider_hop_keeps_encrypted_reasoning
This commit is contained in:
commit
32f4e04d9f
89 changed files with 5125 additions and 661 deletions
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
|
@ -1,5 +1,6 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
/.github/workflows/zizmor.yml @yuneng-berri
|
||||
|
|
|
|||
7
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
7
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
|
|||
## Screenshots / Proof of Fix
|
||||
|
||||
<!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
|
||||
The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
|
||||
Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
|
||||
Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
|
||||
Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
|
||||
Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
|
||||
Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
|
||||
Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
|
||||
|
||||
### Before (<hash>)
|
||||
|
||||
|
|
|
|||
12
.github/workflows/cost-map-guard.yml
vendored
12
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -21,16 +21,18 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head and its merge base
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
|
|
@ -38,6 +40,8 @@ jobs:
|
|||
- name: Run the guard
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
|
||||
COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
|
|
|
|||
91
.github/workflows/cost-map-pins-recheck.yml
vendored
Normal file
91
.github/workflows/cost-map-pins-recheck.yml
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: Cost map pins recheck
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- model_prices_and_context_window.json
|
||||
- litellm/model_prices_and_context_window_backup.json
|
||||
- ci_cd/cost_map_pins.json
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
rerun-cost-map-guard:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Re-run the cost map guard on open pull requests the push could break
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.ref_name }}
|
||||
BEFORE_SHA: ${{ github.event.before }}
|
||||
AFTER_SHA: ${{ github.event.after }}
|
||||
FORCED: ${{ github.event.forced }}
|
||||
COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
|
||||
PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
|
||||
run: |
|
||||
if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
|
||||
|| ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
|
||||
--jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
|
||||
| grep -qx true; then
|
||||
export RECHECK_PATHS="$COST_MAP_PATHS"
|
||||
else
|
||||
export RECHECK_PATHS="$PINS_PATHS"
|
||||
fi
|
||||
rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
|
||||
candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
|
||||
-f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
|
||||
nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
|
||||
pageInfo { hasNextPage endCursor }
|
||||
}
|
||||
}
|
||||
}' \
|
||||
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
|
||||
| {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
|
||||
| select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
|
||||
failed=0
|
||||
declare -A rerun_heads
|
||||
while read -r number head_sha listed; do
|
||||
[ -n "$number" ] || continue
|
||||
if [ "$listed" = false ]; then
|
||||
if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
|
||||
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
|
||||
echo "::error::could not list the files of #$number"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
[ -n "$changed" ] || continue
|
||||
fi
|
||||
[ -z "${rerun_heads[$head_sha]:-}" ] || continue
|
||||
rerun_heads[$head_sha]=1
|
||||
if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
|
||||
echo "::error::could not list the cost map guard runs of #$number"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
if [ -z "$runs" ]; then
|
||||
echo "::warning::no cost map guard run found for #$number"
|
||||
continue
|
||||
fi
|
||||
while read -r run_id created_at; do
|
||||
if [[ "$created_at" < "$rerun_cutoff" ]]; then
|
||||
echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
|
||||
continue
|
||||
fi
|
||||
gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
|
||||
if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
|
||||
echo "::error::could not re-run cost map guard run $run_id for #$number"
|
||||
failed=1
|
||||
fi
|
||||
done <<< "$runs"
|
||||
done <<< "$candidates"
|
||||
exit "$failed"
|
||||
|
|
@ -11,3 +11,12 @@ ignore:
|
|||
package:
|
||||
name: zlib
|
||||
type: apk
|
||||
- vulnerability: GHSA-v4x8-3wr5-3wq8
|
||||
reason: >-
|
||||
filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
|
||||
the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
|
||||
and the image then moves off 3.32.7, which retires this version-scoped rule
|
||||
package:
|
||||
name: filelock
|
||||
version: 3.32.7
|
||||
type: python
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
|
|||
|
||||
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
|
||||
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
|
||||
Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
|
||||
|
||||
For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
|
||||
|
||||
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
|
|
@ -107,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
|
|||
- No file sprawl: deliberate file and folder structure
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
|
||||
|
||||
Follow conventional commits for commit names and PR titles
|
||||
|
||||
|
|
|
|||
|
|
@ -1,37 +1,67 @@
|
|||
"""Guard the cost map on pull requests.
|
||||
|
||||
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
|
||||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
|
||||
file checks: the files parse, the backup copy matches the root file, and every capability pinned in
|
||||
ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
|
||||
otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
|
||||
a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
|
||||
all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
|
||||
whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
|
||||
litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
|
||||
only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
|
||||
(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
|
||||
Bot) get the human checks and may not touch the pins file either.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
PINS_PATH: Final = "ci_cd/cost_map_pins.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
|
||||
TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
|
||||
AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
|
||||
BOT_AUTHOR_TYPE: Final = "Bot"
|
||||
MERGE_CONFLICT_EXIT_CODE: Final = 1
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
CostMap = dict[str, object]
|
||||
_MISSING: Final = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Snapshot:
|
||||
cost_map: str
|
||||
backup: str
|
||||
pins: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MergeFailed:
|
||||
stderr: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pin:
|
||||
key: str
|
||||
field: str
|
||||
value: object
|
||||
checked: str
|
||||
source: str
|
||||
|
||||
|
||||
def _parse_object(text: str, path: str) -> CostMap | str:
|
||||
|
|
@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
|
|||
return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
|
||||
|
||||
|
||||
def _pin(key: str, field: str, spec: object) -> Pin | str:
|
||||
malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
|
||||
if not isinstance(spec, dict) or "value" not in spec:
|
||||
return malformed
|
||||
checked: Final = spec.get("checked")
|
||||
source: Final = spec.get("source")
|
||||
if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
|
||||
return malformed
|
||||
return Pin(key, field, spec["value"], checked, source)
|
||||
|
||||
|
||||
def _pins(parsed: CostMap) -> Iterator[Pin | str]:
|
||||
for key, fields in parsed.items():
|
||||
if not isinstance(fields, dict):
|
||||
yield f"{PINS_PATH}: {key} must map field names to pins"
|
||||
continue
|
||||
for field, spec in fields.items():
|
||||
yield _pin(key, field, spec)
|
||||
|
||||
|
||||
def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
|
||||
for pin in _pins(parsed):
|
||||
if isinstance(pin, str):
|
||||
yield pin
|
||||
elif pin.key not in entries:
|
||||
yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
|
||||
elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
|
||||
yield (
|
||||
f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
|
||||
f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
|
||||
)
|
||||
|
||||
|
||||
def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
|
||||
if not pins_text:
|
||||
return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
|
||||
parsed: Final = _parse_object(pins_text, PINS_PATH)
|
||||
if isinstance(parsed, str):
|
||||
return (parsed,)
|
||||
return tuple(_pin_violations(parsed, _entries(head_map)))
|
||||
|
||||
|
||||
def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
|
||||
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
|
||||
if isinstance(base_map, str):
|
||||
|
|
@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
)
|
||||
|
||||
|
||||
def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
|
||||
if PINS_PATH not in changed_files:
|
||||
return ()
|
||||
if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
|
||||
return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
|
||||
if author_type == BOT_AUTHOR_TYPE:
|
||||
return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
|
||||
return ()
|
||||
|
||||
|
||||
def touches_cost_map(changed_files: Sequence[str]) -> bool:
|
||||
return any(path in GUARDED_PATHS for path in changed_files)
|
||||
return any(path in CHECKED_PATHS for path in changed_files)
|
||||
|
||||
|
||||
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
||||
|
|
@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
|||
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
|
||||
|
||||
|
||||
def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
|
||||
if merged is None:
|
||||
return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
|
||||
if not merged.pins and not base.pins:
|
||||
return ()
|
||||
merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
|
||||
if isinstance(merged_map, str):
|
||||
return (f"after merging into the base branch, {merged_map}",)
|
||||
return pin_failures(merged.pins, merged_map)
|
||||
|
||||
|
||||
def guard_failures(
|
||||
base: Snapshot,
|
||||
head: Snapshot,
|
||||
merged: Snapshot | None,
|
||||
changed_files: Sequence[str],
|
||||
bot: bool,
|
||||
*,
|
||||
|
|
@ -147,6 +241,7 @@ def guard_failures(
|
|||
*_file_failures(
|
||||
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
|
||||
),
|
||||
*_merged_pin_failures(base, merged),
|
||||
*(_bot_failures(base, head_map, changed_files) if bot else ()),
|
||||
)
|
||||
|
||||
|
|
@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
|
|||
|
||||
|
||||
def snapshot(revision: str) -> Snapshot:
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
|
||||
|
||||
|
||||
def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
|
||||
result: Final = subprocess.run(
|
||||
("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
tree: Final = result.stdout.split("\n", 1)[0]
|
||||
if result.returncode == 0:
|
||||
return tree
|
||||
if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
|
||||
return None
|
||||
return MergeFailed(result.stderr.strip())
|
||||
|
||||
|
||||
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
|
||||
|
|
@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base", required=True, help="merge base of the pull request")
|
||||
parser.add_argument("--head", required=True, help="head commit of the pull request")
|
||||
env_target: Final = os.environ.get(TARGET_ENV)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
default=env_target,
|
||||
required=env_target is None,
|
||||
help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
|
||||
)
|
||||
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
|
||||
args: Final = parser.parse_args(argv)
|
||||
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
|
||||
|
|
@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
|
|||
if changed is None:
|
||||
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
|
||||
return 1
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
|
||||
if not bot and not touches_cost_map(changed):
|
||||
print(f"cost map guard passed ({contract_for(bot, changed)})")
|
||||
return 0
|
||||
merged: Final = merged_tree(args.base, args.target, args.head)
|
||||
if isinstance(merged, MergeFailed):
|
||||
print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
|
||||
return 1
|
||||
failures: Final = (
|
||||
*guard_failures(
|
||||
snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
|
||||
),
|
||||
*pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
|
||||
)
|
||||
contract: Final = contract_for(bot, changed)
|
||||
if failures:
|
||||
print(f"cost map guard failed ({contract}):")
|
||||
|
|
|
|||
16
ci_cd/cost_map_pins.json
Normal file
16
ci_cd/cost_map_pins.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"fireworks_ai/accounts/fireworks/models/minimax-m3": {
|
||||
"supports_vision": {
|
||||
"value": true,
|
||||
"checked": "2026-09-26",
|
||||
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
|
||||
}
|
||||
},
|
||||
"fireworks_ai/minimax-m3": {
|
||||
"supports_vision": {
|
||||
"value": true,
|
||||
"checked": "2026-09-26",
|
||||
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
|
|||
####### COMPLETION MODELS ###################
|
||||
from typing import Set
|
||||
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_chat_completion_models: Set[str] = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
|
|
|
|||
|
|
@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None:
|
|||
return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields)
|
||||
|
||||
|
||||
def _genai_content_parts(value: object) -> Sequence[object]:
|
||||
single_part: Final = _validated(_CONFIG_FIELDS, value)
|
||||
return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or ()
|
||||
|
||||
|
||||
def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]:
|
||||
items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or ()
|
||||
return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None)
|
||||
|
|
@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]:
|
|||
fields: Final = _validated(_CONFIG_FIELDS, content)
|
||||
if fields is None:
|
||||
return ()
|
||||
parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or ()
|
||||
parts: Final = _genai_content_parts(fields.get("parts", ()))
|
||||
converted: Final = _genai_message_parts(parts)
|
||||
role: Final = fields.get("role", "user")
|
||||
if role == "user":
|
||||
|
|
|
|||
|
|
@ -5052,6 +5052,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return result_copy
|
||||
|
||||
|
||||
_SENSITIVE_KEYWORDS: Final = (
|
||||
"authorization",
|
||||
"token",
|
||||
"key",
|
||||
"secret",
|
||||
"vertex_credentials",
|
||||
"credentials",
|
||||
"password",
|
||||
"passwd",
|
||||
)
|
||||
|
||||
|
||||
def has_sensitive_keyword(key: str) -> bool:
|
||||
lowered: Final = key.lower()
|
||||
return any(keyword in lowered for keyword in _SENSITIVE_KEYWORDS)
|
||||
|
||||
|
||||
def get_masked_values(
|
||||
sensitive_object: Mapping[str, object],
|
||||
ignore_sensitive_values: bool = False,
|
||||
|
|
@ -5070,16 +5087,6 @@ def get_masked_values(
|
|||
masked_length: Optional length for the masked portion (number of *). If set, will use exactly this many *
|
||||
regardless of original string length. The total length will be unmasked_length + masked_length.
|
||||
"""
|
||||
sensitive_keywords: Final = (
|
||||
"authorization",
|
||||
"token",
|
||||
"key",
|
||||
"secret",
|
||||
"vertex_credentials",
|
||||
"credentials",
|
||||
"password",
|
||||
"passwd",
|
||||
)
|
||||
|
||||
def _mask_value(v: object) -> object:
|
||||
if isinstance(v, dict):
|
||||
|
|
@ -5106,12 +5113,7 @@ def get_masked_values(
|
|||
return v[: unmasked_length // 2] + "*" * (len(v) - unmasked_length) + v[-unmasked_length // 2 :]
|
||||
|
||||
return {
|
||||
k: (
|
||||
v
|
||||
if ignore_sensitive_values
|
||||
or not any(sensitive_keyword in k.lower() for sensitive_keyword in sensitive_keywords)
|
||||
else _mask_value(v)
|
||||
)
|
||||
k: (v if ignore_sensitive_values or not has_sensitive_keyword(k) else _mask_value(v))
|
||||
for k, v in sensitive_object.items()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRAC
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantContentValue,
|
||||
|
|
@ -51,7 +52,7 @@ from litellm.types.utils import (
|
|||
UnreportedUsage,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import print_verbose, token_counter
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
|
@ -1552,6 +1553,9 @@ def stream_chunk_builder(
|
|||
return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
|
||||
|
||||
model: Final = chunks[0]["model"]
|
||||
usage_model: Final = usage_tokenizer_model(
|
||||
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
|
||||
)
|
||||
# Initialize the response dictionary
|
||||
response: Final = processor.build_base_response(chunks)
|
||||
|
||||
|
|
@ -1575,7 +1579,7 @@ def stream_chunk_builder(
|
|||
completion_output = get_content_from_model_response(response)
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks,
|
||||
model=model,
|
||||
model=usage_model,
|
||||
completion_output=completion_output,
|
||||
messages=messages,
|
||||
reasoning_tokens=0,
|
||||
|
|
@ -1766,7 +1770,7 @@ def stream_chunk_builder(
|
|||
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks,
|
||||
model=model,
|
||||
model=usage_model,
|
||||
completion_output=completion_output,
|
||||
messages=messages,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
|
|
@ -49,6 +50,7 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
|
||||
from litellm.litellm_core_utils.url_utils import safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicContentParamSource,
|
||||
|
|
@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding.
|
||||
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
|
||||
know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
|
||||
OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name
|
||||
listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an
|
||||
unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release
|
||||
(`openai/gpt-6.2`) gets it."""
|
||||
model_to_use: Final = _openai_model_name(model) or _fix_model_name(model)
|
||||
if "gpt-4o" in model_to_use:
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return "cl100k_base"
|
||||
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
|
||||
|
||||
|
||||
def unresolved_tokenizer_model(model: str) -> str:
|
||||
"""`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding
|
||||
tiktoken gives the name as written, so a count with the result never drops below that one."""
|
||||
as_written: Final = _fix_model_name(model)
|
||||
as_written_encoding: Final = (
|
||||
"o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base")
|
||||
)
|
||||
moved_off_cl100k: Final = (
|
||||
as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base"
|
||||
)
|
||||
return "gpt-3.5-turbo" if moved_off_cl100k else model
|
||||
|
||||
|
||||
def usage_tokenizer_model(model: str, base_model: str | None) -> str:
|
||||
"""The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a
|
||||
custom deployment name says nothing about its tokenizer, else `model`."""
|
||||
return base_model if base_model is not None and is_openai_model(base_model) else model
|
||||
|
||||
|
||||
def is_openai_model(model: str) -> bool:
|
||||
"""Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by
|
||||
tiktoken, with or without its prefixes."""
|
||||
return _openai_model_name(model) is not None
|
||||
|
||||
|
||||
def deployment_openai_model(deployment: object) -> str | None:
|
||||
"""The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name
|
||||
(`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model."""
|
||||
return _as_openai_model(_served_model(deployment))
|
||||
|
||||
|
||||
def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]:
|
||||
"""The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model."""
|
||||
served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments)
|
||||
return tuple(dict.fromkeys(_as_openai_model(model) for model in served))
|
||||
|
||||
|
||||
def _served_model(deployment: object) -> str | None:
|
||||
return next(iter(deployment_pricing_models(deployment)), None)
|
||||
|
||||
|
||||
def _as_openai_model(model: str | None) -> str | None:
|
||||
return model if model is not None and is_openai_model(model) else None
|
||||
|
||||
|
||||
def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
|
||||
"""An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same
|
||||
encoding, else None."""
|
||||
if not served or None in served:
|
||||
return None
|
||||
encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None)
|
||||
return served[0] if len(encodings) == 1 else None
|
||||
|
||||
|
||||
def _openai_model_name(model: str) -> str | None:
|
||||
return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
|
||||
|
||||
|
||||
_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
|
||||
|
||||
|
||||
def _versioned_openai_model_name(model: str) -> str | None:
|
||||
name: Final = model.rsplit("/", 1)[-1]
|
||||
return name if _VERSIONED_OPENAI_MODEL.match(name) else None
|
||||
|
||||
|
||||
def _tiktoken_o200k_model_name(model: str) -> str | None:
|
||||
name: Final = model.rsplit("/", 1)[-1]
|
||||
return name if _tiktoken_encoding_name(name) == "o200k_base" else None
|
||||
|
||||
|
||||
def _listed_openai_model_name(model: str) -> str | None:
|
||||
segments: Final = model.split("/")
|
||||
suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments)))
|
||||
listed: Final = any(
|
||||
_is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes
|
||||
)
|
||||
return segments[-1] if listed else None
|
||||
|
||||
|
||||
def _is_openai_cost_map_model(model: str) -> bool:
|
||||
return (
|
||||
model in litellm.azure_llms
|
||||
or model in litellm.open_ai_chat_completion_models
|
||||
or _is_azure_openai_text_model(model)
|
||||
)
|
||||
|
||||
|
||||
_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"})
|
||||
|
||||
|
||||
def _is_azure_openai_text_model(model: str) -> bool:
|
||||
cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {}
|
||||
return (
|
||||
cost_info.get("litellm_provider") == "azure"
|
||||
and cost_info.get("mode") in _TEXT_MODEL_MODES
|
||||
and _is_openai_model_name(model.rsplit("/", 1)[-1])
|
||||
)
|
||||
|
||||
|
||||
def _is_openai_model_name(name: str) -> bool:
|
||||
return (
|
||||
name in litellm.open_ai_chat_completion_models
|
||||
or _tiktoken_encoding_name(name) is not None
|
||||
or any(
|
||||
name.startswith(f"{listed}-") or listed.startswith(f"{name}-")
|
||||
for listed in tuple(litellm.open_ai_chat_completion_models)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _tiktoken_encoding_name(model: str) -> str | None:
|
||||
try:
|
||||
return tiktoken.encoding_name_for_model(model)
|
||||
except KeyError:
|
||||
return None
|
||||
|
||||
|
||||
def _suffixed_variant_encoding_name(model: str) -> str | None:
|
||||
known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-"))
|
||||
return MODEL_TO_ENCODING[max(known, key=len)] if known else None
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from ..constants import (
|
|||
)
|
||||
from ..errors import AnthropicContextManagementError
|
||||
from ..placeholders import build_cleared_tool_result_content
|
||||
from .compact import count_system_tokens, history_after_client_compaction
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
|
@ -203,9 +204,10 @@ def _trigger_met(
|
|||
model: str,
|
||||
messages: list[dict[str, object]],
|
||||
tools: list[dict[str, object]] | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
counted_tool_uses: int,
|
||||
) -> tuple[bool, int | None]:
|
||||
"""Return (trigger_met, input_tokens if counted for reuse)."""
|
||||
"""Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt."""
|
||||
trigger_type: Final = trigger.get("type", "input_tokens")
|
||||
threshold = trigger.get("value")
|
||||
|
||||
|
|
@ -216,14 +218,17 @@ def _trigger_met(
|
|||
|
||||
if not isinstance(threshold, int):
|
||||
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
|
||||
current_tokens: Final = litellm.token_counter(
|
||||
message_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
if message_tokens > threshold:
|
||||
return True, message_tokens
|
||||
current_tokens: Final = message_tokens + count_system_tokens(model, system)
|
||||
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
|
||||
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
|
||||
return current_tokens > threshold, current_tokens
|
||||
return current_tokens > threshold, message_tokens
|
||||
|
||||
|
||||
def _resolve_keep_count(keep: dict[str, object]) -> int:
|
||||
|
|
@ -237,7 +242,7 @@ def _resolve_keep_count(keep: dict[str, object]) -> int:
|
|||
|
||||
|
||||
def _last_completed_tool_use_id(
|
||||
messages: list[dict[str, object]],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
) -> str | None:
|
||||
"""Latest completed tool_result id; never cleared."""
|
||||
last_id: str | None = None
|
||||
|
|
@ -253,7 +258,7 @@ def _last_completed_tool_use_id(
|
|||
|
||||
|
||||
def _clear_tool_results(
|
||||
messages: list[dict[str, object]], ids_to_clear: set[str]
|
||||
messages: list[dict[str, object]], ids_to_clear: Set[str]
|
||||
) -> tuple[list[dict[str, object]], int]:
|
||||
"""Clear matching tool_result content; return (messages, cleared_count)."""
|
||||
cleared = 0
|
||||
|
|
@ -311,7 +316,11 @@ def apply_clear_tool_uses_20250919(
|
|||
system: str | list[dict[str, object]] | None,
|
||||
edit_spec: ClearToolUsesEditSpec,
|
||||
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
|
||||
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None)."""
|
||||
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).
|
||||
|
||||
Like the API, it reads only the history after a client compaction block and counts the block's summary as system.
|
||||
"""
|
||||
kept, counted_system = history_after_client_compaction(messages, system)
|
||||
fields: Final = _ClearToolUsesFields.model_validate(edit_spec)
|
||||
clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value
|
||||
exclude_tools: Final = frozenset(fields.exclude_tools or ())
|
||||
|
|
@ -326,7 +335,7 @@ def apply_clear_tool_uses_20250919(
|
|||
"value": DEFAULT_KEEP_TOOL_USES,
|
||||
}
|
||||
|
||||
blocks: Final = _all_blocks(messages)
|
||||
blocks: Final = _all_blocks(kept)
|
||||
results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id")
|
||||
uses: Final = _blocks_by_id(blocks, "tool_use", "id")
|
||||
tiny: Final = _tiny_result_ids(results)
|
||||
|
|
@ -335,15 +344,15 @@ def apply_clear_tool_uses_20250919(
|
|||
for tool_use_id, name in _tool_uses_in_order(blocks)
|
||||
if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools))
|
||||
)
|
||||
met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable))
|
||||
if not met:
|
||||
return messages, None
|
||||
|
||||
keep_count: Final = _resolve_keep_count(keep)
|
||||
if len(clearable) <= keep_count:
|
||||
return messages, None
|
||||
|
||||
last_completed_id: Final = _last_completed_tool_use_id(messages)
|
||||
met, counted_tokens = _trigger_met(trigger, model, kept, tools, counted_system, len(clearable))
|
||||
if not met:
|
||||
return messages, None
|
||||
|
||||
last_completed_id: Final = _last_completed_tool_use_id(kept)
|
||||
candidates: Final = tuple(
|
||||
(tool_use_id, name)
|
||||
for tool_use_id, name in clearable[: len(clearable) - keep_count]
|
||||
|
|
@ -358,7 +367,7 @@ def apply_clear_tool_uses_20250919(
|
|||
_select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids
|
||||
)
|
||||
|
||||
edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear))
|
||||
edited, cleared_count = _clear_tool_results(messages, ids_to_clear)
|
||||
verbose_logger.debug("context_management polyfill: edited: %s", edited)
|
||||
if cleared_count == 0:
|
||||
return messages, None
|
||||
|
|
@ -370,11 +379,12 @@ def apply_clear_tool_uses_20250919(
|
|||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(
|
||||
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
kept_after, _ = history_after_client_compaction(final_messages, system)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
|
|
|
|||
|
|
@ -189,6 +189,21 @@ def _strip_compaction_blocks(
|
|||
return cleaned
|
||||
|
||||
|
||||
def history_after_client_compaction(
|
||||
messages: Sequence[dict[str, object]],
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
|
||||
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
if block is None:
|
||||
return list(messages), system
|
||||
tail: Final = _strip_compaction_blocks(sliced)
|
||||
summary: Final = block.get("content")
|
||||
if not isinstance(summary, str) or not summary:
|
||||
return tail, system
|
||||
return tail, _augment_system_with_summary(_system_to_text(system) or None, summary)
|
||||
|
||||
|
||||
def _augment_system_with_summary(
|
||||
system: str | list[dict[str, object]] | None,
|
||||
summary_text: str,
|
||||
|
|
@ -320,14 +335,16 @@ def _count_effective_tokens(
|
|||
content: Final = compaction_block.get("content") or ""
|
||||
if content:
|
||||
total += litellm.token_counter(model=model, text=content)
|
||||
return total + count_system_tokens(model, system)
|
||||
|
||||
|
||||
def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int:
|
||||
system_text: Final = _system_to_text(system)
|
||||
if system_text:
|
||||
total += litellm.token_counter(model=model, text=system_text)
|
||||
return total
|
||||
return litellm.token_counter(model=model, text=system_text) if system_text else 0
|
||||
|
||||
|
||||
def _system_to_text(
|
||||
system: str | list[dict[str, object]] | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> str:
|
||||
"""Flatten an Anthropic-style ``system`` value into a single string for
|
||||
token counting. Returns ``""`` when ``system`` carries no text."""
|
||||
|
|
@ -338,7 +355,7 @@ def _system_to_text(
|
|||
return "\n".join(
|
||||
text
|
||||
for block in system
|
||||
if isinstance(block, dict)
|
||||
if isinstance(block, Mapping)
|
||||
and block.get("type") == "text"
|
||||
and isinstance(text := block.get("text"), str)
|
||||
and text
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.types.utils import (
|
|||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
convert_to_model_response_object,
|
||||
get_base_model_from_metadata,
|
||||
)
|
||||
|
||||
from ...types.llms.openai import HttpxBinaryResponseContent
|
||||
|
|
@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
azure_client: AzureOpenAI | OpenAI,
|
||||
data: dict,
|
||||
timeout: float | httpx.Timeout,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
):
|
||||
"""
|
||||
Helper to:
|
||||
|
|
@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
except BadRequestError as e:
|
||||
if not is_output_token_limit_error(e):
|
||||
raise
|
||||
return build_output_token_limit_response(e=e, data=data, is_async=False)
|
||||
return build_output_token_limit_response(
|
||||
e=e,
|
||||
data=data,
|
||||
is_async=False,
|
||||
base_model=get_base_model_from_metadata(logging_obj.model_call_details),
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
except BadRequestError as e:
|
||||
if not is_output_token_limit_error(e):
|
||||
raise
|
||||
return build_output_token_limit_response(e=e, data=data, is_async=True)
|
||||
return build_output_token_limit_response(
|
||||
e=e,
|
||||
data=data,
|
||||
is_async=True,
|
||||
base_model=get_base_model_from_metadata(logging_obj.model_call_details),
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
)
|
||||
|
||||
headers, response = self.make_sync_azure_openai_chat_completion_request(
|
||||
azure_client=azure_client, data=data, timeout=timeout
|
||||
azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
|
||||
)
|
||||
if not isinstance(response, BaseModel):
|
||||
raise AzureOpenAIError(
|
||||
|
|
@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
},
|
||||
)
|
||||
headers, response = self.make_sync_azure_openai_chat_completion_request(
|
||||
azure_client=azure_client, data=data, timeout=timeout
|
||||
azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
|
||||
)
|
||||
logging_obj.model_call_details["response_headers"] = headers
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.token_counter import token_counter
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
|
||||
|
|
@ -242,7 +242,7 @@ async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletio
|
|||
|
||||
|
||||
def build_output_token_limit_response(
|
||||
e: openai.BadRequestError, data: Mapping[str, object], is_async: bool
|
||||
e: openai.BadRequestError, data: Mapping[str, object], is_async: bool, base_model: str | None
|
||||
) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]:
|
||||
"""Synthesize the length-truncated response the provider itself returns for slightly larger output budgets.
|
||||
|
||||
|
|
@ -251,7 +251,11 @@ def build_output_token_limit_response(
|
|||
"""
|
||||
model: Final[str] = str(data.get("model", ""))
|
||||
messages: Final = data.get("messages")
|
||||
prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0
|
||||
prompt_tokens: Final = (
|
||||
token_counter(model=usage_tokenizer_model(model, base_model), messages=messages)
|
||||
if isinstance(messages, list)
|
||||
else 0
|
||||
)
|
||||
if not data.get("stream"):
|
||||
return e.response.headers, _output_token_limit_completion(model, prompt_tokens)
|
||||
chunk: Final = _output_token_limit_chunk(model)
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from litellm.utils import (
|
|||
CustomStreamWrapper,
|
||||
ProviderConfigManager,
|
||||
convert_to_model_response_object,
|
||||
get_base_model_from_metadata,
|
||||
)
|
||||
|
||||
from ...types.llms.openai import *
|
||||
|
|
@ -557,7 +558,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
except openai.BadRequestError as e:
|
||||
if not is_output_token_limit_error(e):
|
||||
raise
|
||||
return build_output_token_limit_response(e=e, data=data, is_async=True)
|
||||
return build_output_token_limit_response(
|
||||
e=e,
|
||||
data=data,
|
||||
is_async=True,
|
||||
base_model=get_base_model_from_metadata(logging_obj.model_call_details),
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
|
@ -604,7 +610,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
except openai.BadRequestError as e:
|
||||
if not is_output_token_limit_error(e):
|
||||
raise
|
||||
return build_output_token_limit_response(e=e, data=data, is_async=False)
|
||||
return build_output_token_limit_response(
|
||||
e=e,
|
||||
data=data,
|
||||
is_async=False,
|
||||
base_model=get_base_model_from_metadata(logging_obj.model_call_details),
|
||||
)
|
||||
except Exception as e:
|
||||
if raw_response is not None:
|
||||
raise Exception(
|
||||
|
|
|
|||
|
|
@ -1218,7 +1218,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
|
||||
if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
|
||||
return frozenset()
|
||||
if _drops_params(litellm_params):
|
||||
if drops_params(litellm_params):
|
||||
return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
|
||||
return _COMPACT_TRACKING_PARAMS
|
||||
|
||||
|
|
|
|||
|
|
@ -28071,7 +28071,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
|
|
@ -28345,7 +28345,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"fireworks_ai/qwen3p7-plus": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
|
|
|
|||
|
|
@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase):
|
|||
end_time: datetime | None = None
|
||||
|
||||
|
||||
def proxy_exception_status_code(message: str, code: str) -> str:
|
||||
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
|
||||
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
|
||||
if "No healthy deployment available" in message or "No deployments available" in message:
|
||||
return "429"
|
||||
if RouterErrors.no_deployments_with_tag_routing.value in message:
|
||||
return "401"
|
||||
return code
|
||||
|
||||
|
||||
class ProxyException(Exception):
|
||||
# NOTE: DO NOT MODIFY THIS
|
||||
# This is used to map exactly to OPENAI Exceptions
|
||||
|
|
@ -3844,13 +3854,7 @@ class ProxyException(Exception):
|
|||
headers[k] = str(v)
|
||||
self.headers = headers or {}
|
||||
self.provider_specific_fields = provider_specific_fields
|
||||
# rules for proxyExceptions
|
||||
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
|
||||
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
|
||||
if "No healthy deployment available" in self.message or "No deployments available" in self.message:
|
||||
self.code = "429"
|
||||
elif RouterErrors.no_deployments_with_tag_routing.value in self.message:
|
||||
self.code = "401"
|
||||
self.code = proxy_exception_status_code(self.message, self.code)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Converts the ProxyException instance to a dictionary."""
|
||||
|
|
|
|||
|
|
@ -46,6 +46,11 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
extract_provider_resource_id_from_unified_id,
|
||||
is_base64_encoded_unified_id,
|
||||
)
|
||||
from litellm.models.project import LiteLLM_ProjectTable
|
||||
from litellm.proxy._types import (
|
||||
MODEL_LIST_ROUTES,
|
||||
|
|
@ -165,6 +170,7 @@ from litellm.types.proxy.auth.auth_checks import UserNotFoundError
|
|||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
from litellm.utils import get_utc_datetime
|
||||
from litellm.vector_stores.vector_store_registry import listed_vector_store_ids, vector_store_ids_in_tool
|
||||
|
||||
from .auth_checks_organization import (
|
||||
add_team_org_context_to_request_body,
|
||||
|
|
@ -1063,6 +1069,15 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) ->
|
|||
return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS
|
||||
|
||||
|
||||
class ModelMaxBudgetHolder(Protocol):
|
||||
@property
|
||||
def model_max_budget(self) -> Mapping[str, object] | None: ...
|
||||
|
||||
|
||||
def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] | None:
|
||||
return holder.model_max_budget
|
||||
|
||||
|
||||
def route_skips_budget_checks(route: str) -> bool:
|
||||
return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
|
||||
route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
|
||||
|
|
@ -1505,6 +1520,7 @@ async def common_checks(
|
|||
valid_token=valid_token,
|
||||
user_object=user_object,
|
||||
deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
# 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path)
|
||||
|
|
@ -7976,6 +7992,57 @@ def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tu
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UnreadableVectorStoreIds:
|
||||
pass
|
||||
|
||||
|
||||
def _top_level_vector_store_ids(
|
||||
vector_store_ids: object, drop_policies: Callable[[], tuple[DropPolicy, ...]]
|
||||
) -> tuple[str, ...] | _UnreadableVectorStoreIds:
|
||||
listed: Final = listed_vector_store_ids(vector_store_ids)
|
||||
if vector_store_ids is None or vector_store_ids == [*listed]:
|
||||
return listed
|
||||
if any(policy.drops("vector_store_ids") for policy in drop_policies()):
|
||||
return listed
|
||||
if isinstance(vector_store_ids, str):
|
||||
return (vector_store_ids,)
|
||||
return _UnreadableVectorStoreIds()
|
||||
|
||||
|
||||
def _legacy_requested_vector_store_ids(
|
||||
request_body: Mapping[str, object], drop_policies: Callable[[], tuple[DropPolicy, ...]]
|
||||
) -> tuple[str, ...] | _UnreadableVectorStoreIds:
|
||||
"""
|
||||
Reads tools the way VectorStoreRegistry does and leaves a malformed tool to the proxy's tool
|
||||
validation. A top-level vector_store_ids string names one store, since a query parameter always
|
||||
arrives as a string. Any other value that is not a list of strings names no readable stores unless a
|
||||
drop policy the request can resolve to drops it, and then only the string entries of a list name
|
||||
stores, as in registry retrieval.
|
||||
"""
|
||||
top_level: Final = _top_level_vector_store_ids(request_body.get("vector_store_ids"), drop_policies)
|
||||
if isinstance(top_level, _UnreadableVectorStoreIds):
|
||||
return top_level
|
||||
return tuple(
|
||||
chain(
|
||||
top_level,
|
||||
chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None]]) -> bool:
|
||||
return any(grant is not None and bool(grant.vector_stores) for _, grant in grants)
|
||||
|
||||
|
||||
def _request_drop_policies(
|
||||
request_body: Mapping[str, object], llm_router: Router | None, valid_token: UserAPIKeyAuth | None
|
||||
) -> tuple[DropPolicy, ...]:
|
||||
from litellm.proxy.common_request_processing import counted_drop_policies
|
||||
|
||||
return counted_drop_policies(request_body, llm_router, valid_token.team_id if valid_token is not None else None)
|
||||
|
||||
|
||||
def _require_vector_store_grant(
|
||||
object_type: GrantLayer,
|
||||
vector_store_ids_to_run: Sequence[str],
|
||||
|
|
@ -8023,6 +8090,7 @@ async def vector_store_access_check(
|
|||
*,
|
||||
user_object: LiteLLM_UserTable | None = None,
|
||||
deny_by_default: bool = False,
|
||||
llm_router: Router | None = None,
|
||||
):
|
||||
"""
|
||||
Checks whether the caller may use every vector store the request names.
|
||||
|
|
@ -8031,11 +8099,19 @@ async def vector_store_access_check(
|
|||
`retrieval_config.vector_store_id`. Grants come from each identity's
|
||||
`object_permission.vector_stores`.
|
||||
|
||||
With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
|
||||
nonempty, and stores are only read from the request when a vector store registry is loaded.
|
||||
Both modes read the same request fields, with or without a vector store registry, and answer 400
|
||||
when the top-level `vector_store_ids` is not a list of strings, which legacy mode does only when the
|
||||
key or team has an allowlist and the value is not a string, which it checks as one store id. A
|
||||
LiteLLM-managed store id is compared by the provider store id it decodes to, on both sides. Legacy mode skips that value instead when the request, `litellm_settings`, the router defaults or a deployment of the requested
|
||||
model drops it (`drop_params`, or `additional_drop_params` naming it), the same settings the proxy's
|
||||
tool validation reads, and reads tools the way the registry does, where strict mode answers 400 on a
|
||||
malformed one.
|
||||
|
||||
With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read
|
||||
even without a registry, and a missing record, `null` or `[]` grants nothing:
|
||||
With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
|
||||
nonempty.
|
||||
|
||||
With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), a missing record,
|
||||
`null` or `[]` grants nothing:
|
||||
|
||||
- virtual key: the key must grant every store, and so must its team when it has one, even if
|
||||
the team failed to load
|
||||
|
|
@ -8047,8 +8123,8 @@ async def vector_store_access_check(
|
|||
rescue nor restrict a key or team request.
|
||||
|
||||
Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity
|
||||
that does not grant a requested store, and with the flag on, ProxyException (400,
|
||||
`invalid_request_error`) when a `vector_store_ids` field is not a list of strings.
|
||||
that does not grant a requested store, and ProxyException (400, `invalid_request_error`) on a
|
||||
malformed `vector_store_ids` the request does not drop and, in legacy mode, an allowlist applies to.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
|
||||
|
||||
|
|
@ -8059,21 +8135,23 @@ async def vector_store_access_check(
|
|||
verbose_proxy_logger.debug("Prisma client not found, skipping vector store access check")
|
||||
return True
|
||||
|
||||
registry_ids: Final = (
|
||||
_strict_requested_vector_store_ids(_typed_request_body(request_body))
|
||||
typed_request_body: Final = _typed_request_body(request_body)
|
||||
requested_ids: Final = (
|
||||
_strict_requested_vector_store_ids(typed_request_body)
|
||||
if deny_by_default
|
||||
else (
|
||||
litellm.vector_store_registry.get_vector_store_ids_to_run(
|
||||
non_default_params=request_body, tools=request_body.get("tools", None)
|
||||
)
|
||||
if litellm.vector_store_registry is not None
|
||||
else None
|
||||
else _legacy_requested_vector_store_ids(
|
||||
typed_request_body, lambda: _request_drop_policies(typed_request_body, llm_router, valid_token)
|
||||
)
|
||||
or ()
|
||||
)
|
||||
rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body))
|
||||
if isinstance(requested_ids, _UnreadableVectorStoreIds):
|
||||
if _restricts_vector_stores(
|
||||
await _identity_grants(valid_token, team_object, None, prisma_client, user_api_key_cache)
|
||||
):
|
||||
raise _malformed_vector_store_ids()
|
||||
return True
|
||||
rag_vector_store_id: Final = _get_rag_query_vector_store_id(typed_request_body)
|
||||
rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
|
||||
vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
|
||||
vector_store_ids_to_run: Final = tuple(dict.fromkeys((*requested_ids, *rag_ids)))
|
||||
if not vector_store_ids_to_run:
|
||||
return True
|
||||
|
||||
|
|
@ -8100,6 +8178,12 @@ async def vector_store_access_check(
|
|||
return True
|
||||
|
||||
|
||||
def _provider_vector_store_id(vector_store_id: str) -> str:
|
||||
if not is_base64_encoded_unified_id(vector_store_id):
|
||||
return vector_store_id
|
||||
return extract_provider_resource_id_from_unified_id(vector_store_id) or vector_store_id
|
||||
|
||||
|
||||
def _can_object_call_vector_stores(
|
||||
object_type: Literal["key", "team", "org", "user"],
|
||||
vector_store_ids_to_run: Sequence[str],
|
||||
|
|
@ -8118,8 +8202,9 @@ def _can_object_call_vector_stores(
|
|||
if len(object_permissions.vector_stores) == 0:
|
||||
return True
|
||||
|
||||
allowed_stores: Final = frozenset(map(_provider_vector_store_id, object_permissions.vector_stores))
|
||||
for vector_store_id in vector_store_ids_to_run:
|
||||
if vector_store_id not in object_permissions.vector_stores:
|
||||
if _provider_vector_store_id(vector_store_id) not in allowed_stores:
|
||||
raise ProxyException(
|
||||
message=f"User not allowed to access vector store. Tried to access {vector_store_id}. Only allowed to access {object_permissions.vector_stores}",
|
||||
type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type),
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
|||
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
|
||||
from litellm.models.end_user import LiteLLM_EndUserTable
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
|
||||
from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports
|
||||
|
|
@ -85,6 +86,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
jwt_key_mapping_cache_key,
|
||||
key_access_group_grants_model,
|
||||
key_model_aliases_for_auth_check,
|
||||
model_max_budget_of,
|
||||
resolve_and_validate_end_user_id,
|
||||
resolve_default_end_user_budget,
|
||||
route_skips_budget_checks,
|
||||
|
|
@ -384,6 +386,16 @@ class _TeamModelBudgetLimiter(Protocol):
|
|||
) -> bool: ...
|
||||
|
||||
|
||||
class _EndUserModelBudgetLimiter(Protocol):
|
||||
async def is_end_user_within_model_budget(
|
||||
self,
|
||||
end_user_id: str,
|
||||
end_user_model_max_budget: Mapping[str, object],
|
||||
model: str,
|
||||
team_id: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class _EveryModelBudgetLimiter(Protocol):
|
||||
async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
|
||||
|
||||
|
|
@ -530,6 +542,49 @@ async def _check_team_model_budget(
|
|||
)
|
||||
|
||||
|
||||
async def _end_user_model_max_budget(
|
||||
end_user_id: str | None,
|
||||
end_user_object: LiteLLM_EndUserTable | None,
|
||||
key_end_user_budget_id: str | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
parent_otel_span: Span | None,
|
||||
) -> Mapping[str, object] | None:
|
||||
budget_table: Final = (
|
||||
end_user_object.litellm_budget_table
|
||||
if end_user_object is not None
|
||||
else (
|
||||
await resolve_default_end_user_budget(
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
key_end_user_budget_id=key_end_user_budget_id,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if end_user_id and prisma_client is not None
|
||||
else None
|
||||
)
|
||||
)
|
||||
return model_max_budget_of(budget_table) if budget_table is not None else None
|
||||
|
||||
|
||||
async def _check_end_user_model_budget(
|
||||
end_user_id: str | None,
|
||||
end_user_model_max_budget: Mapping[str, object] | None,
|
||||
model_max_budget_limiter: _EndUserModelBudgetLimiter,
|
||||
models: Sequence[str],
|
||||
team_id: str | None,
|
||||
) -> None:
|
||||
if not end_user_id or not end_user_model_max_budget:
|
||||
return
|
||||
for model_name in models:
|
||||
await model_max_budget_limiter.is_end_user_within_model_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_model_max_budget=end_user_model_max_budget,
|
||||
model=model_name,
|
||||
team_id=team_id,
|
||||
)
|
||||
|
||||
|
||||
async def _is_within_every_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _EveryModelBudgetLimiter,
|
||||
|
|
@ -2094,6 +2149,10 @@ async def _resolve_object_permission_for_unresolvable_team(
|
|||
)
|
||||
|
||||
|
||||
def _custom_auth_token_for_this_request(response: object) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth.model_validate(response).model_copy()
|
||||
|
||||
|
||||
async def user_api_key_auth_builder(
|
||||
request: Request,
|
||||
api_key: str,
|
||||
|
|
@ -2172,7 +2231,7 @@ async def user_api_key_auth_builder(
|
|||
request=request, api_key=api_key, user_custom_auth=user_custom_auth
|
||||
)
|
||||
if response is not None and isinstance(response, UserAPIKeyAuth):
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
validated = _custom_auth_token_for_this_request(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
validated = await _run_post_custom_auth_checks(
|
||||
valid_token=validated,
|
||||
|
|
@ -2188,7 +2247,7 @@ async def user_api_key_auth_builder(
|
|||
custom_auth_api_key = True
|
||||
elif user_custom_auth is not None:
|
||||
response = await user_custom_auth(request=request, api_key=api_key)
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
validated = _custom_auth_token_for_this_request(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
validated = await _run_post_custom_auth_checks(
|
||||
valid_token=validated,
|
||||
|
|
@ -3129,7 +3188,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
)
|
||||
|
||||
# Recompute after a potential budget-fallback rewrite so
|
||||
# the end-user check below validates the final model
|
||||
# the user check below validates the final model
|
||||
current_model = _get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
|
|
@ -3147,23 +3206,6 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
models=current_models,
|
||||
)
|
||||
|
||||
# Check 5b. End-user model max budget
|
||||
end_user_mmb: Final = valid_token.end_user_model_max_budget
|
||||
if (
|
||||
end_user_mmb is not None
|
||||
and isinstance(end_user_mmb, dict)
|
||||
and len(end_user_mmb) > 0
|
||||
and current_models
|
||||
and valid_token.end_user_id is not None
|
||||
):
|
||||
for model_name in current_models:
|
||||
await model_max_budget_limiter.is_end_user_within_model_budget(
|
||||
end_user_id=valid_token.end_user_id,
|
||||
end_user_model_max_budget=end_user_mmb,
|
||||
model=model_name,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
|
||||
# Check 6: Additional Common Checks across jwt + key auth
|
||||
if valid_token.team_id is not None:
|
||||
try:
|
||||
|
|
@ -3733,9 +3775,7 @@ async def run_centralized_common_checks(
|
|||
project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (
|
||||
None if isinstance(project_result, BaseException) else project_result
|
||||
)
|
||||
end_user_object: Final[LiteLLM_EndUserTable | None] = (
|
||||
None if isinstance(end_user_result, BaseException) else end_user_result
|
||||
)
|
||||
end_user_object: Final = end_user_result if isinstance(end_user_result, LiteLLM_EndUserTable) else None
|
||||
global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result
|
||||
carry_team_and_user_budget_state(
|
||||
valid_token=user_api_key_auth_obj,
|
||||
|
|
@ -3795,6 +3835,21 @@ async def run_centralized_common_checks(
|
|||
parent_otel_span=parent_otel_span,
|
||||
keep_token_limits=user_custom_auth is not None,
|
||||
)
|
||||
end_user_model_max_budget: Final = await _end_user_model_max_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_object=end_user_object,
|
||||
key_end_user_budget_id=key_end_user_budget_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if end_user_id and (
|
||||
not user_api_key_auth_obj.authenticated_by_custom_auth
|
||||
or user_api_key_auth_obj.end_user_model_max_budget is None
|
||||
):
|
||||
user_api_key_auth_obj.end_user_model_max_budget = ( # rebind-ok: the spend hook charges from this token
|
||||
dict(end_user_model_max_budget) if end_user_model_max_budget else None
|
||||
)
|
||||
|
||||
skip_budget_checks: Final = _should_skip_budget_checks(
|
||||
request_data=request_data,
|
||||
|
|
@ -3887,6 +3942,21 @@ async def run_centralized_common_checks(
|
|||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await _check_end_user_model_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_model_max_budget=user_api_key_auth_obj.end_user_model_max_budget,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
team_id=user_api_key_auth_obj.team_id,
|
||||
models=_get_model_names_for_budget_checks(
|
||||
model=_get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
team_id=user_api_key_auth_obj.team_id,
|
||||
)
|
||||
),
|
||||
)
|
||||
if sub_call and RouteChecks.is_llm_api_route(route=route):
|
||||
await virtual_key_max_budget_check(
|
||||
valid_token=user_api_key_auth_obj,
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, TypeVar
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import click
|
||||
import requests
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CODEX_BASE_INSTRUCTIONS_PATH,
|
||||
|
|
@ -497,6 +499,113 @@ def _fetch_model_listing(
|
|||
return listing.data
|
||||
|
||||
|
||||
class _DeploymentParams(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model: str
|
||||
custom_llm_provider: str | None = None
|
||||
api_base: str | None = None
|
||||
litellm_credential_name: str | None = None
|
||||
reasoning_effort: JsonValue = None
|
||||
|
||||
|
||||
class _DeploymentInfo(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
team_public_model_name: str | None = None
|
||||
blocked: bool | None = None
|
||||
|
||||
|
||||
class _Deployment(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model_name: str
|
||||
litellm_params: _DeploymentParams
|
||||
model_info: _DeploymentInfo | None = None
|
||||
|
||||
|
||||
class _DeploymentListing(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_Deployment, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelUpstream:
|
||||
"""The OpenAI model every deployment behind a public name routes to, and the reasoning effort they all pin."""
|
||||
|
||||
model: str
|
||||
reasoning_effort: str | None = None
|
||||
|
||||
|
||||
_NO_UPSTREAMS: Final[Mapping[str, ModelUpstream]] = MappingProxyType({})
|
||||
_Pinned: Final = TypeVar("_Pinned", bound=Hashable)
|
||||
|
||||
|
||||
def _public_name(deployment: _Deployment) -> str:
|
||||
team_name: Final = deployment.model_info.team_public_model_name if deployment.model_info is not None else None
|
||||
return team_name or deployment.model_name
|
||||
|
||||
|
||||
def _is_paused(deployment: _Deployment) -> bool:
|
||||
return deployment.model_info is not None and deployment.model_info.blocked is True
|
||||
|
||||
|
||||
def _is_openai_host(params: _DeploymentParams) -> bool:
|
||||
if params.api_base is None:
|
||||
return not params.litellm_credential_name
|
||||
host: Final = urlsplit(params.api_base).hostname or ""
|
||||
return host == "api.openai.com" or host.endswith(".api.openai.com")
|
||||
|
||||
|
||||
def _openai_model(params: _DeploymentParams) -> str | None:
|
||||
if params.custom_llm_provider not in (None, "openai") or not _is_openai_host(params):
|
||||
return None
|
||||
provider, slash, name = params.model.partition("/")
|
||||
if not slash:
|
||||
return params.model
|
||||
return name if provider == "openai" else None
|
||||
|
||||
|
||||
def _pinned_effort(params: _DeploymentParams) -> str | None:
|
||||
return params.reasoning_effort if isinstance(params.reasoning_effort, str) else None
|
||||
|
||||
|
||||
def _shared_by_every_deployment(values: Iterable[tuple[str, _Pinned]]) -> Mapping[str, _Pinned]:
|
||||
distinct: Final = frozenset(values)
|
||||
values_per_name: Final = Counter(name for name, _ in distinct)
|
||||
return MappingProxyType({name: value for name, value in distinct if values_per_name[name] == 1})
|
||||
|
||||
|
||||
def _upstreams_by_public_name(deployments: Sequence[_Deployment]) -> Mapping[str, ModelUpstream]:
|
||||
live: Final = tuple(d for d in deployments if not _is_paused(d))
|
||||
models: Final = _shared_by_every_deployment((_public_name(d), _openai_model(d.litellm_params)) for d in live)
|
||||
efforts: Final = _shared_by_every_deployment((_public_name(d), _pinned_effort(d.litellm_params)) for d in live)
|
||||
return MappingProxyType(
|
||||
{name: ModelUpstream(model, efforts.get(name)) for name, model in models.items() if model is not None}
|
||||
)
|
||||
|
||||
|
||||
def _fetch_model_upstreams(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
*,
|
||||
get: Callable[..., requests.Response],
|
||||
) -> Mapping[str, ModelUpstream]:
|
||||
url: Final = base_url.rstrip("/") + "/v1/model/info"
|
||||
try:
|
||||
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
|
||||
except requests.RequestException:
|
||||
return _NO_UPSTREAMS
|
||||
if resp.status_code != 200:
|
||||
return _NO_UPSTREAMS
|
||||
try:
|
||||
listing: Final = _DeploymentListing.model_validate_json(resp.content)
|
||||
except ValidationError:
|
||||
return _NO_UPSTREAMS
|
||||
return _upstreams_by_public_name(listing.data)
|
||||
|
||||
|
||||
class _OpenCodeLimit(LiteLLMBaseModel):
|
||||
context: int
|
||||
output: int
|
||||
|
|
@ -576,8 +685,21 @@ def opencode_model_sync_env(
|
|||
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
|
||||
|
||||
|
||||
def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
|
||||
return CodexCatalogRow(
|
||||
id=model.id,
|
||||
mode=model.mode,
|
||||
max_input_tokens=model.max_input_tokens,
|
||||
upstream_model=upstream.model if upstream is not None else None,
|
||||
reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def codex_model_catalog(
|
||||
models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
|
||||
models: Sequence[ListedModel],
|
||||
stock: Sequence[CodexStockModel],
|
||||
instructions: str,
|
||||
upstreams: Mapping[str, ModelUpstream],
|
||||
) -> str | None:
|
||||
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
|
||||
|
||||
|
|
@ -586,14 +708,23 @@ def codex_model_catalog(
|
|||
installed Codex knows keeps that Codex's own entry and the proxy only
|
||||
decides its place in the picker: the listing orders it, lists it even when
|
||||
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
|
||||
when the model it points at is served too. A model Codex does not know gets the fallback
|
||||
when the model it points at is served too. An alias of such a model, per
|
||||
`upstreams`, takes that entry too, minus the defaults Codex picks for the
|
||||
model: no service tier or upgrade nudge, the reasoning effort its
|
||||
deployments pin, and a window no larger than its listed max_input_tokens.
|
||||
A model Codex does not know gets the fallback
|
||||
entry, with the same base instructions Codex itself uses so the agent never
|
||||
runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
|
||||
serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
|
||||
runs without a system prompt. Apart from aliases, the catalog is the one
|
||||
`/v1/models?client_version=...` serves, built from the installed Codex's own
|
||||
entries instead of the proxy's vendored ones.
|
||||
"""
|
||||
rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
|
||||
rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
|
||||
body: Final = codex_models_response_json(
|
||||
rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
|
||||
rows,
|
||||
stock=MappingProxyType({m.slug: m for m in stock}),
|
||||
instructions=instructions,
|
||||
byte_limit=None,
|
||||
alias_defaults="deployment",
|
||||
)
|
||||
return body.json if body.listed else None
|
||||
|
||||
|
|
@ -678,12 +809,14 @@ def codex_model_sync_args(
|
|||
Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
|
||||
must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
|
||||
atomically replaced on every launch. The Codex at `binary` first lists its
|
||||
own models, so the ones the proxy serves keep that Codex's entries, and then
|
||||
own models, so the ones the proxy serves keep that Codex's entries, an alias
|
||||
included when every unpaused deployment /v1/model/info lists for it routes
|
||||
to the same OpenAI model and Codex knows that model, and then
|
||||
reads the file back once before it is handed over. The key never lands in
|
||||
the file. A failed fetch, read, listing, write or read-back is reported
|
||||
rather than raised: Codex still launches with its built-in catalog and takes
|
||||
a proxy model by name via -m, and a rejected file stays on disk to be looked
|
||||
at.
|
||||
at. An unavailable /v1/model/info only leaves every alias on the fallback entry.
|
||||
"""
|
||||
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
|
||||
if isinstance(listing, ModelSyncSkipped):
|
||||
|
|
@ -700,7 +833,8 @@ def codex_model_sync_args(
|
|||
stock: Final = _stock_codex_models(binary, base_env, run=run)
|
||||
if isinstance(stock, ModelSyncSkipped):
|
||||
return stock
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions)
|
||||
upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
|
||||
if catalog is None:
|
||||
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -724,7 +724,7 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
|
|||
return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
|
||||
|
||||
|
||||
def _counted_drop_policies(
|
||||
def counted_drop_policies(
|
||||
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
|
||||
) -> tuple[DropPolicy, ...]:
|
||||
requested: Final = {**_router_defaults(llm_router), **data}
|
||||
|
|
@ -740,7 +740,7 @@ def _counted_drop_policies(
|
|||
def resolve_counted_tool_fields(
|
||||
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
|
||||
) -> Mapping[str, object] | ProxyException:
|
||||
resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
|
||||
resolved: Final = resolve_tool_fields(data, lambda: counted_drop_policies(data, llm_router, team_id))
|
||||
if isinstance(resolved, MalformedField):
|
||||
return ProxyException(
|
||||
message=resolved.message,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from functools import cache
|
|||
from itertools import accumulate, pairwise
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType, NoneType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -110,6 +110,12 @@ class CodexStockUpgrade(LiteLLMBaseModel):
|
|||
model: str
|
||||
|
||||
|
||||
class CodexReasoningLevel(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
effort: str
|
||||
|
||||
|
||||
class CodexStockModel(LiteLLMBaseModel):
|
||||
"""One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
|
||||
|
||||
|
|
@ -128,6 +134,8 @@ class CodexStockModel(LiteLLMBaseModel):
|
|||
upgrade: CodexStockUpgrade | None = None
|
||||
service_tiers: tuple[CodexServiceTier, ...] = ()
|
||||
default_service_tier: str | None = None
|
||||
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
|
||||
context_window: int | None = None
|
||||
|
||||
|
||||
class CodexStockCatalog(LiteLLMBaseModel):
|
||||
|
|
@ -135,19 +143,21 @@ class CodexStockCatalog(LiteLLMBaseModel):
|
|||
|
||||
|
||||
CodexCatalogEntry = CodexFallbackModel | CodexStockModel
|
||||
CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCatalogRow:
|
||||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it, and the `model_info` values the operator set;
|
||||
`service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
|
||||
where a deployment sets none."""
|
||||
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
|
||||
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
|
||||
value per deployment behind the id, None where a deployment sets none."""
|
||||
|
||||
id: str
|
||||
mode: str | None = None
|
||||
max_input_tokens: int | None = None
|
||||
upstream_model: str | None = None
|
||||
reasoning_effort: str | None = None
|
||||
display_name: str | None = None
|
||||
service_tiers: tuple[object, ...] = ()
|
||||
|
||||
|
|
@ -254,12 +264,42 @@ def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> Co
|
|||
return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
|
||||
|
||||
|
||||
def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
|
||||
if row.max_input_tokens is None or stock_model.context_window is None:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
|
||||
|
||||
|
||||
def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
|
||||
efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
|
||||
return row.reasoning_effort if row.reasoning_effort in efforts else None
|
||||
|
||||
|
||||
def _deployment_defaults(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]:
|
||||
"""What an alias's deployments decide in place of the defaults Codex picks for the model they route to:
|
||||
no service tier, verbosity, upgrade nudge or startup promo of Codex's own, so what the deployment
|
||||
pins (or OpenAI's API default) stays in force, the pinned reasoning effort when Codex offers it and
|
||||
none otherwise, and a window no larger than the alias's own limit."""
|
||||
return MappingProxyType(
|
||||
{
|
||||
"service_tiers": (),
|
||||
"default_service_tier": None,
|
||||
"default_verbosity": None,
|
||||
"upgrade": None,
|
||||
"availability_nux": None,
|
||||
"default_reasoning_level": _alias_reasoning_level(row, stock_model),
|
||||
**_alias_context_window(row, stock_model),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _entry(
|
||||
index: int,
|
||||
row: CodexCatalogRow,
|
||||
stock: Mapping[str, CodexStockModel],
|
||||
served: frozenset[str],
|
||||
instructions: str,
|
||||
alias_defaults: CodexAliasDefaults,
|
||||
) -> CodexCatalogEntry:
|
||||
stock_model: Final = _stock_for(row, stock)
|
||||
stock_tiers: Final = MappingProxyType(
|
||||
|
|
@ -278,7 +318,8 @@ def _entry(
|
|||
upgrade: Final = (
|
||||
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
|
||||
)
|
||||
stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
|
||||
is_alias: Final = row.id != stock_model.slug
|
||||
stock_name: Final = row.id if is_alias else stock_model.display_name
|
||||
listing: Final = {
|
||||
"slug": row.id,
|
||||
"display_name": row.display_name if row.display_name is not None else stock_name,
|
||||
|
|
@ -286,6 +327,7 @@ def _entry(
|
|||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": upgrade,
|
||||
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
|
||||
}
|
||||
if configured is None:
|
||||
return stock_model.model_copy(update=listing)
|
||||
|
|
@ -351,6 +393,7 @@ def codex_models_response_json(
|
|||
stock: Mapping[str, CodexStockModel] | None = None,
|
||||
instructions: str | None = None,
|
||||
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
|
||||
alias_defaults: CodexAliasDefaults = "codex",
|
||||
) -> CodexCatalogBody:
|
||||
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
|
||||
|
||||
|
|
@ -359,13 +402,18 @@ def codex_models_response_json(
|
|||
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
the ones offering a service tier taken first and the rest in listing order, an entry too large
|
||||
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
|
||||
keeps every entry.
|
||||
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
|
||||
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
|
||||
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
|
||||
them, as Codex would use them for the model's own name.
|
||||
"""
|
||||
known: Final = bundled_codex_models() if stock is None else stock
|
||||
prompt: Final = codex_base_instructions() if instructions is None else instructions
|
||||
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
|
||||
served: Final = frozenset(row.id for row in drivable)
|
||||
entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
|
||||
entries: Final = tuple(
|
||||
_entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
|
||||
)
|
||||
serialized: Final = tuple(_serialized(entry) for entry in entries)
|
||||
kept: Final = _kept_positions(entries, serialized, byte_limit)
|
||||
return CodexCatalogBody(
|
||||
|
|
|
|||
|
|
@ -2,14 +2,17 @@
|
|||
``type`` is a required string and ``param`` is nullable, neither of which the literal
|
||||
string ``"None"`` satisfies."""
|
||||
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from fastapi import status
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from litellm.constants import STRINGIFIED_NONE
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
|
||||
|
||||
|
|
@ -21,6 +24,19 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
|
|||
}
|
||||
)
|
||||
|
||||
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
|
||||
|
||||
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
|
||||
|
||||
_FAILOVER_REFUSALS: Final = (
|
||||
"is not currently accepting connections",
|
||||
"the database system is starting up",
|
||||
"the database system is shutting down",
|
||||
"the database system is in recovery mode",
|
||||
"the database system is not yet accepting connections",
|
||||
"terminating connection due to administrator command",
|
||||
)
|
||||
|
||||
|
||||
def attribute_of(value: object, name: str, default: object = None) -> object:
|
||||
return getattr(value, name, default)
|
||||
|
|
@ -61,6 +77,82 @@ def openai_error_param(exc: object) -> str | None:
|
|||
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
|
||||
|
||||
|
||||
def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
|
||||
match exc:
|
||||
case ProxyException():
|
||||
return exc
|
||||
case _ if _is_database_outage(exc):
|
||||
return ProxyException(
|
||||
message=PrismaDBExceptionHandler.database_unavailable_message(exc),
|
||||
type=ProxyErrorTypes.no_db_connection.value,
|
||||
param=None,
|
||||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
case HTTPException(status_code=status_code, detail=detail):
|
||||
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
|
||||
case _:
|
||||
return _openai_shaped_proxy_exception(
|
||||
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
|
||||
)
|
||||
|
||||
|
||||
def _is_database_outage(exc: Exception) -> bool:
|
||||
return any(map(_is_database_outage_link, exception_chain(exc)))
|
||||
|
||||
|
||||
def _is_database_outage_link(exc: BaseException) -> bool:
|
||||
match exc:
|
||||
case DBLookupDeadlineExceeded():
|
||||
return True
|
||||
case Exception() if _prisma_is_loaded():
|
||||
return _is_prisma_outage(exc)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _prisma_is_loaded() -> bool:
|
||||
return sys.modules.get("prisma") is not None
|
||||
|
||||
|
||||
def _is_prisma_outage(exc: Exception) -> bool:
|
||||
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc):
|
||||
return True
|
||||
if type(exc).__module__.partition(".")[0] != "prisma":
|
||||
return False
|
||||
return PrismaDBExceptionHandler.is_database_service_unavailable_error(exc) or _is_failover_refusal(exc)
|
||||
|
||||
|
||||
def _is_failover_refusal(exc: Exception) -> bool:
|
||||
if PrismaDBExceptionHandler.is_read_only_transaction_error(exc):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.postgres_sqlstate(exc) in _FAILOVER_SQLSTATES:
|
||||
return True
|
||||
message: Final = str(exc).lower()
|
||||
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
|
||||
|
||||
|
||||
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=_catch_all_error_type(exc, answered_status),
|
||||
param=openai_error_param(exc),
|
||||
code=answered_status,
|
||||
)
|
||||
|
||||
|
||||
def _catch_all_error_type(exc: Exception, status_code: int) -> str:
|
||||
if _rejects_the_caller(exc, status_code):
|
||||
return ProxyErrorTypes.auth_error.value
|
||||
return openai_error_type(exc, status_code)
|
||||
|
||||
|
||||
def _rejects_the_caller(exc: Exception, status_code: int) -> bool:
|
||||
raised_as_http_answer: Final = isinstance(exc, HTTPException)
|
||||
status_from_upstream: Final = error_status_code(exc, 0) == status_code
|
||||
return status_code in _AUTH_FAILURE_STATUS_CODES and (raised_as_http_answer or not status_from_upstream)
|
||||
|
||||
|
||||
def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None: # mutable-ok: ProxyException.headers
|
||||
if litellm_call_id is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ CRUD endpoints for storing reusable credentials.
|
|||
import time
|
||||
import unicodedata
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
Annotated,
|
||||
Final,
|
||||
|
|
@ -23,7 +24,7 @@ from litellm.constants import (
|
|||
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
|
||||
)
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
||||
from litellm.litellm_core_utils.litellm_logging import get_masked_values, has_sensitive_keyword
|
||||
from litellm.llms.anthropic.wif import (
|
||||
ExportedJwks,
|
||||
NotAnInternalIssuerCredential,
|
||||
|
|
@ -83,6 +84,11 @@ _CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
|||
_GITHUB_COPILOT_PROVIDER: Final = "github_copilot"
|
||||
_DISPLAY_NAME_MAX_LENGTH: Final = 255
|
||||
_DISPLAY_NAME_FORBIDDEN_CATEGORIES: Final = frozenset({"Cc", "Cf", "Cs", "Zl", "Zp"})
|
||||
_MASK_UNMASKED_LENGTH: Final = 4
|
||||
_MASK_ASTERISKS: Final = 4
|
||||
_SHORT_SECRET_MASK: Final = "*****"
|
||||
_MASKED_READ_BACK_DEPTH: Final = 20
|
||||
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _normalized_github_copilot_values(
|
||||
|
|
@ -395,7 +401,14 @@ async def get_credentials(
|
|||
"""
|
||||
try:
|
||||
masked_credentials: Final = [
|
||||
_credential_view(credential, get_masked_values(credential.credential_values))
|
||||
_credential_view(
|
||||
credential,
|
||||
get_masked_values(
|
||||
credential.credential_values,
|
||||
unmasked_length=_MASK_UNMASKED_LENGTH,
|
||||
number_of_asterisks=_MASK_ASTERISKS,
|
||||
),
|
||||
)
|
||||
for credential in litellm.credential_list
|
||||
]
|
||||
return {"success": True, "credentials": masked_credentials}
|
||||
|
|
@ -423,7 +436,11 @@ async def get_credential_by_name(
|
|||
if credential.credential_name == credential_name:
|
||||
return _credential_view(
|
||||
credential,
|
||||
get_masked_values(credential.credential_values, unmasked_length=4, number_of_asterisks=4),
|
||||
get_masked_values(
|
||||
credential.credential_values,
|
||||
unmasked_length=_MASK_UNMASKED_LENGTH,
|
||||
number_of_asterisks=_MASK_ASTERISKS,
|
||||
),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
|
|
@ -521,8 +538,8 @@ async def get_credential_by_model(
|
|||
raise HTTPException(status_code=404, detail="Model not found")
|
||||
masked_credential_values: Final = get_masked_values(
|
||||
credential_values,
|
||||
unmasked_length=4,
|
||||
number_of_asterisks=4,
|
||||
unmasked_length=_MASK_UNMASKED_LENGTH,
|
||||
number_of_asterisks=_MASK_ASTERISKS,
|
||||
)
|
||||
credential: Final = CredentialItem(
|
||||
credential_name=f"{model.model_name}-credential-{model_id}",
|
||||
|
|
@ -908,6 +925,48 @@ def update_db_credential(
|
|||
return merged_credential
|
||||
|
||||
|
||||
def _is_masked_read_back(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
if value == _SHORT_SECRET_MASK:
|
||||
return True
|
||||
shown: Final = _MASK_UNMASKED_LENGTH // 2
|
||||
return len(value) == _MASK_UNMASKED_LENGTH + _MASK_ASTERISKS and value[shown:-shown] == "*" * _MASK_ASTERISKS
|
||||
|
||||
|
||||
def _as_fields(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _CREDENTIAL_DICT_ADAPTER.validate_python(value, strict=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _kept_value(key: str, value: object, stored: Mapping[str, object], depth: int) -> tuple[str, object] | None:
|
||||
if not has_sensitive_keyword(key):
|
||||
return key, value
|
||||
fields: Final = _as_fields(value)
|
||||
if fields is not None and depth < _MASKED_READ_BACK_DEPTH:
|
||||
return key, _without_masked_read_backs(fields, _as_fields(stored.get(key)) or _NO_FIELDS, depth + 1)
|
||||
if not _is_masked_read_back(value):
|
||||
return key, value
|
||||
if key in stored:
|
||||
return key, stored[key]
|
||||
return None
|
||||
|
||||
|
||||
def _without_masked_read_backs(
|
||||
requested: Mapping[str, object], stored: Mapping[str, object], depth: int = 0
|
||||
) -> Mapping[str, object]:
|
||||
kept: Final = (_kept_value(key, value, stored, depth) for key, value in requested.items())
|
||||
return _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
MappingProxyType(dict(entry for entry in kept if entry is not None))
|
||||
)
|
||||
|
||||
|
||||
def _stored_objects(stored_values: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType({key: value for key, value in stored_values.items() if _as_fields(value) is not None})
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/credentials/{credential_name:path}",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
@ -936,12 +995,12 @@ async def update_credential(
|
|||
)
|
||||
requested_display_name: Final = _normalized_display_name(credential.display_name)
|
||||
_reject_overlapping_credential_values(credential)
|
||||
incoming_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
requested_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
_resolve_deployment_credentials(llm_router, credential.model_id)
|
||||
if credential.model_id
|
||||
else credential.credential_values or {}
|
||||
)
|
||||
_reject_non_admin_wif_fields(_incoming_wif_fields(incoming_values, credential), user_api_key_dict)
|
||||
_reject_non_admin_wif_fields(_incoming_wif_fields(requested_values, credential), user_api_key_dict)
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
|
|
@ -952,19 +1011,20 @@ async def update_credential(
|
|||
if db_credential is None:
|
||||
raise _not_found_unless_config_defined(credential_name, "Credential not found in DB.")
|
||||
_reject_non_admin_wif_fields(_stored_wif_fields(db_credential), user_api_key_dict)
|
||||
decrypted_stored_credential: Final = decrypted_credential(db_credential)
|
||||
decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
decrypted_stored_credential.credential_values
|
||||
)
|
||||
incoming_values: Final = _without_masked_read_backs(requested_values, _stored_objects(decrypted_values))
|
||||
patch: Final = CredentialItem(
|
||||
credential_name=credential_name,
|
||||
display_name=(
|
||||
requested_display_name if "display_name" in credential.model_fields_set else db_credential.display_name
|
||||
),
|
||||
credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info),
|
||||
credential_values=incoming_values,
|
||||
credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(incoming_values),
|
||||
credential_values_to_delete=credential.credential_values_to_delete,
|
||||
)
|
||||
decrypted_stored_credential: Final = decrypted_credential(db_credential)
|
||||
decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
decrypted_stored_credential.credential_values
|
||||
)
|
||||
stored_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
|
||||
db_credential.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ _LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
|
|||
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
|
||||
|
||||
|
||||
def _exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
def exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
current = e # rebind-ok: advances one link per iteration of the bounded walk
|
||||
for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
|
||||
yield current
|
||||
|
|
@ -49,7 +49,7 @@ def _exception_chain(e: BaseException) -> Iterator[BaseException]:
|
|||
def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]:
|
||||
return tuple(
|
||||
link
|
||||
for link in _exception_chain(e)
|
||||
for link in exception_chain(e)
|
||||
if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
from collections.abc import Mapping
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Annotated, Final
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, ORJSONResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
|
||||
from litellm.proxy._types import *
|
||||
|
|
@ -27,14 +27,67 @@ from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
|||
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
|
||||
|
||||
|
||||
def _google_repeated_field(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case None:
|
||||
return []
|
||||
case dict():
|
||||
return [value]
|
||||
case list():
|
||||
return [item for item in value if item is not None]
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _google_content(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case {"parts": parts}:
|
||||
return {**value, "parts": _google_repeated_field(parts)}
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _google_contents(value: JsonValue) -> JsonValue:
|
||||
match _google_repeated_field(value):
|
||||
case list() as contents:
|
||||
return [_google_content(content) for content in contents]
|
||||
case other:
|
||||
return other
|
||||
|
||||
|
||||
_GOOGLE_PART_TEXT_AND_METADATA_FIELDS: Final = frozenset(
|
||||
{"text", "thought", "thoughtSignature", "thought_signature", "partMetadata", "part_metadata"}
|
||||
)
|
||||
|
||||
|
||||
def _google_part_has_non_text_content(part: Mapping[str, JsonValue]) -> bool:
|
||||
return any(field is not None for key, field in part.items() if key not in _GOOGLE_PART_TEXT_AND_METADATA_FIELDS)
|
||||
|
||||
|
||||
def _google_system_part(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case dict() if value.get("text") is None and _google_part_has_non_text_content(value):
|
||||
return {key: field for key, field in value.items() if key != "text"}
|
||||
case dict():
|
||||
return {"text": None, **value}
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
_GOOGLE_REPEATED_FIELD: Final = BeforeValidator(_google_repeated_field)
|
||||
|
||||
|
||||
class _GoogleCountTokensTextPart(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
text: str
|
||||
text: str | None
|
||||
|
||||
|
||||
class _GoogleCountTokensSystemInstruction(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
parts: tuple[_GoogleCountTokensTextPart, ...]
|
||||
parts: Annotated[
|
||||
tuple[Annotated[_GoogleCountTokensTextPart, BeforeValidator(_google_system_part)], ...],
|
||||
_GOOGLE_REPEATED_FIELD,
|
||||
] = ()
|
||||
|
||||
|
||||
class _GoogleCountTokensFunctionDeclaration(BaseModel):
|
||||
|
|
@ -47,16 +100,16 @@ class _GoogleCountTokensFunctionDeclaration(BaseModel):
|
|||
|
||||
class _GoogleCountTokensTool(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, populate_by_name=True)
|
||||
function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field(
|
||||
function_declarations: Annotated[tuple[_GoogleCountTokensFunctionDeclaration, ...], _GOOGLE_REPEATED_FIELD] = Field(
|
||||
default=(), alias="functionDeclarations"
|
||||
)
|
||||
|
||||
|
||||
class _GoogleCountTokensRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
|
||||
contents: tuple[Mapping[str, JsonValue], ...] = ()
|
||||
contents: Annotated[tuple[Mapping[str, JsonValue], ...], BeforeValidator(_google_contents)] = ()
|
||||
system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction")
|
||||
tools: tuple[_GoogleCountTokensTool, ...] = ()
|
||||
tools: Annotated[tuple[_GoogleCountTokensTool, ...], _GOOGLE_REPEATED_FIELD] = ()
|
||||
|
||||
|
||||
router: Final = APIRouter(
|
||||
|
|
@ -112,7 +165,11 @@ def _google_count_tokens_request(model_name: str, data: object) -> TokenCountReq
|
|||
GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
|
||||
)
|
||||
system: Final = (
|
||||
[{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts]
|
||||
[
|
||||
{"type": "text", "text": part.text}
|
||||
for part in validated_data.system_instruction.parts
|
||||
if part.text is not None
|
||||
]
|
||||
if validated_data.system_instruction is not None
|
||||
else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ from litellm.proxy._types import (
|
|||
EnterpriseLicenseData,
|
||||
Litellm_EntityType,
|
||||
LitellmUserRoles,
|
||||
ProxyErrorTypes,
|
||||
ProxyException,
|
||||
SpecialModelNames,
|
||||
UserAPIKeyAuth,
|
||||
|
|
@ -47,6 +46,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
from litellm.proxy.auth.model_checks import get_key_models
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils import http_parsing_utils
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.db.health_check_latest import (
|
||||
|
|
@ -683,21 +683,9 @@ async def health_services_endpoint(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
def convert_health_check_to_dict(check) -> dict:
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.spend_tracking.input_tokens import tokenizer_models
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
|||
return resolve_model_group_alias(llm_router.model_group_alias, model)
|
||||
|
||||
|
||||
def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return tokenizer_models(model, llm_router)
|
||||
|
||||
|
||||
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -884,6 +891,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
] = _resolve_ptu_team_ceiling_via_proxy_router,
|
||||
fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
|
||||
force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
|
||||
tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router,
|
||||
):
|
||||
self.internal_usage_cache = internal_usage_cache
|
||||
self._time_provider = time_provider or datetime.now
|
||||
|
|
@ -893,6 +901,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
|
||||
self._tokenizer_models_resolver = tokenizer_models_resolver
|
||||
if self.internal_usage_cache.dual_cache.redis_cache is not None:
|
||||
self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
|
||||
BATCH_RATE_LIMITER_SCRIPT
|
||||
|
|
@ -4009,15 +4018,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
try:
|
||||
estimate: Final = max(
|
||||
0,
|
||||
int(
|
||||
token_counter(
|
||||
model=model or "",
|
||||
messages=countable_messages,
|
||||
text=selected_text,
|
||||
tools=countable_tools,
|
||||
tool_choice=countable_tool_choice,
|
||||
use_default_image_token_count=True,
|
||||
*(
|
||||
int(
|
||||
token_counter(
|
||||
model=tokenizer_model,
|
||||
messages=countable_messages,
|
||||
text=selected_text,
|
||||
tools=countable_tools,
|
||||
tool_choice=countable_tool_choice,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
)
|
||||
for tokenizer_model in self._tokenizer_models_resolver(model or "")
|
||||
),
|
||||
)
|
||||
return estimate + audio_token_estimate
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from litellm.proxy.auth.password_policy import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
AUTH_OBJECTS_TARGET,
|
||||
object_permission_cache_key,
|
||||
|
|
@ -1729,21 +1730,9 @@ async def user_update(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
async def bulk_update_processed_users(
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
|
|||
coordination_redis_cache,
|
||||
publish_config_change,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
|
||||
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
|
||||
|
|
@ -4087,21 +4088,9 @@ async def update_key_fn(
|
|||
# update based on remaining passed in values
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
@ -7244,21 +7233,9 @@ async def list_keys(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in list_keys: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"error({e})"),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
async def _apply_non_admin_alias_scope(
|
||||
|
|
@ -7402,21 +7379,9 @@ async def key_aliases(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in key_aliases: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"error({e})"),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
def _validate_sort_params(
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, is_team_admin
|
||||
|
|
@ -2845,21 +2846,9 @@ async def delete_model(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
async def delete_team_model_alias(
|
||||
|
|
@ -2899,6 +2888,13 @@ async def delete_team_model_alias(
|
|||
return removed_model_aliases
|
||||
|
||||
|
||||
def _model_not_added_to_db() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Failed to add model to db. Check your server logs for more details."},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/model/new",
|
||||
description="Allows adding new models to the model list in the config.yaml",
|
||||
|
|
@ -3080,6 +3076,8 @@ async def add_new_model(
|
|||
if isinstance(e, HTTPException):
|
||||
raise
|
||||
verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
|
||||
if model_response is None:
|
||||
raise _model_not_added_to_db() from e
|
||||
|
||||
else:
|
||||
raise HTTPException(
|
||||
|
|
@ -3088,10 +3086,7 @@ async def add_new_model(
|
|||
)
|
||||
|
||||
if model_response is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Failed to add model to db. Check your server logs for more details."},
|
||||
)
|
||||
raise _model_not_added_to_db()
|
||||
|
||||
## CREATE AUDIT LOG ##
|
||||
asyncio.create_task(
|
||||
|
|
@ -3121,21 +3116,9 @@ async def add_new_model(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
#### MODEL MANAGEMENT ####
|
||||
|
|
@ -3364,21 +3347,9 @@ async def update_model(
|
|||
return None if model_response is None else _model_write_response(model_response, member_write)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
|
|
@ -1396,21 +1397,9 @@ async def organization_member_add(
|
|||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error adding member to organization: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
async def find_member_if_email(user_email: str, prisma_client: PrismaClient) -> LiteLLM_UserTable:
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
|||
)
|
||||
from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
|
||||
from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
team_membership_auth_cache_key,
|
||||
|
|
@ -5526,21 +5527,9 @@ async def team_info(
|
|||
e,
|
||||
traceback.format_exc(),
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.get(
|
||||
|
|
|
|||
|
|
@ -100,7 +100,11 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
UnsupportedContentTypeError,
|
||||
deployment_openai_model,
|
||||
offload_token_count,
|
||||
)
|
||||
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
||||
HUGGING_FACE_SOURCE,
|
||||
KimiK3Encodings,
|
||||
|
|
@ -524,6 +528,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
LITELLM_CALL_ID_HEADER,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
proxy_exception_for,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.periodic_reload_schedule import (
|
||||
|
|
@ -15541,7 +15546,7 @@ async def _count_request_tokens(
|
|||
count_source="local_estimate",
|
||||
)
|
||||
estimate: Final = await _local_token_estimate(
|
||||
model=model_to_use,
|
||||
model=deployment_openai_model(deployment_view) or model_to_use,
|
||||
prompt=prompt if counted_messages is None else None,
|
||||
messages=counted_messages,
|
||||
tools=counted_tools,
|
||||
|
|
@ -18536,25 +18541,13 @@ async def async_queue_request(
|
|||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
elif isinstance(e, litellm.BadRequestError):
|
||||
if isinstance(e, litellm.BadRequestError):
|
||||
raise ProxyException(
|
||||
message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code
|
||||
)
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
) from e
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@app.get("/fallback/login", tags=["experimental"], include_in_schema=False)
|
||||
|
|
@ -19833,21 +19826,9 @@ async def update_config(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
### CONFIG GENERAL SETTINGS
|
||||
|
|
@ -21030,21 +21011,9 @@ async def get_config(
|
|||
}
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
@router.get(
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata
|
|||
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
OpenAIResponsesHandler,
|
||||
|
|
@ -96,6 +97,7 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.responses.main import DeleteResponseResult
|
||||
from litellm.types.token_count import TokenCountSourceFields
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse
|
||||
from litellm.utils import get_base_model_from_metadata
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
|
@ -537,7 +539,10 @@ async def cursor_model_list(
|
|||
|
||||
def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]:
|
||||
return lambda: count_request_input_tokens(
|
||||
model=str(logging_obj.model_call_details.get("model") or ""),
|
||||
model=usage_tokenizer_model(
|
||||
str(logging_obj.model_call_details.get("model") or ""),
|
||||
get_base_model_from_metadata(logging_obj.model_call_details),
|
||||
),
|
||||
request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")),
|
||||
system_input=None,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import time
|
|||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast
|
||||
|
||||
|
|
@ -39,7 +40,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
tag_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -1599,7 +1600,19 @@ async def count_request_input_tokens(
|
|||
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
|
||||
if not models:
|
||||
return MappingProxyType({})
|
||||
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
|
||||
counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models})
|
||||
counts: Final = await count_input_tokens(
|
||||
request_body=request_body,
|
||||
raw_body=raw_body,
|
||||
models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))),
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
model: count
|
||||
for model, names in counted_as.items()
|
||||
if (count := max((counts[name] for name in names if name in counts), default=None)) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _estimate_input_tokens(
|
||||
|
|
|
|||
|
|
@ -15,14 +15,16 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
billable_input_since_newest_compaction,
|
||||
holds_compaction_summary,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.rust_bridge import runtime
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge.token_counter import (
|
||||
|
|
@ -33,6 +35,9 @@ from litellm.rust_bridge.token_counter import (
|
|||
rust_tokenizer,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
|
||||
|
||||
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
|
||||
|
|
@ -47,6 +52,25 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int:
|
|||
return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS)
|
||||
|
||||
|
||||
def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]:
|
||||
"""The names to count a request for `model` with, keeping the largest count.
|
||||
|
||||
An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets
|
||||
their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding
|
||||
tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's
|
||||
tokenizer. A name the router does not index counts as written."""
|
||||
if llm_router is None:
|
||||
return (model,)
|
||||
group: Final = resolve_model_group_alias(llm_router.model_group_alias, model) or model
|
||||
served: Final = served_openai_models(llm_router.deployments_named(group) or ())
|
||||
if not served:
|
||||
return (model,)
|
||||
openai_served: Final = tuple(served_model for served_model in served if served_model is not None)
|
||||
if len(openai_served) == len(served):
|
||||
return openai_served
|
||||
return (unresolved_tokenizer_model(model), *openai_served)
|
||||
|
||||
|
||||
async def count_input_tokens(
|
||||
request_body: dict,
|
||||
raw_body: bytes | None,
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ from litellm.litellm_core_utils.served_output_texts import (
|
|||
record_served_output_texts,
|
||||
served_stream_output_texts,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count, usage_tokenizer_model
|
||||
from litellm.llms import load_guardrail_translation_mappings
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -1139,7 +1139,9 @@ def _estimated_prompt_usage(
|
|||
None,
|
||||
)
|
||||
return _estimate_dispatched_failure_usage(
|
||||
model=str(model_call_details.get("model") or ""),
|
||||
model=usage_tokenizer_model(
|
||||
str(model_call_details.get("model") or ""), get_base_model_from_metadata(model_call_details)
|
||||
),
|
||||
request_input=(
|
||||
messages_since_newest_compaction(dispatched_input)
|
||||
if dispatched_call_type in _RESPONSES_CALL_TYPES
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.responses.utils import ResponsesAPIRequestUtils
|
|||
from litellm.types.llms.openai import (
|
||||
PART_UNION_TYPES,
|
||||
BaseLiteLLMOpenAIResponseObject,
|
||||
ChatCompletionAnnotation,
|
||||
ContentPartAddedEvent,
|
||||
ContentPartDoneEvent,
|
||||
ContentPartDonePartOutputText,
|
||||
|
|
@ -56,6 +57,7 @@ from litellm.types.llms.openai import (
|
|||
WebSearchCallInProgressEvent,
|
||||
WebSearchCallSearchingEvent,
|
||||
)
|
||||
from litellm.types.responses.main import GenericResponseOutputItemContentAnnotation
|
||||
from litellm.types.utils import Delta as ChatCompletionDelta
|
||||
from litellm.types.utils import (
|
||||
ModelResponse,
|
||||
|
|
@ -105,7 +107,6 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
|
|||
getattr(delta, "reasoning_content", None),
|
||||
delta.tool_calls,
|
||||
delta.function_call,
|
||||
getattr(delta, "annotations", None),
|
||||
getattr(delta, "images", None),
|
||||
getattr(delta, "audio", None),
|
||||
)
|
||||
|
|
@ -150,7 +151,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.sent_output_text_done_event: bool = False
|
||||
self.sent_output_content_part_done_event: bool = False
|
||||
self.sent_output_item_done_event: bool = False
|
||||
self.sent_annotation_events: bool = False
|
||||
self._annotation_index: int = 0
|
||||
self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
|
||||
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
|
||||
self.completed_response = None
|
||||
self.final_text: str = ""
|
||||
|
|
@ -339,7 +341,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
namespace_kwargs: Final = {"namespace": tool_namespace} if tool_namespace else {}
|
||||
return {**item_kwargs, "name": tool_name, **namespace_kwargs}
|
||||
|
||||
def _is_reasoning_end(self, chunk):
|
||||
def _is_reasoning_end(self, chunk: ModelResponseStream) -> bool:
|
||||
if not chunk.choices:
|
||||
return False
|
||||
delta: Final = chunk.choices[0].delta
|
||||
|
|
@ -350,7 +352,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if hasattr(delta, "thinking_blocks") and delta.thinking_blocks:
|
||||
return False
|
||||
|
||||
return delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None
|
||||
return bool(
|
||||
delta.content or delta.function_call or delta.tool_calls or chunk.choices[0].finish_reason is not None
|
||||
)
|
||||
|
||||
def _reserve_web_search_indexes(self, provider_fields: object) -> None:
|
||||
if not isinstance(provider_fields, dict):
|
||||
|
|
@ -428,6 +432,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
else:
|
||||
fn_name = str(getattr(fn, "name", "") or "")
|
||||
fn_args_delta = serialize_tool_call_arguments(getattr(fn, "arguments", ""))
|
||||
if call_id not in self._tool_output_index_by_call_id:
|
||||
self._queue_held_annotations()
|
||||
output_index = self._get_or_assign_tool_output_index(call_id)
|
||||
|
||||
if self._is_tool_search_call(call_id, fn_name):
|
||||
|
|
@ -782,8 +788,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return event
|
||||
|
||||
def _queue_message_item_added_events(self) -> None:
|
||||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
item_id: Final = self._message_item_id()
|
||||
self.sent_message_item_added_event = True
|
||||
self.sent_content_part_added_event = True
|
||||
self._sequence_number += 1
|
||||
|
|
@ -792,7 +797,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
output_index=self._message_index(),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**{
|
||||
"id": self._cached_item_id,
|
||||
"id": item_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "in_progress",
|
||||
|
|
@ -1081,6 +1086,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self.litellm_model_response:
|
||||
self._queue_compaction_item_added()
|
||||
self._queue_compaction_item_done()
|
||||
self._queue_held_annotations()
|
||||
if self._pending_response_events:
|
||||
return self._pending_response_events.pop(0)
|
||||
# If tool calls exist, emit tool events before finishing/response.completed.
|
||||
|
|
@ -1163,7 +1169,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
delta: Final = chunk.choices[0].delta if chunk.choices else None
|
||||
if delta and getattr(delta, "reasoning_content", None):
|
||||
self._accumulated_reasoning_content_parts.append(delta.reasoning_content)
|
||||
if not self._is_reasoning_end(chunk):
|
||||
if self._is_reasoning_end(chunk):
|
||||
self._queue_reasoning_done_events()
|
||||
|
||||
def _queue_reasoning_done_events(self) -> None:
|
||||
if not self._reasoning_active or self._reasoning_done_emitted:
|
||||
return
|
||||
reasoning_content: Final = "".join(self._accumulated_reasoning_content_parts)
|
||||
self._cached_reasoning_item_id = (
|
||||
|
|
@ -1237,9 +1247,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self._snapshot_chunk_for_stream_chunk_builder(chunk)
|
||||
)
|
||||
self._queue_reasoning_done_events_on_reasoning_end(chunk)
|
||||
response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
|
||||
if response_api_chunk:
|
||||
self._pending_response_events.append(response_api_chunk)
|
||||
self._queue_chunk_events(chunk)
|
||||
|
||||
if self._pending_response_events:
|
||||
return self._pending_response_events.pop(0)
|
||||
|
|
@ -1300,9 +1308,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self._snapshot_chunk_for_stream_chunk_builder(cast(ModelResponseStream, chunk))
|
||||
)
|
||||
self._queue_reasoning_done_events_on_reasoning_end(chunk)
|
||||
response_api_chunk = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
|
||||
if response_api_chunk:
|
||||
self._pending_response_events.append(response_api_chunk)
|
||||
self._queue_chunk_events(chunk)
|
||||
if self._pending_response_events:
|
||||
return self._pending_response_events.pop(0)
|
||||
# Otherwise, loop to next chunk
|
||||
|
|
@ -1313,6 +1319,58 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.finished = True
|
||||
raise e
|
||||
|
||||
def _queue_chunk_events(self, chunk: ModelResponseStream) -> None:
|
||||
self._hold_chunk_annotations(chunk)
|
||||
response_api_chunk: Final = self._transform_chat_completion_chunk_to_response_api_chunk(chunk)
|
||||
if response_api_chunk:
|
||||
self._pending_response_events.append(response_api_chunk)
|
||||
if self.sent_message_item_added_event:
|
||||
self._queue_held_annotations()
|
||||
|
||||
def _hold_chunk_annotations(self, chunk: ModelResponseStream) -> None:
|
||||
chunk_annotations: Final = (getattr(chunk.choices[0].delta, "annotations", None) or ()) if chunk.choices else ()
|
||||
self._annotations_awaiting_message = (*self._annotations_awaiting_message, *chunk_annotations)
|
||||
|
||||
def _queue_held_annotations(self) -> None:
|
||||
annotations: Final = self._annotations_awaiting_message
|
||||
if not annotations:
|
||||
return
|
||||
self._annotations_awaiting_message = ()
|
||||
self._queue_reasoning_done_events()
|
||||
self._queue_annotation_added_events(annotations)
|
||||
|
||||
def _queue_annotation_added_events(self, annotations: Sequence[ChatCompletionAnnotation]) -> None:
|
||||
if not self.sent_message_item_added_event:
|
||||
self._queue_message_item_added_events()
|
||||
item_id: Final = self._message_item_id()
|
||||
response_annotations: Final = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
|
||||
annotations=annotations
|
||||
)
|
||||
)
|
||||
self._pending_response_events.extend(
|
||||
self._annotation_added_event(item_id, annotation) for annotation in response_annotations
|
||||
)
|
||||
|
||||
def _message_item_id(self) -> str:
|
||||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
return self._cached_item_id
|
||||
|
||||
def _annotation_added_event(
|
||||
self, item_id: str, annotation: GenericResponseOutputItemContentAnnotation
|
||||
) -> OutputTextAnnotationAddedEvent:
|
||||
event: Final = OutputTextAnnotationAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
|
||||
item_id=item_id,
|
||||
output_index=self._message_index(),
|
||||
content_index=0,
|
||||
annotation_index=self._annotation_index,
|
||||
annotation=annotation.model_dump(),
|
||||
)
|
||||
self._annotation_index += 1
|
||||
return event
|
||||
|
||||
def _transform_chat_completion_chunk_to_response_api_chunk(
|
||||
self, chunk: ModelResponseStream
|
||||
) -> ResponsesAPIStreamingResponse | None:
|
||||
|
|
@ -1321,37 +1379,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
|
||||
This currently handles emitting the OutputTextDeltaEvent, which is used by other tools using the responses API
|
||||
and the ReasoningSummaryTextDeltaEvent, which is used by the responses API to emit reasoning content.
|
||||
It also handles emitting annotation.added events when annotations are detected in the chunk.
|
||||
"""
|
||||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
item_id: Final = self._cached_item_id
|
||||
item_id: Final = self._message_item_id()
|
||||
|
||||
# Check if this chunk has annotations first (before processing text/reasoning)
|
||||
# This ensures we detect and queue annotation events from the annotation chunk
|
||||
if chunk.choices and hasattr(chunk.choices[0].delta, "annotations"):
|
||||
annotations: Final = chunk.choices[0].delta.annotations
|
||||
if annotations and self.sent_annotation_events is False:
|
||||
self.sent_annotation_events = True
|
||||
# Store annotation events to emit them one by one
|
||||
if not hasattr(self, "_pending_annotation_events"):
|
||||
response_annotations = LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
|
||||
annotations=annotations
|
||||
)
|
||||
self._pending_annotation_events = []
|
||||
for idx, annotation in enumerate(response_annotations):
|
||||
annotation_dict = (
|
||||
annotation.model_dump() if hasattr(annotation, "model_dump") else dict(annotation)
|
||||
)
|
||||
event = OutputTextAnnotationAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED,
|
||||
item_id=item_id,
|
||||
output_index=self._message_index(),
|
||||
content_index=0,
|
||||
annotation_index=idx,
|
||||
annotation=annotation_dict,
|
||||
)
|
||||
self._pending_annotation_events.append(event)
|
||||
# Priority 1: Handle reasoning content (highest priority)
|
||||
if (
|
||||
chunk.choices
|
||||
|
|
@ -1395,13 +1425,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self._pending_tool_events:
|
||||
return self._pending_tool_events.pop(0)
|
||||
|
||||
# Priority 4: If we have pending annotation events, emit the next one
|
||||
# This happens when the current chunk has no text/reasoning content
|
||||
if hasattr(self, "_pending_annotation_events") and self._pending_annotation_events:
|
||||
event = self._pending_annotation_events.pop(0)
|
||||
return event
|
||||
|
||||
# Priority 5: If we have pending tool events (from earlier chunk), emit the next one
|
||||
if self._pending_tool_events:
|
||||
return self._pending_tool_events.pop(0)
|
||||
|
||||
|
|
@ -1444,7 +1467,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
def _is_unstreamed_empty_message(self, item: object) -> bool:
|
||||
if getattr(item, "type", None) != "message" or self._message_output_index is not None:
|
||||
return False
|
||||
return not any(getattr(part, "text", None) for part in attribute_items(item, "content"))
|
||||
return not any(
|
||||
getattr(part, "text", None) or getattr(part, "annotations", None)
|
||||
for part in attribute_items(item, "content")
|
||||
)
|
||||
|
||||
def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:
|
||||
if not isinstance(self.litellm_model_response, ModelResponse):
|
||||
|
|
|
|||
|
|
@ -3555,7 +3555,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
choice=choice,
|
||||
)
|
||||
message_output_items.extend(image_generation_items)
|
||||
elif choice.message.content is not None:
|
||||
elif choice.message.content is not None or getattr(choice.message, "annotations", None):
|
||||
message_output_items.append(
|
||||
GenericResponseOutputItem(
|
||||
type="message",
|
||||
|
|
@ -3629,13 +3629,13 @@ class LiteLLMCompletionResponsesConfig:
|
|||
|
||||
return OutputText(
|
||||
type="output_text",
|
||||
text=message.content,
|
||||
text=message.content or "",
|
||||
annotations=transformed_annotations,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def transform_chat_completion_annotations_to_response_output_annotations(
|
||||
annotations: list[ChatCompletionAnnotation] | None,
|
||||
annotations: Sequence[ChatCompletionAnnotation] | None,
|
||||
) -> list[GenericResponseOutputItemContentAnnotation]:
|
||||
response_output_annotations: Final[list[GenericResponseOutputItemContentAnnotation]] = []
|
||||
|
||||
|
|
|
|||
|
|
@ -162,7 +162,11 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
|
|||
SensitiveDataMasker,
|
||||
mask_sensitive_structure,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
offload_token_count,
|
||||
served_openai_models,
|
||||
single_encoding_openai_model,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception
|
||||
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
|
||||
|
|
@ -1533,6 +1537,7 @@ class Router:
|
|||
self._cached_deployment_tracks_usage = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(
|
||||
self._deployment_tracks_usage
|
||||
)
|
||||
self.served_openai_models = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._served_openai_models)
|
||||
self._discovered_model_info_cache: InMemoryCache = InMemoryCache(
|
||||
max_size_in_memory=max(len(model_list or ()), 1),
|
||||
default_ttl=2 * MODEL_INFO_REFRESH_SECONDS,
|
||||
|
|
@ -11890,6 +11895,9 @@ class Router:
|
|||
indices: Final = self.model_name_to_deployment_indices.get(model_name)
|
||||
return None if indices is None else tuple(self.model_list[idx] for idx in indices)
|
||||
|
||||
def _served_openai_models(self, model_name: str) -> tuple[str | None, ...]:
|
||||
return served_openai_models(self.deployments_named(model_name) or ())
|
||||
|
||||
def team_public_model_names_of(self, team_id: str) -> frozenset[str]:
|
||||
"""The public names `team_id` serves its own deployments under."""
|
||||
with self._deployment_index_lock:
|
||||
|
|
@ -14044,6 +14052,7 @@ class Router:
|
|||
self._ptu_share_teams.cache_clear()
|
||||
self._cached_team_ptu_ceiling.cache_clear()
|
||||
self._cached_deployment_tracks_usage.cache_clear()
|
||||
self.served_openai_models.cache_clear()
|
||||
self._zero_cost_cache.clear()
|
||||
self._routing_group_rows = None
|
||||
|
||||
|
|
@ -14303,6 +14312,7 @@ class Router:
|
|||
messages: Sequence[Mapping[str, object]] | None,
|
||||
input: str | list[object] | None,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
tokenizer_model: str = "",
|
||||
) -> int:
|
||||
"""
|
||||
Count input tokens for context-window pre-call checks.
|
||||
|
|
@ -14333,7 +14343,7 @@ class Router:
|
|||
system_message: Final = anthropic_system_to_openai_message(extras.get("system"))
|
||||
if messages is not None:
|
||||
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
|
||||
return litellm.token_counter(messages=counted_messages, tools=tools)
|
||||
return litellm.token_counter(model=tokenizer_model, messages=counted_messages, tools=tools)
|
||||
if input is not None:
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
|
|
@ -14348,6 +14358,7 @@ class Router:
|
|||
only_since_compaction=True,
|
||||
)
|
||||
return litellm.token_counter(
|
||||
model=tokenizer_model,
|
||||
messages=cast(list, input_messages), # cast-ok: transformed chat messages
|
||||
tools=tools,
|
||||
)
|
||||
|
|
@ -14404,6 +14415,10 @@ class Router:
|
|||
messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter
|
||||
input=cast(str | list | None, input), # cast-ok: forwarded to the sync counter
|
||||
request_kwargs=request_kwargs,
|
||||
tokenizer_model=single_encoding_openai_model(
|
||||
self.served_openai_models(model) or served_openai_models(healthy_deployments)
|
||||
)
|
||||
or "",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
|
||||
verbose_router_logger.error(
|
||||
|
|
@ -14418,6 +14433,8 @@ class Router:
|
|||
request_kwargs: Mapping[str, object] | None,
|
||||
input_token_count: int | None,
|
||||
count_inline: bool,
|
||||
model: str,
|
||||
deployments: Sequence[Mapping[str, object]],
|
||||
) -> int | None:
|
||||
if input_token_count is not None:
|
||||
return input_token_count
|
||||
|
|
@ -14428,6 +14445,10 @@ class Router:
|
|||
messages=cast(list[dict[str, str]] | None, messages), # cast-ok: forwarded to the sync counter
|
||||
input=cast(str | list[object] | None, input), # cast-ok: forwarded to the sync counter
|
||||
request_kwargs=request_kwargs,
|
||||
tokenizer_model=single_encoding_openai_model(
|
||||
self.served_openai_models(model) or served_openai_models(deployments)
|
||||
)
|
||||
or "",
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_router_logger.error(
|
||||
|
|
@ -14622,6 +14643,8 @@ class Router:
|
|||
count_inline=has_countable_input
|
||||
and not skip_inline_token_count
|
||||
and any(limit is not None for limit in deployment_max_input_tokens),
|
||||
model=model,
|
||||
deployments=_returned_deployments,
|
||||
)
|
||||
request_settings: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else {}
|
||||
requested_params: Final[frozenset[str]] = (
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm import token_counter
|
|||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
|
||||
from litellm.types.router import RouterCacheEnum, RouterErrors
|
||||
|
|
@ -127,6 +127,12 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] |
|
|||
return itpm_key, otpm_key
|
||||
|
||||
|
||||
def _tokenizer_model(deployment: object) -> str:
|
||||
params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None
|
||||
served: Final = params.get("model") if is_str_keyed_mapping(params) else None
|
||||
return deployment_openai_model(deployment) or (served if isinstance(served, str) else "")
|
||||
|
||||
|
||||
def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int:
|
||||
if not request_kwargs:
|
||||
return 0
|
||||
|
|
@ -427,7 +433,7 @@ def io_token_pre_call_check(
|
|||
return deployment
|
||||
|
||||
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
|
||||
_model: Final = (deployment.get("litellm_params") or {}).get("model") or ""
|
||||
_model: Final = _tokenizer_model(deployment)
|
||||
estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
|
||||
max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
|
||||
|
||||
|
|
@ -491,7 +497,7 @@ async def async_io_token_pre_call_check(
|
|||
return deployment
|
||||
|
||||
request_kwargs: Final = get_io_token_rate_limit_request_kwargs()
|
||||
_model: Final = (deployment.get("litellm_params") or {}).get("model") or ""
|
||||
_model: Final = _tokenizer_model(deployment)
|
||||
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
|
||||
max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment)
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ else:
|
|||
_DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
|
||||
_TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
|
||||
_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
|
||||
_STORE_ID_ITEMS: Final = TypeAdapter(list[object] | tuple[object, ...], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
def _named_store_ids(store_ids: object) -> tuple[str, ...]:
|
||||
|
|
@ -49,7 +49,11 @@ def _named_store_ids(store_ids: object) -> tuple[str, ...]:
|
|||
return tuple(store_id for store_id in items if isinstance(store_id, str))
|
||||
|
||||
|
||||
def _tool_vector_store_ids(tool: object) -> tuple[str, ...]:
|
||||
def listed_vector_store_ids(store_ids: object) -> tuple[str, ...]:
|
||||
return _named_store_ids(store_ids) if isinstance(store_ids, (list, tuple)) else ()
|
||||
|
||||
|
||||
def vector_store_ids_in_tool(tool: object) -> tuple[str, ...]:
|
||||
try:
|
||||
return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids"))
|
||||
except ValidationError:
|
||||
|
|
@ -195,7 +199,7 @@ class VectorStoreRegistry:
|
|||
tools_to_remove: Final[list[int]] = []
|
||||
|
||||
for i, tool in enumerate(tools):
|
||||
tool_vector_store_ids = _tool_vector_store_ids(tool)
|
||||
tool_vector_store_ids = vector_store_ids_in_tool(tool)
|
||||
if not tool_vector_store_ids:
|
||||
continue
|
||||
|
||||
|
|
@ -305,7 +309,7 @@ class VectorStoreRegistry:
|
|||
List of vector stores with tool parameters merged into litellm_params
|
||||
"""
|
||||
# Pop vector_store_ids from params
|
||||
vector_store_ids: Final[list[str]] = non_default_params.pop("vector_store_ids", None) or []
|
||||
vector_store_ids: Final = [*listed_vector_store_ids(non_default_params.pop("vector_store_ids", None))]
|
||||
|
||||
# Extract params from tools and collect IDs
|
||||
params_by_id = self.get_and_pop_recognised_vector_store_tools(tools=tools, vector_store_ids=vector_store_ids)
|
||||
|
|
@ -353,7 +357,7 @@ class VectorStoreRegistry:
|
|||
List of vector stores with tool parameters merged into litellm_params
|
||||
"""
|
||||
# Pop vector_store_ids from params
|
||||
vector_store_ids: Final[list[str]] = non_default_params.pop("vector_store_ids", None) or []
|
||||
vector_store_ids: Final = [*listed_vector_store_ids(non_default_params.pop("vector_store_ids", None))]
|
||||
|
||||
# Extract params from tools and collect IDs
|
||||
params_by_id = self.get_and_pop_recognised_vector_store_tools(tools=tools, vector_store_ids=vector_store_ids)
|
||||
|
|
@ -423,7 +427,7 @@ class VectorStoreRegistry:
|
|||
validated_tools: Final = _ITEMS.validate_python(tools)
|
||||
except ValidationError:
|
||||
return vector_store_ids
|
||||
return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)]
|
||||
return [*vector_store_ids, *chain.from_iterable(vector_store_ids_in_tool(tool) for tool in validated_tools)]
|
||||
|
||||
def load_vector_stores_from_config(self, vector_stores_config: list[dict]):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -28071,7 +28071,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
|
|
@ -28345,7 +28345,7 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": true
|
||||
},
|
||||
"fireworks_ai/qwen3p7-plus": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
|
|
|
|||
103
osv-scanner.toml
103
osv-scanner.toml
|
|
@ -1,27 +1,102 @@
|
|||
[[IgnoredVulns]]
|
||||
id = "GHSA-w8v5-vhqr-4h9v"
|
||||
ignoreUntil = 2026-11-01
|
||||
reason = "diskcache has no fixed release published; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-v4x8-3wr5-3wq8"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
|
||||
reason = "filelock ReadWriteLock advisory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-vfj7-8cjw-p6xm"
|
||||
ignoreUntil = 2026-11-03
|
||||
reason = "braces has no fixed release published (3.0.3, 2024-05-21, is latest and last_affected); dev-only via knip > fast-glob > micromatch; remove this entry once one exists"
|
||||
id = "GHSA-567q-5qvw-pf92"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "Windows-only local DoS (CVSS 2.5) in filelock soft locks; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-6q48-6c7w-9gw2"
|
||||
ignoreUntil = 2026-10-23
|
||||
reason = "certifi 2026.10.19 (published 2026-10-20) is inside the 3-day uv exclude-newer window until 2026-10-23 02:24 UTC; bump certifi and remove this entry once it clears"
|
||||
id = "GHSA-9vhv-jxww-xfm5"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "Windows-only local DoS (CVSS 3.3) in filelock ReadWriteLock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-6627-m3xw-93mp"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock StrictSoftFileLock claim directories ignore the lock mode, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-69hj-f68j-2879"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock UnixFileLock truncates and chmods a file another user renames or hard-links in at the lock path; litellm's CLI takes FileLock only inside ~/.litellm, which it creates owner-only, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; the attack needs another local user who can rename or link files in that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-8mx5-7m5m-r77g"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock WindowsFileLock ignores mode; the proxy image is Linux, so only Windows installs run it; there litellm's CLI takes FileLock only inside the owner-only ~/.litellm, whose lock file holds no data, and huggingface_hub takes it in the Hugging Face cache, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-9c4x-gm8v-wgc4"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock StrictSoftFileLock and SoftReadWriteLock adopt another user's protocol directory, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-9j6q-wvj5-jgjg"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock SoftFileLock symlink swap leaks whether a path exists; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-cwqr-hh33-gh44"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock ReadWriteLock stalls on a crafted lock database, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-fq56-3cx3-fjxp"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock SoftFileLock stale-lock break removes a live lock on a reused inode; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-j8f7-rjxc-mr56"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock ReadWriteLock follows a symlink into SQLite, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed in 4.0.9, which litellm's own filelock<4.0 cap blocks on every Python until it is lifted (#48635, #48802); remove once that lands"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-m462-7rxw-wvpg"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock StrictSoftFileLock follows a symlink swapped in at <path>.filelock, which no package in the lock takes (litellm and huggingface_hub, filelock's only dependents, use just FileLock and SoftFileLock); fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-rfpc-qcw5-3pc4"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "filelock SoftFileLock removes a live holder's marker when another user forges its PID; litellm never takes SoftFileLock, but huggingface_hub falls back to it in the Hugging Face cache when the filesystem has no flock, which litellm reaches only to download the Kimi K3 tokenizer for Bedrock token counting, and the attack needs another local user who can write that cache; fixed only in 4.2.1, and filelock 4.1.1+ needs Python >= 3.11; litellm's own filelock<4.0 cap keeps every Python on 3.32.7 until it is lifted for 3.11+ (#48635, #48802); remove once that lands and 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-pcg2-g6cj-q92c"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's HTTP header parsing is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-pw9p-v7r6-gx4p"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's RedirectHandler is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-r4j5-9xpp-3985"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-w8v5-vhqr-4h9v"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-45c3-73f7-pv4m"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-xcx6-jx8p-x3h5"
|
||||
ignoreUntil = 2026-11-24
|
||||
reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0"
|
||||
|
||||
[[PackageOverrides]]
|
||||
name = "pillow"
|
||||
version = "12.3.0"
|
||||
ecosystem = "PyPI"
|
||||
vulnerability.ignore = true
|
||||
effectiveUntil = 2026-10-27
|
||||
reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups stays on 12.3.0; 3.11+ is on 13.0.0"
|
||||
effectiveUntil = 2026-11-27
|
||||
reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups and of the mlflow extra stays on 12.3.0; 3.11+ is on 13.0.0; pillow has never shipped an older-major release after the next major (no 10.x after 11.0.0, no 11.x after 12.0.0), so 13.x is the only fix line; extended to 2026-11-27 with maintainer approval, same date as the dnspython 3.10 entry; remove once 3.10 is dropped"
|
||||
|
|
|
|||
|
|
@ -360,11 +360,13 @@ cache-keys = [
|
|||
]
|
||||
constraint-dependencies = [
|
||||
"tornado>=6.5.8",
|
||||
"tornado>=6.6; python_version >= '3.11'",
|
||||
"aiohttp>=3.14.2,<4.0",
|
||||
"packaging>=24.0",
|
||||
"soupsieve>=2.8.4",
|
||||
"httplib2>=0.32.0",
|
||||
"setuptools>=84.1.0",
|
||||
"dnspython>=2.9.0; python_version >= '3.11'",
|
||||
]
|
||||
override-dependencies = [
|
||||
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
|
||||
CONCATENATED_VALUES: Final = 20_000
|
||||
|
||||
|
||||
def _popped_values(accumulator: JSONFragmentAccumulator) -> Iterator[object]:
|
||||
while True:
|
||||
found, value = accumulator.pop_next_value()
|
||||
if not found:
|
||||
return
|
||||
yield value
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_draining_many_concatenated_values() -> None:
|
||||
accumulator: Final = JSONFragmentAccumulator()
|
||||
accumulator.append('{"a": 1}' * CONCATENATED_VALUES)
|
||||
|
||||
drained: Final = sum(1 for _ in _popped_values(accumulator))
|
||||
|
||||
assert drained == CONCATENATED_VALUES
|
||||
|
|
@ -175,6 +175,7 @@ ignored_function_names = [
|
|||
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
|
||||
"_served_openai_models", # Cached as served_openai_models, tested through the pre-call check token count tests in test_router.py
|
||||
"priority_header_update", # Tested through the priority header tests in test_common_request_processing.py
|
||||
"_drops_params", # Read by split_request_priority and priority_header_update, tested through both
|
||||
"_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py
|
||||
|
|
|
|||
|
|
@ -746,7 +746,7 @@ def test_internal_user_model_creation_prohibition_refuses_team_admin_but_not_pro
|
|||
"error": {
|
||||
"message": "Model creation is disabled for internal users by disable_model_add_for_internal_users.",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def _escalation_error(field: str) -> dict[str, JsonValue]:
|
|||
"error": {
|
||||
"message": f"{{'error': \"Non-admin users cannot modify '{field}' on their own record. Contact your proxy admin.\"}}",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
}
|
||||
|
|
@ -49,7 +49,7 @@ def _role_change_error() -> dict[str, JsonValue]:
|
|||
"error": {
|
||||
"message": "Only proxy admins can modify user roles.",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -255,8 +255,8 @@ def _team_not_found_body(team_id: str) -> dict[str, JsonValue]:
|
|||
return {
|
||||
"error": {
|
||||
"message": f"{{'message': 'Team not found, passed team id: {team_id}.'}}",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "404",
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,6 +158,18 @@ def test_native_function_response_preserves_json_when_media_is_malformed(invalid
|
|||
assert messages == [{"role": "tool", "tool_call_id": "call_lookup", "content": '{"ok": true}'}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["user", "model"])
|
||||
def test_one_object_content_parts_translate_like_their_list_form(role: str) -> None:
|
||||
part: Final = {"text": "Reply with OK"}
|
||||
|
||||
def messages(parts: object) -> object:
|
||||
return GoogleGenAIAdapter().translate_generate_content_to_completion(
|
||||
model="anthropic/claude-haiku-5-5", contents=[{"role": role, "parts": parts}]
|
||||
)["messages"]
|
||||
|
||||
assert messages(part) == messages([part]) != []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "expected_arguments"),
|
||||
[(None, "null"), ('{"query":"x"}', '"{\\"query\\":\\"x\\"}"')],
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import json
|
||||
import time
|
||||
import tracemalloc
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
|
|
@ -175,40 +178,48 @@ def test_accumulation_of_many_fragments_is_not_quadratic():
|
|||
assert elapsed_ms < 50, f"1000-fragment append took {elapsed_ms:.1f} ms (expected < 50 ms); O(n^2) regression?"
|
||||
|
||||
|
||||
def test_draining_many_concatenated_values_is_not_quadratic():
|
||||
"""
|
||||
Regression guard: peeling N JSON values already sitting in one buffer,
|
||||
one pop_next_value() call per value with no new fragments in between,
|
||||
must be O(n) total. Re-copying the shrinking remainder on every pop
|
||||
(slicing a new string instead of advancing a cursor) makes total drain
|
||||
time scale with the square of the buffer size.
|
||||
def _popped_values(accumulator: JSONFragmentAccumulator) -> Iterator[object]:
|
||||
while True:
|
||||
found, value = accumulator.pop_next_value()
|
||||
if not found:
|
||||
return
|
||||
yield value
|
||||
|
||||
Uses a doubling ratio rather than an absolute ms budget so it isn't
|
||||
flaky on a slower or busier CI runner: doubling the input should
|
||||
roughly double an O(n) drain's time but roughly quadruple an O(n^2)
|
||||
drain's time, and that ratio holds regardless of machine speed.
|
||||
"""
|
||||
|
||||
def drain_time_ms(n: int) -> float:
|
||||
accumulator = JSONFragmentAccumulator()
|
||||
accumulator.append('{"a": 1}' * n)
|
||||
start = time.perf_counter()
|
||||
drained = 0
|
||||
while True:
|
||||
found, _ = accumulator.pop_next_value()
|
||||
if not found:
|
||||
break
|
||||
drained += 1
|
||||
assert drained == n
|
||||
return (time.perf_counter() - start) * 1000
|
||||
def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator:
|
||||
accumulator: Final = JSONFragmentAccumulator()
|
||||
for fragment in fragments:
|
||||
accumulator.append(fragment)
|
||||
return accumulator
|
||||
|
||||
small_ms = drain_time_ms(40_000)
|
||||
large_ms = drain_time_ms(80_000)
|
||||
|
||||
ratio = large_ms / max(small_ms, 0.001)
|
||||
assert ratio < 3.0, (
|
||||
f"doubling drained values scaled time by {ratio:.2f}x ({small_ms:.1f} ms -> {large_ms:.1f} ms); "
|
||||
"expected roughly 2x for O(n); O(n^2) regression?"
|
||||
def _drain_peak_bytes(fragments: tuple[str, ...]) -> int:
|
||||
accumulator: Final = _streamed(fragments)
|
||||
first_found, _ = accumulator.pop_next_value()
|
||||
assert first_found is True
|
||||
already_tracing: Final = tracemalloc.is_tracing()
|
||||
if not already_tracing:
|
||||
tracemalloc.start()
|
||||
tracemalloc.reset_peak()
|
||||
baseline, _ = tracemalloc.get_traced_memory()
|
||||
try:
|
||||
drained: Final = sum(1 for _ in _popped_values(accumulator))
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
if not already_tracing:
|
||||
tracemalloc.stop()
|
||||
assert drained == len(fragments) - 1
|
||||
return peak - baseline
|
||||
|
||||
|
||||
def test_draining_many_concatenated_values_never_copies_the_remainder():
|
||||
value: Final = '{"text": "' + "x" * 20 + '"}'
|
||||
fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000))
|
||||
payload_bytes: Final = sum(len(fragment) for fragment in fragments)
|
||||
peak_bytes: Final = _drain_peak_bytes(fragments)
|
||||
assert peak_bytes < payload_bytes // 4, (
|
||||
f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, "
|
||||
"copying the remainder on every pop?"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2519,3 +2519,39 @@ def test_stream_chunk_builder_carries_the_bedrock_guardrail_trace_and_records_it
|
|||
(entry,) = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"]
|
||||
assert entry["guardrail_usage"] == {"topicPolicyUnits": 2}
|
||||
assert entry["guardrail_cost"] == pytest.approx(2 * 0.00015)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_model", "counted_as"), [("azure/gpt-6.1-sol", "azure/gpt-6.1-sol"), ("claude-sonnet-4-5", "my-gpt-6")]
|
||||
)
|
||||
def test_stream_chunk_builder_counts_a_custom_named_deployments_missing_usage_with_its_base_model_only_when_openai(
|
||||
base_model: str, counted_as: str
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
chunks: Final = [
|
||||
ModelResponseStream(id="s", model="my-gpt-6", choices=[StreamingChoices(index=0, delta=Delta(content="hi"))]),
|
||||
ModelResponseStream(
|
||||
id="s", model="my-gpt-6", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]
|
||||
),
|
||||
]
|
||||
logging_obj: Final = Logging(
|
||||
model="my-gpt-6",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
call_type="acompletion",
|
||||
start_time=None,
|
||||
litellm_call_id="call",
|
||||
function_id="fn",
|
||||
)
|
||||
logging_obj.model_call_details["litellm_params"] = {"metadata": {"model_info": {"base_model": base_model}}}
|
||||
|
||||
built: Final = stream_chunk_builder(chunks=chunks, messages=messages, logging_obj=logging_obj)
|
||||
|
||||
assert built is not None
|
||||
assert litellm.token_counter(model=base_model, messages=messages) != litellm.token_counter(
|
||||
model="my-gpt-6", messages=messages
|
||||
)
|
||||
assert built.usage.prompt_tokens == litellm.token_counter(model=counted_as, messages=messages)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.litellm_core_utils.token_counter import (
|
|||
calculate_img_tokens,
|
||||
estimate_image_tokens,
|
||||
high_detail_image_token_upper_bound,
|
||||
is_openai_model,
|
||||
offload_token_count,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new
|
||||
|
|
@ -2600,3 +2601,95 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
|
|||
)
|
||||
== 200_000
|
||||
)
|
||||
|
||||
|
||||
_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"gpt-6-sol",
|
||||
"gpt-6.1-sol",
|
||||
"gpt-6.1-terra",
|
||||
"azure/gpt-41",
|
||||
"azure/gpt-5.1",
|
||||
"openai/gpt-5.1",
|
||||
"openai/gpt-4o",
|
||||
"gpt-41",
|
||||
"eu/gpt-6.1-sol",
|
||||
"azure/eu/gpt-6.1-sol",
|
||||
"azure/gpt-5.1-chat",
|
||||
"azure/gpt-6.1-sol-2026-09-29",
|
||||
"openai/gpt-6.1-sol-2026-09-29",
|
||||
"azure_ai/gpt-6.1-sol",
|
||||
"chatgpt/gpt-6.1-sol",
|
||||
"perplexity/openai/gpt-5",
|
||||
],
|
||||
)
|
||||
def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str):
|
||||
expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
|
||||
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected, (
|
||||
f"{model} should count with o200k_base. OpenAI's /v1/responses/input_tokens gave gpt-5, gpt-5.1, gpt-6-sol, "
|
||||
"gpt-6-luna, gpt-6-astra, gpt-6.1-sol and gpt-6.1-terra the same count on this sample, checked 2026-10-28"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["openai/gpt-5.9", "chatgpt/gpt-5.3-instant", "azure/gpt-5.9-mini"])
|
||||
def test_token_counter_counts_unlisted_openai_names_with_the_encoding_tiktoken_gives_them(model: str):
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len(
|
||||
tiktoken.encoding_for_model(model.rsplit("/", 1)[-1]).encode(_MULTILINGUAL_SAMPLE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-6.2", "openai/gpt-6.2", "azure/gpt-6.2-mini", "azure/eu/o9-mini", "gpt-10"])
|
||||
def test_token_counter_counts_unlisted_names_of_newer_openai_releases_with_o200k(model: str):
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len(
|
||||
tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"groq/qwen3-32b",
|
||||
"openai/my-llama",
|
||||
"text-embedding-ada-002-v2",
|
||||
"azure/gpt-4",
|
||||
"azure/my-deployment",
|
||||
"azure/ada",
|
||||
"azure/mistral-large-latest",
|
||||
"azure/command-r-plus",
|
||||
"azure_ai/mistral-large-latest",
|
||||
],
|
||||
)
|
||||
def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str):
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len(
|
||||
tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)
|
||||
)
|
||||
|
||||
|
||||
def test_openai_model_check_holds_while_openai_models_are_registered_concurrently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "azure/azure-only-chat-model", {"litellm_provider": "azure", "mode": "chat"})
|
||||
registered: Final = tuple(f"concurrently-registered-model-{index}" for index in range(200))
|
||||
checked: Final = threading.Semaphore(0)
|
||||
|
||||
def register() -> None:
|
||||
for name in registered:
|
||||
if not checked.acquire(timeout=10):
|
||||
return
|
||||
litellm.open_ai_chat_completion_models.add(name)
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
registering: Final = pool.submit(register)
|
||||
while not registering.done():
|
||||
assert not is_openai_model("azure-only-chat-model")
|
||||
checked.release()
|
||||
registering.result()
|
||||
finally:
|
||||
litellm.open_ai_chat_completion_models.difference_update(registered)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled
|
||||
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled, prepare_context_managed_request
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
PolyfillResult,
|
||||
|
|
@ -690,6 +690,115 @@ def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, tri
|
|||
assert (applied is not None) == triggers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
["Reference notes. " * 400, [{"type": "text", "text": "Reference notes. " * 400}]],
|
||||
ids=["string", "blocks"],
|
||||
)
|
||||
def test_input_tokens_trigger_counts_the_system_prompt(system: str | list[dict[str, object]]) -> None:
|
||||
"""Like the API, the trigger compares the whole prompt: a system prompt can push it over."""
|
||||
messages = _make_history(n_pairs=3)
|
||||
history_tokens: Final = litellm.token_counter(model=MODEL, messages=messages)
|
||||
edit_spec: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": history_tokens + 100},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
_, without_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL, messages=deepcopy(messages), tools=None, system=None, edit_spec=edit_spec
|
||||
)
|
||||
_, with_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL, messages=deepcopy(messages), tools=None, system=system, edit_spec=edit_spec
|
||||
)
|
||||
_, met_without_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL,
|
||||
messages=deepcopy(messages),
|
||||
tools=None,
|
||||
system=None,
|
||||
edit_spec={**edit_spec, "trigger": {"type": "input_tokens", "value": 0}},
|
||||
)
|
||||
assert without_system is None
|
||||
assert with_system is not None
|
||||
assert met_without_system is not None
|
||||
assert with_system["cleared_tool_uses"] == 2
|
||||
assert with_system["cleared_input_tokens"] == met_without_system["cleared_input_tokens"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compact_position", [None, "after", "before"], ids=["alone", "before_compact", "after_compact"])
|
||||
async def test_input_tokens_trigger_counts_a_client_compaction_summary(compact_position: str | None) -> None:
|
||||
"""A client compaction summary counts toward the trigger and calls beside it stay clearable, in any edit order."""
|
||||
tail: Final = _make_history(n_pairs=3)
|
||||
edit_spec: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": litellm.token_counter(model=MODEL, messages=tail) + 100},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
compaction: Final = {"type": "compaction", "content": "Earlier work. " * 400}
|
||||
compacted_history: Final = [tail[0], {**tail[1], "content": [compaction, *tail[1]["content"]]}, *tail[2:]]
|
||||
compact_spec: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}
|
||||
edits: Final = {None: [edit_spec], "after": [edit_spec, compact_spec], "before": [compact_spec, edit_spec]}[
|
||||
compact_position
|
||||
]
|
||||
|
||||
async def prepare(messages: list[dict[str, object]]) -> PolyfillResult | None:
|
||||
return await prepare_context_managed_request(
|
||||
model=MODEL,
|
||||
messages=messages,
|
||||
tools=None,
|
||||
system=None,
|
||||
context_management_spec={"edits": edits},
|
||||
litellm_metadata=None,
|
||||
additional_drop_params=None,
|
||||
llm_router=None,
|
||||
)
|
||||
|
||||
def cleared(result: PolyfillResult | None) -> list[object]:
|
||||
applied: Final = result.applied_edits if result is not None else []
|
||||
return [edit.get("cleared_tool_uses") for edit in applied if edit.get("type") == "clear_tool_uses_20250919"]
|
||||
|
||||
without_summary: Final = await prepare(deepcopy(tail))
|
||||
with_summary: Final = await prepare(deepcopy(compacted_history))
|
||||
|
||||
assert cleared(without_summary) == []
|
||||
assert cleared(with_summary) == [2]
|
||||
|
||||
|
||||
async def test_compact_after_clearing_replaces_the_client_compaction_summary(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model")
|
||||
summary: Final = MagicMock()
|
||||
summary.choices = [MagicMock(message=MagicMock(content="<summary>Fresh summary</summary>"))]
|
||||
summary.usage = MagicMock(prompt_tokens=50, completion_tokens=100)
|
||||
router: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=summary)
|
||||
tail: Final = _make_history(n_pairs=3)
|
||||
result: Final = await prepare_context_managed_request(
|
||||
model=MODEL,
|
||||
messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Old summary. " * 20_000}]}, *tail],
|
||||
tools=None,
|
||||
system="You are a helpful assistant.",
|
||||
context_management_spec={
|
||||
"edits": [
|
||||
{
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": 1_000},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
},
|
||||
{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}},
|
||||
]
|
||||
},
|
||||
litellm_metadata=None,
|
||||
additional_drop_params=None,
|
||||
llm_router=router,
|
||||
)
|
||||
assert result is not None
|
||||
assert isinstance(result.system, str)
|
||||
assert "Fresh summary" in result.system
|
||||
assert "Old summary" not in result.system
|
||||
assert result.system.endswith("You are a helpful assistant.")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)])
|
||||
def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared(
|
||||
clear_tool_inputs: bool, triggers: bool
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ async def test_sync_editor_counts_tokens_off_the_event_loop():
|
|||
{
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": 10_000_000},
|
||||
"keep": {"type": "tool_uses", "value": 0},
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -322,6 +322,29 @@ def test_mapped_response_still_bills_the_prompt_the_provider_processed(provider)
|
|||
assert litellm.completion_cost(completion_response=response) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["openai", "azure"])
|
||||
@pytest.mark.parametrize(
|
||||
("base_model", "counted_as"), [("azure/gpt-6.1-sol", "azure/gpt-6.1-sol"), ("claude-sonnet-4-5", "my-gpt-6")]
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_mapped_response_bills_a_custom_named_deployment_with_its_base_model_only_when_it_is_openai(
|
||||
provider: str, base_model: str, counted_as: str
|
||||
) -> None:
|
||||
messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
overrides: Final = {"model": f"{provider}/my-gpt-6", "messages": messages, "base_model": base_model}
|
||||
|
||||
sync_response: Final = litellm.completion(
|
||||
**_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), **overrides)
|
||||
)
|
||||
async_response: Final = await litellm.acompletion(
|
||||
**_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), **overrides)
|
||||
)
|
||||
|
||||
expected: Final = token_counter(model=counted_as, messages=messages)
|
||||
assert token_counter(model=base_model, messages=messages) != token_counter(model="my-gpt-6", messages=messages)
|
||||
assert (sync_response.usage.prompt_tokens, async_response.usage.prompt_tokens) == (expected, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["openai", "azure"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_output_limit_400_maps_to_length_truncated_response(provider):
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
tag_registry_cache_key,
|
||||
)
|
||||
from litellm.utils import get_utc_datetime
|
||||
from litellm.llms.base_llm.managed_resources.utils import encode_unified_id, generate_unified_id_string
|
||||
from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
|
||||
from tests._master_key import MASTER_KEY
|
||||
|
||||
|
|
@ -1551,37 +1552,24 @@ async def test_get_team_db_check_does_not_call_new_team_if_exists(mock_new_team,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"prisma_client,vector_store_registry,expected_result",
|
||||
[
|
||||
(None, MagicMock(), True), # No prisma client
|
||||
(MagicMock(), None, True), # No vector store registry
|
||||
(MagicMock(), MagicMock(), True), # No vector stores to run
|
||||
],
|
||||
)
|
||||
async def test_vector_store_access_check_early_returns(prisma_client, vector_store_registry, expected_result):
|
||||
@pytest.mark.parametrize("prisma_client", [None, MagicMock()], ids=["no-prisma-client", "no-vector-stores-to-run"])
|
||||
async def test_vector_store_access_check_early_returns(prisma_client):
|
||||
"""Test vector_store_access_check returns True for early exit conditions"""
|
||||
request_body = {"messages": [{"role": "user", "content": "test"}]}
|
||||
|
||||
if vector_store_registry:
|
||||
vector_store_registry.get_vector_store_ids_to_run.return_value = None
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", prisma_client),
|
||||
patch("litellm.vector_store_registry", vector_store_registry),
|
||||
):
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", prisma_client):
|
||||
result = await vector_store_access_check(
|
||||
request_body=request_body,
|
||||
team_object=None,
|
||||
valid_token=None,
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vector_store_access_check_skips_db_lookup_when_no_vector_stores_requested():
|
||||
"""Registry returns [] (not None) for plain requests; no object permission DB lookup should happen."""
|
||||
"""A plain request names no vector store, so no object permission DB lookup should happen."""
|
||||
valid_token = UserAPIKeyAuth(token="test-token", object_permission_id="perm-123")
|
||||
team_object = MagicMock()
|
||||
team_object.object_permission_id = "team-permission"
|
||||
|
|
@ -1590,16 +1578,8 @@ async def test_vector_store_access_check_skips_db_lookup_when_no_vector_stores_r
|
|||
find_unique = AsyncMock()
|
||||
mock_prisma_client.db.litellm_objectpermissiontable.find_unique = find_unique
|
||||
|
||||
mock_vector_store_registry = MagicMock()
|
||||
mock_vector_store_registry.get_vector_store_ids_to_run.return_value = []
|
||||
|
||||
with (
|
||||
patch( # test-quality-ok: production auth reads these module globals; no dependency injection seam exists
|
||||
"litellm.proxy.proxy_server.prisma_client", mock_prisma_client
|
||||
),
|
||||
patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists
|
||||
"litellm.vector_store_registry", mock_vector_store_registry
|
||||
),
|
||||
with patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists
|
||||
"litellm.proxy.proxy_server.prisma_client", mock_prisma_client
|
||||
):
|
||||
result = await vector_store_access_check(
|
||||
request_body={"messages": [{"role": "user", "content": "test"}]},
|
||||
|
|
@ -1668,102 +1648,6 @@ def test_can_object_call_vector_stores_scenarios(object_permissions, vector_stor
|
|||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vector_store_access_check_with_permissions():
|
||||
"""Test vector_store_access_check with actual permission checking"""
|
||||
request_body = {"tools": [{"type": "function", "function": {"name": "test"}}]}
|
||||
|
||||
# Test with valid token that has access
|
||||
valid_token = UserAPIKeyAuth(
|
||||
token="test-token",
|
||||
object_permission_id="perm-123",
|
||||
models=["gpt-4"],
|
||||
max_budget=100.0,
|
||||
)
|
||||
|
||||
mock_prisma_client = MagicMock()
|
||||
mock_permissions = LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="perm-123", vector_stores=["store-1", "store-2"]
|
||||
)
|
||||
mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=mock_permissions)
|
||||
|
||||
mock_vector_store_registry = MagicMock()
|
||||
mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["store-1"]
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),
|
||||
patch("litellm.vector_store_registry", mock_vector_store_registry),
|
||||
):
|
||||
result = await vector_store_access_check(
|
||||
request_body=request_body,
|
||||
team_object=None,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
# Test with denied access
|
||||
mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["store-3"]
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),
|
||||
patch("litellm.vector_store_registry", mock_vector_store_registry),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await vector_store_access_check(
|
||||
request_body=request_body,
|
||||
team_object=None,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
assert exc_info.value.type == ProxyErrorTypes.key_vector_store_access_denied
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vector_store_access_check_with_team_permissions():
|
||||
"""Ensure teams restricted to specific vector stores cannot access others."""
|
||||
request_body = {}
|
||||
valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None)
|
||||
|
||||
team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission")
|
||||
|
||||
mock_prisma_client = MagicMock()
|
||||
team_permissions = LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="team-permission", vector_stores=["team-store-allowed"]
|
||||
)
|
||||
mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions)
|
||||
|
||||
mock_vector_store_registry = MagicMock()
|
||||
mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["team-store-allowed"]
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),
|
||||
patch("litellm.vector_store_registry", mock_vector_store_registry),
|
||||
):
|
||||
result = await vector_store_access_check(
|
||||
request_body=request_body,
|
||||
team_object=team_object,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["team-store-denied"]
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),
|
||||
patch("litellm.vector_store_registry", mock_vector_store_registry),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await vector_store_access_check(
|
||||
request_body=request_body,
|
||||
team_object=team_object,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
assert exc_info.value.type == ProxyErrorTypes.team_vector_store_access_denied
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"requested_vector_store_id,expected_error_type",
|
||||
|
|
@ -1823,6 +1707,188 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query(
|
|||
assert exc_info.value.type == expected_error_type
|
||||
|
||||
|
||||
def _file_search_tool_request(vector_store_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": "what is in this store?",
|
||||
"tools": [{"type": "file_search", "vector_store_ids": [vector_store_id]}],
|
||||
}
|
||||
|
||||
|
||||
def _file_search_tool_string_id_request(vector_store_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": "what is in this store?",
|
||||
"tools": [{"type": "file_search", "vector_store_ids": vector_store_id}],
|
||||
}
|
||||
|
||||
|
||||
def _vector_store_ids_request(vector_store_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5.4-mini",
|
||||
"messages": [{"role": "user", "content": "what is in this store?"}],
|
||||
"vector_store_ids": [vector_store_id],
|
||||
}
|
||||
|
||||
|
||||
def _drop_params_request(vector_store_ids: object) -> dict[str, object]:
|
||||
return {
|
||||
"model": "gpt-5.4-mini",
|
||||
"messages": [{"role": "user", "content": "what is in this store?"}],
|
||||
"drop_params": True,
|
||||
"vector_store_ids": vector_store_ids,
|
||||
}
|
||||
|
||||
|
||||
def _allowlist(object_permission_id: str, vector_stores: list[str] | None) -> LiteLLM_ObjectPermissionTable | None:
|
||||
if vector_stores is None:
|
||||
return None
|
||||
return LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, vector_stores=vector_stores)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("key_vector_stores", "team_vector_stores", "request_body", "denied_by"),
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
_file_search_tool_request("vs_other_team"),
|
||||
ProxyErrorTypes.team_vector_store_access_denied,
|
||||
id="team-file-search-tool-other-store",
|
||||
),
|
||||
pytest.param(None, ["vs_allowed"], _file_search_tool_request("vs_allowed"), None, id="team-file-search-tool"),
|
||||
pytest.param(
|
||||
["vs_allowed"],
|
||||
None,
|
||||
_vector_store_ids_request("vs_other_team"),
|
||||
ProxyErrorTypes.key_vector_store_access_denied,
|
||||
id="key-vector-store-ids-other-store",
|
||||
),
|
||||
pytest.param(["vs_allowed"], None, _vector_store_ids_request("vs_allowed"), None, id="key-vector-store-ids"),
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
_file_search_tool_string_id_request("vs_other_team"),
|
||||
ProxyErrorTypes.team_vector_store_access_denied,
|
||||
id="team-file-search-tool-string-id-other-store",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
None,
|
||||
_file_search_tool_string_id_request("vs_other_team"),
|
||||
None,
|
||||
id="unrestricted-file-search-tool-string-id",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
_drop_params_request(["vs_other_team", {"id": "vs_allowed"}]),
|
||||
ProxyErrorTypes.team_vector_store_access_denied,
|
||||
id="team-drop-params-list-with-an-object-other-store",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
{
|
||||
**_file_search_tool_request("vs_other_team"),
|
||||
"tools": [{"type": "file_search", "vector_store_ids": ["vs_other_team", 1]}],
|
||||
},
|
||||
ProxyErrorTypes.team_vector_store_access_denied,
|
||||
id="team-file-search-tool-with-an-int-other-store",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
{
|
||||
**_vector_store_ids_request("vs_other_team"),
|
||||
"additional_drop_params": ["vector_store_ids"],
|
||||
"vector_store_ids": {"vs_other_team": True},
|
||||
},
|
||||
None,
|
||||
id="team-additional-drop-params-object-names-no-store",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
["vs_allowed"],
|
||||
_drop_params_request({"vs_other_team": True}),
|
||||
None,
|
||||
id="team-drop-params-object-names-no-store",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_vector_store_access_check_enforces_allowlists_on_the_stores_the_request_names(
|
||||
key_vector_stores: list[str] | None,
|
||||
team_vector_stores: list[str] | None,
|
||||
request_body: dict[str, object],
|
||||
denied_by: ProxyErrorTypes | None,
|
||||
vector_store_registry: VectorStoreRegistry | None,
|
||||
) -> None:
|
||||
valid_token: Final = UserAPIKeyAuth(
|
||||
token="team-a-key",
|
||||
team_id="team-a",
|
||||
object_permission_id="key-permission",
|
||||
object_permission=_allowlist("key-permission", key_vector_stores),
|
||||
)
|
||||
team_object: Final = LiteLLM_TeamTable(
|
||||
team_id="team-a",
|
||||
object_permission_id="team-permission",
|
||||
object_permission=_allowlist("team-permission", team_vector_stores),
|
||||
)
|
||||
with (
|
||||
patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists
|
||||
"litellm.proxy.proxy_server.prisma_client", MagicMock()
|
||||
),
|
||||
patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists
|
||||
"litellm.vector_store_registry", vector_store_registry
|
||||
),
|
||||
):
|
||||
await _assert_rag_query_outcome(
|
||||
vector_store_access_check(request_body=request_body, team_object=team_object, valid_token=valid_token),
|
||||
denied_by,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_litellm_settings_drop_params_skips_a_malformed_vector_store_ids(
|
||||
vector_store_registry: VectorStoreRegistry | None,
|
||||
) -> None:
|
||||
with patch("litellm.drop_params", True): # test-quality-ok: litellm_settings.drop_params is this module global
|
||||
assert await _common_checks_for_rag_query(
|
||||
{},
|
||||
_virtual_key(object_permission_id="key-permission"),
|
||||
SimpleNamespace(vector_stores=["KBSTOREA"]),
|
||||
request_body={
|
||||
"model": "gpt-4o-mini",
|
||||
"input": "what is in this KB?",
|
||||
"vector_store_ids": {"KBSTOREB": True},
|
||||
},
|
||||
vector_store_registry=vector_store_registry,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params",
|
||||
[{"drop_params": True}, {"additional_drop_params": ["vector_store_ids"]}],
|
||||
ids=["drop-params", "additional-drop-params"],
|
||||
)
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_a_deployment_that_drops_vector_store_ids_skips_a_malformed_one(
|
||||
deployment_params: Mapping[str, object], vector_store_registry: VectorStoreRegistry | None
|
||||
) -> None:
|
||||
assert await _common_checks_for_rag_query(
|
||||
{},
|
||||
_virtual_key(object_permission_id="key-permission"),
|
||||
SimpleNamespace(vector_stores=["KBSTOREA"]),
|
||||
request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", "vector_store_ids": ["KBSTOREA", 1]},
|
||||
vector_store_registry=vector_store_registry,
|
||||
llm_router=_gpt_4o_mini_router(deployment_params),
|
||||
)
|
||||
|
||||
|
||||
_KEY_DENIED: Final = ProxyErrorTypes.key_vector_store_access_denied
|
||||
_TEAM_DENIED: Final = ProxyErrorTypes.team_vector_store_access_denied
|
||||
_USER_DENIED: Final = ProxyErrorTypes.user_vector_store_access_denied
|
||||
|
|
@ -1866,6 +1932,7 @@ async def _common_checks_for_rag_query(
|
|||
request_body: Mapping[str, object] | None = None,
|
||||
user_api_key_cache: UserApiKeyCache | None = None,
|
||||
vector_store_registry: VectorStoreRegistry | None = None,
|
||||
llm_router: "Router | None" = None,
|
||||
) -> bool:
|
||||
permissions: Final = {
|
||||
"key-permission": _permission_row("key-permission", key_permission),
|
||||
|
|
@ -1906,13 +1973,24 @@ async def _common_checks_for_rag_query(
|
|||
global_proxy_spend=None,
|
||||
general_settings=dict(general_settings),
|
||||
route="/v1/rag/query",
|
||||
llm_router=None,
|
||||
llm_router=llm_router,
|
||||
proxy_logging_obj=MagicMock(),
|
||||
valid_token=valid_token,
|
||||
request=MagicMock(spec=Request),
|
||||
)
|
||||
|
||||
|
||||
def _gpt_4o_mini_router(litellm_params: Mapping[str, object]) -> "Router":
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test", **litellm_params},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _assert_rag_query_outcome(request: Awaitable[bool], denied_by: ProxyErrorTypes | None) -> None:
|
||||
if denied_by is None:
|
||||
assert await request is True
|
||||
|
|
@ -2343,6 +2421,175 @@ async def test_deny_by_default_rejects_malformed_vector_store_ids_as_bad_request
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_body",
|
||||
[
|
||||
{"vector_store_ids": {"KBSTOREB": True}},
|
||||
{"vector_store_ids": ["KBSTOREA", {"id": "KBSTOREB"}]},
|
||||
{"vector_store_ids": 7},
|
||||
],
|
||||
ids=["object-top-level-ids", "object-top-level-id", "int-top-level-ids"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"general_settings", [{"vector_store_deny_by_default": True}, {}], ids=["deny-by-default", "legacy"]
|
||||
)
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_malformed_vector_store_ids_are_rejected_as_bad_request(
|
||||
request_body: Mapping[str, object],
|
||||
general_settings: Mapping[str, object],
|
||||
vector_store_registry: VectorStoreRegistry | None,
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _common_checks_for_rag_query(
|
||||
general_settings,
|
||||
_virtual_key(object_permission_id="key-permission"),
|
||||
SimpleNamespace(vector_stores=["KBSTOREA"]),
|
||||
request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body},
|
||||
vector_store_registry=vector_store_registry,
|
||||
llm_router=_gpt_4o_mini_router({}),
|
||||
)
|
||||
assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
|
||||
"invalid_request_error",
|
||||
"vector_store_ids",
|
||||
"400",
|
||||
)
|
||||
|
||||
|
||||
_MALFORMED_TOP_LEVEL_IDS: Final = pytest.mark.parametrize(
|
||||
"vector_store_ids",
|
||||
[{"KBSTOREB": True}, ["KBSTOREA", {"id": "KBSTOREB"}], 7],
|
||||
ids=["object-top-level-ids", "object-top-level-id", "int-top-level-ids"],
|
||||
)
|
||||
|
||||
|
||||
async def _team_key_query_with_top_level_ids(
|
||||
key_vector_stores: list[str] | None,
|
||||
team_vector_stores: list[str] | None,
|
||||
vector_store_ids: object,
|
||||
vector_store_registry: VectorStoreRegistry | None,
|
||||
) -> bool:
|
||||
return await _common_checks_for_rag_query(
|
||||
{},
|
||||
_virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission", team_id="team-1"),
|
||||
None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores),
|
||||
LiteLLM_TeamTable(
|
||||
team_id="team-1", object_permission_id=None if team_vector_stores is None else "team-permission"
|
||||
),
|
||||
None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores),
|
||||
request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", "vector_store_ids": vector_store_ids},
|
||||
vector_store_registry=vector_store_registry,
|
||||
llm_router=_gpt_4o_mini_router({}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"vector_store_ids",
|
||||
["KBSTOREB", {"KBSTOREB": True}, ["KBSTOREA", {"id": "KBSTOREB"}]],
|
||||
ids=["string-top-level-ids", "object-top-level-ids", "object-top-level-id"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("key_vector_stores", "team_vector_stores"),
|
||||
[(None, None), ([], [])],
|
||||
ids=["no-permission-rows", "empty-allowlists"],
|
||||
)
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_legacy_mode_passes_malformed_vector_store_ids_through_without_an_allowlist(
|
||||
vector_store_ids: object,
|
||||
key_vector_stores: list[str] | None,
|
||||
team_vector_stores: list[str] | None,
|
||||
vector_store_registry: VectorStoreRegistry | None,
|
||||
):
|
||||
assert (
|
||||
await _team_key_query_with_top_level_ids(
|
||||
key_vector_stores, team_vector_stores, vector_store_ids, vector_store_registry
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_MALFORMED_TOP_LEVEL_IDS
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_legacy_mode_rejects_malformed_vector_store_ids_under_a_team_allowlist(
|
||||
vector_store_ids: object, vector_store_registry: VectorStoreRegistry | None
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _team_key_query_with_top_level_ids(None, ["KBSTOREA"], vector_store_ids, vector_store_registry)
|
||||
assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
|
||||
"invalid_request_error",
|
||||
"vector_store_ids",
|
||||
"400",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("vector_store_ids", "denied_by"), [("KBSTOREA", None), ("KBSTOREB", _TEAM_DENIED)])
|
||||
@pytest.mark.parametrize("vector_store_registry", [VectorStoreRegistry(), None], ids=["registry", "no-registry"])
|
||||
async def test_legacy_mode_checks_a_top_level_vector_store_ids_string_as_one_store(
|
||||
vector_store_ids: str, denied_by: ProxyErrorTypes | None, vector_store_registry: VectorStoreRegistry | None
|
||||
):
|
||||
await _assert_rag_query_outcome(
|
||||
_team_key_query_with_top_level_ids(None, ["KBSTOREA"], vector_store_ids, vector_store_registry), denied_by
|
||||
)
|
||||
|
||||
|
||||
def _managed_vector_store_id(unified_uuid: str, provider_vector_store_id: str) -> str:
|
||||
return encode_unified_id(
|
||||
generate_unified_id_string(
|
||||
resource_type="vector_store",
|
||||
unified_uuid=unified_uuid,
|
||||
target_model_names=["gpt-4o-mini"],
|
||||
provider_resource_id=provider_vector_store_id,
|
||||
model_id="deployment-1",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("team_vector_stores", "requested_id", "denied_by"),
|
||||
[
|
||||
(["KBSTOREA"], _managed_vector_store_id("uuid-a", "KBSTOREA"), None),
|
||||
([_managed_vector_store_id("uuid-a", "KBSTOREA")], "KBSTOREA", None),
|
||||
([_managed_vector_store_id("uuid-a", "KBSTOREA")], _managed_vector_store_id("uuid-a2", "KBSTOREA"), None),
|
||||
(["KBSTOREA"], _managed_vector_store_id("uuid-b", "KBSTOREB"), _TEAM_DENIED),
|
||||
(["uuid-a"], _managed_vector_store_id("uuid-a", "KBSTOREB"), _TEAM_DENIED),
|
||||
([_managed_vector_store_id("uuid-a", "KBSTOREA")], "KBSTOREB", _TEAM_DENIED),
|
||||
(["KBSTOREA"], "litellm_proxy:vector_store;unified_id,uuid-a;resource_id,KBSTOREA", _TEAM_DENIED),
|
||||
],
|
||||
ids=[
|
||||
"provider-id-allows-its-managed-id",
|
||||
"managed-id-allows-its-provider-id",
|
||||
"managed-ids-for-one-provider-store",
|
||||
"managed-id-for-another-store",
|
||||
"managed-uuid-cannot-vouch-for-another-store",
|
||||
"managed-id-for-another-provider-store",
|
||||
"undecoded-managed-id-is-its-own-store",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("deny_by_default", [True, False], ids=["strict", "default"])
|
||||
async def test_allowlist_compares_a_managed_vector_store_id_by_the_provider_store_it_names(
|
||||
team_vector_stores: list[str], requested_id: str, denied_by: ProxyErrorTypes | None, deny_by_default: bool
|
||||
):
|
||||
await _assert_rag_query_outcome(
|
||||
_common_checks_for_rag_query(
|
||||
{"vector_store_deny_by_default": deny_by_default},
|
||||
_virtual_key(object_permission_id="key-permission", team_id="team-1"),
|
||||
SimpleNamespace(vector_stores=[requested_id]),
|
||||
LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission"),
|
||||
SimpleNamespace(vector_stores=team_vector_stores),
|
||||
request_body={
|
||||
"model": "gpt-4o-mini",
|
||||
"input": "what is in this KB?",
|
||||
"tools": [{"type": "file_search", "vector_store_ids": [requested_id]}],
|
||||
},
|
||||
),
|
||||
denied_by,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("request_body", "denied_by"),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import os
|
|||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
|
|
@ -37,6 +37,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
|
|||
team_member_budget_entity_id,
|
||||
)
|
||||
from litellm.proxy._types import (
|
||||
JWTAuthBuilderResult,
|
||||
LiteLLMRoutes,
|
||||
LiteLLM_JWTAuth,
|
||||
LiteLLM_BudgetTable,
|
||||
|
|
@ -55,7 +56,7 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
JWTRoutingOverride,
|
||||
)
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
OrganizationNotFoundError,
|
||||
TeamNotFoundError,
|
||||
|
|
@ -724,6 +725,44 @@ async def test_user_custom_auth_runs_post_custom_auth_checks_when_opt_in():
|
|||
litellm.enable_post_custom_auth_checks = original_flag
|
||||
|
||||
|
||||
async def _authenticate_through_custom_auth() -> UserAPIKeyAuth:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_builder
|
||||
|
||||
request: Final = Request(scope={"type": "http"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
return await user_api_key_auth_builder(
|
||||
request=request,
|
||||
api_key="Bearer sk-custom-auth-shared",
|
||||
azure_api_key_header="",
|
||||
anthropic_api_key_header=None,
|
||||
google_ai_studio_api_key_header=None,
|
||||
azure_apim_header=None,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_custom_auth_returning_one_shared_token_keeps_each_requests_end_user_apart(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
|
||||
shared_token: Final = UserAPIKeyAuth(api_key="sk-custom-auth-shared", user_id="custom-user-123")
|
||||
proxy_attrs: Final = _proxy_server_attrs_for_custom_auth(user_custom_auth=AsyncMock(return_value=shared_token))
|
||||
for attr, val in proxy_attrs.items():
|
||||
monkeypatch.setattr(_proxy_server_mod, attr, val, raising=False)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", False, raising=False)
|
||||
|
||||
capped_end_users_request: Final = await _authenticate_through_custom_auth()
|
||||
capped_end_users_request.end_user_id = "eu-capped"
|
||||
capped_end_users_request.end_user_model_max_budget = {"gpt-4o": {"max_budget": 0.00001}}
|
||||
next_end_users_request: Final = await _authenticate_through_custom_auth()
|
||||
|
||||
assert (next_end_users_request.end_user_id, next_end_users_request.end_user_model_max_budget) == (None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enterprise_custom_auth_skips_post_custom_auth_checks_by_default():
|
||||
"""
|
||||
|
|
@ -5293,6 +5332,559 @@ async def test_centralized_common_checks_enforces_team_model_max_budget_from_the
|
|||
assert limiter.calls == [("t1", team_caps, key_caps, "gpt-4o")]
|
||||
|
||||
|
||||
_END_USER_MODEL_CAPS: Final = {"gpt-4o": {"budget_limit": 0.00004, "time_period": "1d"}}
|
||||
|
||||
|
||||
def _virtual_key_token_for_end_user() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(api_key="sk-test", token="hashed", end_user_id="eu-1")
|
||||
|
||||
|
||||
def _master_key_token_cached_with_another_end_users_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
user_id="admin",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
end_user_id="eu-1",
|
||||
end_user_model_max_budget={"gpt-4o": {"budget_limit": 1000.0, "time_period": "1d"}},
|
||||
)
|
||||
|
||||
|
||||
def _end_user_linked_to_the_capped_budget() -> LiteLLM_EndUserTable:
|
||||
return LiteLLM_EndUserTable(
|
||||
user_id="eu-1",
|
||||
blocked=False,
|
||||
budget_id="eu-budget",
|
||||
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-budget", model_max_budget=_END_USER_MODEL_CAPS),
|
||||
)
|
||||
|
||||
|
||||
def _end_user_on_the_proxy_default_budget() -> LiteLLM_EndUserTable:
|
||||
return LiteLLM_EndUserTable(user_id="eu-1", blocked=False)
|
||||
|
||||
|
||||
def _no_end_user_row_yet() -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _jwt_token(end_user_id: str | None, end_user_object: LiteLLM_EndUserTable | None) -> UserAPIKeyAuth:
|
||||
return JWTAuthManager.user_api_key_auth_from_result(
|
||||
JWTAuthBuilderResult(
|
||||
is_proxy_admin=False,
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
end_user_object=end_user_object,
|
||||
org_object=None,
|
||||
token="jwt",
|
||||
team_id=None,
|
||||
user_id=None,
|
||||
user_email=None,
|
||||
end_user_id=end_user_id,
|
||||
org_id=None,
|
||||
team_membership=None,
|
||||
jwt_claims={},
|
||||
agent_id=None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _jwt_token_with_an_end_user_claim() -> UserAPIKeyAuth:
|
||||
return _jwt_token(end_user_id="eu-1", end_user_object=_end_user_linked_to_the_capped_budget())
|
||||
|
||||
|
||||
def _jwt_token_naming_the_end_user_only_in_the_body() -> UserAPIKeyAuth:
|
||||
return _jwt_token(end_user_id=None, end_user_object=None)
|
||||
|
||||
|
||||
async def _run_centralized_checks_for_end_user(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
token: UserAPIKeyAuth,
|
||||
end_user: LiteLLM_EndUserTable | None,
|
||||
gpt_4o_spend: float,
|
||||
model: str,
|
||||
route: str,
|
||||
default_budget_id: str | None,
|
||||
user_custom_auth: Callable[[Request, str], Awaitable[UserAPIKeyAuth]] | None,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(litellm, "max_end_user_budget_id", default_budget_id)
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
if end_user is not None:
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=end_user,
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
for budget_id in ("eu-default", "eu-key-budget"):
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=f"default_end_user_budget:{budget_id}",
|
||||
value=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget=_END_USER_MODEL_CAPS),
|
||||
model_type=LiteLLM_BudgetTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="admin",
|
||||
value=LiteLLM_UserTable(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
model_type=LiteLLM_UserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=Litellm_EntityType.END_USER,
|
||||
entity_id="eu-1",
|
||||
budget_model="gpt-4o",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=gpt_4o_spend,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
|
||||
],
|
||||
)
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(user_custom_auth=user_custom_auth),
|
||||
"general_settings": {
|
||||
"disable_budget_reservation": True,
|
||||
"custom_auth_run_common_checks": user_custom_auth is not None,
|
||||
},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http"})
|
||||
request._url = URL(url=route)
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=token,
|
||||
request=request,
|
||||
request_data={"model": model, "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route=route,
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"),
|
||||
[
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="virtual-key-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_cached_with_another_end_users_cap,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="master-key-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_jwt_token_with_an_end_user_claim,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="jwt-claim-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_jwt_token_naming_the_end_user_only_in_the_body,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="jwt-body-user-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_no_end_user_row_yet,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="new-end-user-on-the-proxy-default-budget-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_on_the_proxy_default_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="proxy-default-budget-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.00001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
False,
|
||||
id="under-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o-mini",
|
||||
"/v1/chat/completions",
|
||||
False,
|
||||
id="model-without-a-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_cached_with_another_end_users_cap,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/spend/calculate",
|
||||
False,
|
||||
id="budget-exempt-route",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
|
||||
gpt_4o_spend: float,
|
||||
model: str,
|
||||
route: str,
|
||||
must_raise: bool,
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=make_end_user(),
|
||||
gpt_4o_spend=gpt_4o_spend,
|
||||
model=model,
|
||||
route=route,
|
||||
default_budget_id="eu-default",
|
||||
user_custom_auth=None,
|
||||
)
|
||||
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
|
||||
|
||||
|
||||
def _virtual_key_token_assigning_end_users_a_budget() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="sk-test", token="hashed", end_user_id="eu-1", metadata={"end_user_budget_id": "eu-key-budget"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "make_end_user"),
|
||||
[
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
id="end-users-own-budget",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_assigning_end_users_a_budget,
|
||||
_end_user_on_the_proxy_default_budget,
|
||||
id="keys-end-user-budget",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_assigning_end_users_a_budget,
|
||||
_no_end_user_row_yet,
|
||||
id="keys-end-user-budget-for-a-new-end-user",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget_without_a_proxy_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=make_end_user(),
|
||||
gpt_4o_spend=0.0001,
|
||||
model="gpt-4o",
|
||||
route="/v1/chat/completions",
|
||||
default_budget_id=None,
|
||||
user_custom_auth=None,
|
||||
)
|
||||
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
|
||||
|
||||
|
||||
async def _custom_auth_that_is_never_called(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
raise AssertionError("the shared checks never call custom auth")
|
||||
|
||||
|
||||
def _token_built_by_custom_auth_with_its_own_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="sk-custom", end_user_id="eu-1", end_user_model_max_budget=_END_USER_MODEL_CAPS
|
||||
).model_copy(update={"authenticated_by_custom_auth": True})
|
||||
|
||||
|
||||
def _master_key_token_custom_auth_handed_back_with_another_end_users_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
user_id="admin",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
end_user_id="eu-1",
|
||||
end_user_model_max_budget=_END_USER_MODEL_CAPS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "must_raise", "expected_cap"),
|
||||
[
|
||||
pytest.param(
|
||||
_token_built_by_custom_auth_with_its_own_cap,
|
||||
True,
|
||||
_END_USER_MODEL_CAPS,
|
||||
id="custom-auth-token-keeps-its-own-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_custom_auth_handed_back_with_another_end_users_cap,
|
||||
False,
|
||||
None,
|
||||
id="key-auth-token-behind-custom-auth-drops-another-end-users-cap",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_keeps_only_a_cap_custom_auth_set(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
must_raise: bool,
|
||||
expected_cap: Mapping[str, object] | None,
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=_end_user_on_the_proxy_default_budget(),
|
||||
gpt_4o_spend=0.0001,
|
||||
model="gpt-4o",
|
||||
route="/v1/chat/completions",
|
||||
default_budget_id=None,
|
||||
user_custom_auth=_custom_auth_that_is_never_called,
|
||||
)
|
||||
assert token.end_user_model_max_budget == expected_cap
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_holds_the_end_user_to_the_model_a_team_member_fallback_picked(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
budget_id: Final = "member-budget-1"
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=team_membership_reservation_cache_key(user_id="u1", team_id="t1"),
|
||||
value=LiteLLM_TeamMembership(user_id="u1", team_id="t1", budget_id=budget_id),
|
||||
model_type=LiteLLM_TeamMembership,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="team_id:t1",
|
||||
value=LiteLLM_TeamTableCachedObj(
|
||||
team_id="t1",
|
||||
models=["gpt-4o", "gpt-4o-mini"],
|
||||
metadata={"team_member_budget_id": budget_id},
|
||||
),
|
||||
model_type=LiteLLM_TeamTableCachedObj,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=f"team_member_default_budget:{budget_id}",
|
||||
value=LiteLLM_BudgetTable(
|
||||
budget_id=budget_id,
|
||||
model_max_budget={
|
||||
"gpt-4o": {"max_budget": 0.00004, "budget_duration": "1d"},
|
||||
"gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"},
|
||||
},
|
||||
),
|
||||
model_type=LiteLLM_BudgetTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="u1",
|
||||
value=LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
model_type=LiteLLM_UserTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=_end_user_linked_to_the_capped_budget(),
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
for entity_type, entity_id in (
|
||||
(Litellm_EntityType.TEAM_MEMBER, team_member_budget_entity_id(user_id="u1", team_id="t1")),
|
||||
(Litellm_EntityType.END_USER, "eu-1"),
|
||||
):
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model="gpt-4o",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=0.0001,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
|
||||
],
|
||||
fallbacks=[{"gpt-4o": ["gpt-4o-mini"]}],
|
||||
)
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(),
|
||||
"general_settings": {"disable_budget_reservation": True},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
|
||||
request._url = URL(url="/v1/chat/completions")
|
||||
request_data: Final = {"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]}
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
token="hashed",
|
||||
user_id="u1",
|
||||
team_id="t1",
|
||||
end_user_id="eu-1",
|
||||
models=["gpt-4o", "gpt-4o-mini"],
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
),
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
route="/v1/chat/completions",
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
assert request_data["model"] == "gpt-4o-mini"
|
||||
|
||||
|
||||
def _access_groups_only_team_t1_puts_gpt_4o_in(model: str, team_id: str | None) -> frozenset[str]:
|
||||
return frozenset({"t1-group"}) if (model, team_id) == ("gpt-4o", "t1") else frozenset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "must_raise"),
|
||||
[
|
||||
pytest.param("t1", True, id="team-whose-access-group-the-end-user-capped"),
|
||||
pytest.param(None, False, id="no-team-so-the-group-cap-does-not-apply"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_applies_an_end_user_cap_on_the_tokens_team_access_group(
|
||||
team_id: str | None,
|
||||
must_raise: bool,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
group_cap: Final = {"t1-group": {"budget_limit": 0.00004, "time_period": "1d"}}
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="team_id:t1",
|
||||
value=LiteLLM_TeamTableCachedObj(team_id="t1", models=["gpt-4o"]),
|
||||
model_type=LiteLLM_TeamTableCachedObj,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=LiteLLM_EndUserTable(
|
||||
user_id="eu-1",
|
||||
blocked=False,
|
||||
budget_id="eu-group-budget",
|
||||
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-group-budget", model_max_budget=group_cap),
|
||||
),
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(
|
||||
dual_cache=DualCache(), model_access_groups=_access_groups_only_team_t1_puts_gpt_4o_in
|
||||
)
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=Litellm_EntityType.END_USER,
|
||||
entity_id="eu-1",
|
||||
budget_model="t1-group",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=0.0001,
|
||||
)
|
||||
router: Final = litellm.Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}}])
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(),
|
||||
"general_settings": {"disable_budget_reservation": True},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
|
||||
request._url = URL(url="/v1/chat/completions")
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=UserAPIKeyAuth(
|
||||
api_key="sk-test", token="hashed", team_id=team_id, end_user_id="eu-1", models=["gpt-4o"]
|
||||
),
|
||||
request=request,
|
||||
request_data={"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route="/v1/chat/completions",
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_loads_member_budget_and_exempts_discovery_routes():
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ _STOCK_MODELS = {
|
|||
"base_instructions": "You are Codex, a coding agent based on GPT-5.6.",
|
||||
"apply_patch_tool_type": "freeform",
|
||||
"supports_parallel_tool_calls": True,
|
||||
"supports_search_tool": True,
|
||||
"tool_mode": "code_mode_only",
|
||||
"context_window": 272000,
|
||||
"comp_hash": "terra-hash",
|
||||
},
|
||||
|
|
@ -669,10 +671,16 @@ class TestCodexModelSync:
|
|||
def _row(model_id, **extra):
|
||||
return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra}
|
||||
|
||||
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None):
|
||||
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, model_info=None):
|
||||
captured = {}
|
||||
|
||||
def fake_get(url, headers, timeout):
|
||||
if headers.get("Authorization") != "Bearer sk-key":
|
||||
return _FakeResponse(401)
|
||||
if url.endswith("/v1/model/info"):
|
||||
if isinstance(model_info, Exception):
|
||||
raise model_info
|
||||
return model_info if model_info is not None else _FakeResponse(200, {"data": []})
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return _FakeResponse(200, listing)
|
||||
|
|
@ -737,6 +745,202 @@ class TestCodexModelSync:
|
|||
assert entries["gpt-5.4"]["upgrade"] == _STOCK_MODELS["gpt-5.4"]["upgrade"]
|
||||
assert [entries["gpt-5.4"]["priority"], entries["gpt-5.6-terra"]["priority"]] == [0, 1]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment",
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}, "model_info": None},
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": ""},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_team-1_d1",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://api.openai.com/v1"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
|
||||
},
|
||||
],
|
||||
ids=["provider_prefixed", "bare_without_model_info", "empty_team_public_name", "team_public_model_name"],
|
||||
)
|
||||
def test_alias_of_a_codex_model_gets_that_models_entry_under_its_own_name(self, tmp_path, deployment):
|
||||
listing = self._listing(self._row("api-terra"), self._row("m"))
|
||||
_, result = self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
|
||||
|
||||
assert isinstance(result, ModelSyncArgs)
|
||||
entries = self._entries(tmp_path)
|
||||
assert entries["api-terra"] == {
|
||||
**_STOCK_MODELS["gpt-5.6-terra"],
|
||||
"slug": "api-terra",
|
||||
"display_name": "api-terra",
|
||||
"priority": 0,
|
||||
"service_tiers": [],
|
||||
"default_service_tier": None,
|
||||
"default_verbosity": None,
|
||||
"availability_nux": None,
|
||||
"default_reasoning_level": None,
|
||||
}
|
||||
assert "supports_search_tool" not in entries["m"]
|
||||
|
||||
def test_alias_gets_no_upgrade_nudge_away_from_its_own_name(self, tmp_path):
|
||||
listing = self._listing(self._row("gpt-5.6-terra"), self._row("api-54"))
|
||||
deployment = {"model_name": "api-54", "litellm_params": {"model": "openai/gpt-5.4"}}
|
||||
self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
|
||||
|
||||
entry = self._entries(tmp_path)["api-54"]
|
||||
assert (entry["comp_hash"], entry["upgrade"]) == (_STOCK_MODELS["gpt-5.4"]["comp_hash"], None)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pins", "expected"),
|
||||
[
|
||||
(("high",), "high"),
|
||||
(("high", "high"), "high"),
|
||||
(("high", None), None),
|
||||
(("high", "low"), None),
|
||||
(({"effort": "high"},), None),
|
||||
],
|
||||
ids=["one_deployment", "every_deployment_agrees", "one_unpinned", "pins_differ", "non_string_pin"],
|
||||
)
|
||||
def test_alias_starts_at_the_reasoning_effort_every_deployment_pins(self, tmp_path, pins, expected):
|
||||
deployments = [
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", **({"reasoning_effort": p} if p else {})},
|
||||
}
|
||||
for p in pins
|
||||
]
|
||||
self._sync(
|
||||
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
|
||||
)
|
||||
|
||||
entry = self._entries(tmp_path)["api-terra"]
|
||||
assert (entry["comp_hash"], entry["default_reasoning_level"]) == ("terra-hash", expected)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments",
|
||||
[
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}},
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "openai"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.5"},
|
||||
"model_info": {"id": "paused", "blocked": True},
|
||||
},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://eu.api.openai.com/v1"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6-terra",
|
||||
"api_base": "https://api.openai.com/v1",
|
||||
"litellm_credential_name": "openai-key",
|
||||
},
|
||||
}
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"same_model_with_and_without_prefix",
|
||||
"openai_custom_llm_provider",
|
||||
"paused_deployment_ignored",
|
||||
"openai_regional_host",
|
||||
"credential_under_an_openai_host",
|
||||
],
|
||||
)
|
||||
def test_alias_whose_deployments_all_route_to_one_openai_model_gets_its_entry(self, tmp_path, deployments):
|
||||
self._sync(
|
||||
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
|
||||
)
|
||||
|
||||
assert self._entries(tmp_path)["api-terra"]["comp_hash"] == _STOCK_MODELS["gpt-5.6-terra"]["comp_hash"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments",
|
||||
[
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.5"}},
|
||||
{
|
||||
"model_name": "model_name_team-1_d1",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
|
||||
},
|
||||
],
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "anthropic/claude-opus-4-7"}},
|
||||
],
|
||||
[{"model_name": "api-terra", "litellm_params": {"model": "azure/gpt-5.6-terra"}}],
|
||||
[{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "azure"}}],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "http://vllm.example.invalid/v1"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "gpt-5.6-terra", "api_base": "https://api.openai.com.example.invalid"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "litellm_credential_name": "gateway"},
|
||||
}
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"team_row_and_global_row_differ",
|
||||
"load_balanced_across_providers",
|
||||
"azure_prefix",
|
||||
"azure_provider",
|
||||
"openai_prefix_on_another_host",
|
||||
"openai_lookalike_host",
|
||||
"host_only_a_credential_knows",
|
||||
],
|
||||
)
|
||||
def test_alias_without_one_shared_openai_upstream_keeps_the_fallback_entry(self, tmp_path, deployments):
|
||||
listing = self._listing(self._row("api-terra"))
|
||||
self._sync(listing, tmp_path / "with", model_info=_FakeResponse(200, {"data": deployments}))
|
||||
self._sync(listing, tmp_path / "without")
|
||||
|
||||
assert self._entries(tmp_path / "with")["api-terra"] == self._entries(tmp_path / "without")["api-terra"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
requests.ConnectionError("refused"),
|
||||
_FakeResponse(500, {"data": [{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}}]}),
|
||||
_FakeResponse(200, {"data": "nope"}),
|
||||
_FakeResponse(200, {"data": [{"model_name": "api-terra"}]}),
|
||||
],
|
||||
ids=["unreachable", "http_500", "unexpected_body", "deployment_without_litellm_params"],
|
||||
)
|
||||
def test_unavailable_model_info_keeps_the_alias_on_the_fallback_entry(self, tmp_path, model_info):
|
||||
_, result = self._sync(self._listing(self._row("api-terra")), tmp_path, model_info=model_info)
|
||||
|
||||
catalog = tmp_path / "litellm-models.json"
|
||||
assert result == ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(catalog))}"))
|
||||
entry = self._entries(tmp_path)["api-terra"]
|
||||
assert entry["slug"] == "api-terra"
|
||||
assert "supports_search_tool" not in entry
|
||||
|
||||
def test_stock_catalog_is_decoded_as_utf8_regardless_of_locale(self, tmp_path):
|
||||
description = "Modelo equilibrado para el trabajo diario, con acentos y ñ."
|
||||
catalog = {"models": [{**_STOCK_MODELS["gpt-5.5"], "description": description}]}
|
||||
|
|
|
|||
|
|
@ -135,6 +135,84 @@ def test_alias_of_a_known_upstream_model_takes_stock_metadata_under_its_own_name
|
|||
assert (entry["slug"], entry["display_name"]) == ("team-55", "team-55")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("max_input_tokens", "expected"), [(100000, 100000), (922000, 272000), (None, 272000)])
|
||||
def test_alias_with_deployment_defaults_caps_codexs_window_at_its_listed_max_input_tokens(max_input_tokens, expected):
|
||||
stock = MappingProxyType({"gpt-5.5": _stock("gpt-5.5").model_copy(update={"context_window": 272000})})
|
||||
alias = _row("team-55", upstream_model="openai/gpt-5.5", max_input_tokens=max_input_tokens)
|
||||
|
||||
body = codex_models_response_json((alias,), stock=stock, instructions=_PROMPT, alias_defaults="deployment")
|
||||
|
||||
assert json.loads(body.json)["models"][0]["context_window"] == expected
|
||||
|
||||
|
||||
def test_alias_with_deployment_defaults_drops_codexs_tier_and_upgrade_while_the_native_name_keeps_them():
|
||||
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"), _row("team-55", upstream_model="gpt-5.5"))
|
||||
codex = {entry["slug"]: entry for entry in _models(*rows)}
|
||||
deployment = {entry["slug"]: entry for entry in _models(*rows, alias_defaults="deployment")}
|
||||
|
||||
assert deployment["gpt-6-sol"] == codex["gpt-6-sol"]
|
||||
assert (codex["team-sol"]["default_service_tier"], codex["team-55"]["upgrade"]) == (
|
||||
"priority",
|
||||
{"model": "gpt-6-sol"},
|
||||
)
|
||||
assert _tiers(codex["team-sol"]) == _tiers(codex["gpt-6-sol"]) != []
|
||||
assert [
|
||||
(deployment[slug]["service_tiers"], deployment[slug]["default_service_tier"])
|
||||
for slug in ("team-sol", "team-55")
|
||||
] == [
|
||||
([], None),
|
||||
([], None),
|
||||
]
|
||||
assert deployment["team-55"]["upgrade"] is None
|
||||
assert deployment["team-sol"]["model_messages"] == codex["team-sol"]["model_messages"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("pinned", "expected"), [("high", "high"), ("turbo", None), (None, None)])
|
||||
def test_alias_with_deployment_defaults_starts_only_at_a_reasoning_effort_its_deployments_pin(pinned, expected):
|
||||
stock = CodexStockModel.model_validate(
|
||||
{
|
||||
"slug": "gpt-5.5",
|
||||
"display_name": "GPT-5.5",
|
||||
"priority": 1,
|
||||
"visibility": "list",
|
||||
"default_reasoning_level": "medium",
|
||||
"supported_reasoning_levels": [{"effort": e, "description": e} for e in ("low", "medium", "high")],
|
||||
}
|
||||
)
|
||||
alias = _row("team-55", upstream_model="gpt-5.5", reasoning_effort=pinned)
|
||||
|
||||
body = codex_models_response_json(
|
||||
(alias,), stock=MappingProxyType({"gpt-5.5": stock}), instructions=_PROMPT, alias_defaults="deployment"
|
||||
)
|
||||
|
||||
assert json.loads(body.json)["models"][0]["default_reasoning_level"] == expected
|
||||
|
||||
|
||||
def test_alias_with_deployment_defaults_sends_no_verbosity_or_promo_of_codexs_while_the_native_name_keeps_them():
|
||||
stock = CodexStockModel.model_validate(
|
||||
{
|
||||
"slug": "gpt-6-sol",
|
||||
"display_name": "GPT-6 Sol",
|
||||
"priority": 1,
|
||||
"visibility": "list",
|
||||
"support_verbosity": True,
|
||||
"default_verbosity": "low",
|
||||
"availability_nux": {"message": "GPT-6 Sol is here"},
|
||||
}
|
||||
)
|
||||
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"))
|
||||
|
||||
body = codex_models_response_json(
|
||||
rows, stock=MappingProxyType({"gpt-6-sol": stock}), instructions=_PROMPT, alias_defaults="deployment"
|
||||
)
|
||||
native, alias = json.loads(body.json)["models"]
|
||||
|
||||
assert [(e["support_verbosity"], e["default_verbosity"], e["availability_nux"]) for e in (native, alias)] == [
|
||||
(True, "low", {"message": "GPT-6 Sol is here"}),
|
||||
(True, None, None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("row", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,16 +1,26 @@
|
|||
import json
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import prisma.errors
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from prisma._async_http import Response
|
||||
from prisma.engine.utils import handle_response_errors
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
error_status_code,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
proxy_exception_for,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.types.router import RouterErrors
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -203,3 +213,287 @@ def test_a_proxy_error_keeps_the_call_id_it_was_raised_with():
|
|||
|
||||
assert with_litellm_call_id(raised_with_id, "second").headers == {"x-litellm-call-id": "first"}
|
||||
assert with_litellm_call_id(ProxyException(message="nope", type="None", param=None, code=400), None).headers == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raised_status, expected_type",
|
||||
[
|
||||
(400, "invalid_request_error"),
|
||||
(404, "invalid_request_error"),
|
||||
(401, "auth_error"),
|
||||
(403, "auth_error"),
|
||||
(422, "invalid_request_error"),
|
||||
(500, "internal_server_error"),
|
||||
],
|
||||
)
|
||||
def test_a_catch_all_handler_keeps_the_status_its_route_answered_with(raised_status: int, expected_type: str) -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
HTTPException(status_code=raised_status, detail={"message": "boom"}),
|
||||
default_status_code=400,
|
||||
)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(raised_status), expected_type)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_status_code, expected_type", [(400, "invalid_request_error"), (500, "internal_server_error")]
|
||||
)
|
||||
def test_an_exception_carrying_no_status_falls_back_to_the_routes_default(default_status_code: int, expected_type: str) -> None:
|
||||
wrapped: Final = proxy_exception_for(ValueError("boom"), default_status_code=default_status_code)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(default_status_code), expected_type)
|
||||
|
||||
|
||||
def test_a_proxy_exception_passes_through_with_the_type_it_already_named() -> None:
|
||||
raised: Final = ProxyException(
|
||||
message="Budget has been exceeded",
|
||||
type=ProxyErrorTypes.budget_exceeded.value,
|
||||
param="max_budget",
|
||||
code=400,
|
||||
)
|
||||
|
||||
assert proxy_exception_for(raised, default_status_code=500) is raised
|
||||
|
||||
|
||||
def test_the_message_is_what_the_route_raised_without_an_invented_prefix() -> None:
|
||||
assert proxy_exception_for(ValueError("boom"), default_status_code=400).message == "boom"
|
||||
|
||||
|
||||
def test_an_http_exceptions_detail_survives_the_wrapping() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
HTTPException(status_code=400, detail={"error": "Invalid budget_duration 'not-a-duration'."}),
|
||||
default_status_code=400,
|
||||
)
|
||||
|
||||
assert "Invalid budget_duration" in wrapped.message
|
||||
|
||||
|
||||
def test_a_carried_type_survives_the_wrapping() -> None:
|
||||
class _Carrier(Exception):
|
||||
type: str = "context_window_exceeded"
|
||||
status_code: int = 400
|
||||
|
||||
assert proxy_exception_for(_Carrier("boom"), default_status_code=500).type == "context_window_exceeded"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc", [HTTPException(status_code=404, detail="nope"), ValueError("boom")])
|
||||
def test_the_wrapped_error_serializes_as_an_openai_error_object(exc: Exception) -> None:
|
||||
body: Final = json.loads(json.dumps({"error": proxy_exception_for(exc, default_status_code=400).to_dict()}))
|
||||
|
||||
assert body["error"]["param"] is None
|
||||
assert body["error"]["type"] == "invalid_request_error"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_failure, expected",
|
||||
[
|
||||
(
|
||||
litellm.AuthenticationError(message="invalid x-api-key", llm_provider="anthropic", model="claude-haiku-5-5"),
|
||||
("401", "authentication_error"),
|
||||
),
|
||||
(
|
||||
litellm.PermissionDeniedError(
|
||||
message="model not allowed",
|
||||
llm_provider="anthropic",
|
||||
model="claude-haiku-5-5",
|
||||
response=httpx.Response(403, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")),
|
||||
),
|
||||
("403", "permission_error"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_provider_credential_failure_is_labelled_the_way_chat_completions_labels_it(
|
||||
provider_failure: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
wrapped: Final = proxy_exception_for(provider_failure, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [401, 403])
|
||||
def test_a_rejection_the_route_raised_keeps_the_auth_label(status_code: int) -> None:
|
||||
wrapped: Final = proxy_exception_for(HTTPException(status_code=status_code, detail="no"), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(status_code), "auth_error")
|
||||
|
||||
|
||||
def test_an_exception_without_a_message_is_named_by_its_type() -> None:
|
||||
wrapped: Final = proxy_exception_for(TimeoutError(), default_status_code=500)
|
||||
|
||||
assert wrapped.message == "TimeoutError"
|
||||
|
||||
|
||||
def test_a_database_outage_answers_503_so_callers_retry_instead_of_fixing_their_request() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
prisma.errors.PrismaError("Can't reach database server at `db:5432`"), default_status_code=400
|
||||
)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"postgres_refusal",
|
||||
[
|
||||
'FATAL: database "litellm" is not currently accepting connections',
|
||||
"FATAL: the database system is starting up",
|
||||
"FATAL: the database system is shutting down",
|
||||
"FATAL: the database system is in recovery mode",
|
||||
"FATAL: the database system is not yet accepting connections",
|
||||
"FATAL: terminating connection due to administrator command",
|
||||
"ERROR: cannot execute UPDATE in a read-only transaction",
|
||||
],
|
||||
)
|
||||
def test_a_database_refusing_work_during_a_failover_answers_503(postgres_refusal: str) -> None:
|
||||
refused: Final = prisma.errors.DataError(
|
||||
{"user_facing_error": {"message": f"Error in connector: Error querying the database: {postgres_refusal}"}}
|
||||
)
|
||||
|
||||
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"localized_refusal",
|
||||
[
|
||||
{
|
||||
"error_code": "P2010",
|
||||
"message": "Raw query failed. Message: `FEHLER: kann UPDATE nicht in einer Nur-Lese-Transaktion ausführen`",
|
||||
"meta": {"code": "25006"},
|
||||
},
|
||||
{
|
||||
"message": 'PostgresError { code: "57P01", message: "Verbindung wird abgebrochen", severity: "FATAL" }',
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_a_failover_refusal_in_a_localized_postgres_answers_503(localized_refusal: dict[str, object]) -> None:
|
||||
refused: Final = prisma.errors.DataError({"user_facing_error": localized_refusal})
|
||||
|
||||
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_failover_phrase_outside_the_database_is_not_reported_as_a_database_outage() -> None:
|
||||
callback_down: Final = ValueError("FATAL: the database system is starting up")
|
||||
|
||||
wrapped: Final = proxy_exception_for(callback_down, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("400", "invalid_request_error")
|
||||
|
||||
|
||||
def test_a_database_dropped_mid_query_answers_503_though_prisma_fails_to_name_the_error() -> None:
|
||||
engine_reply_with_null_meta: Final = [{"user_facing_error": {"error_code": "P1001", "message": "", "meta": None}}]
|
||||
with pytest.raises(AttributeError) as raised:
|
||||
handle_response_errors(Response(httpx.Response(500)), engine_reply_with_null_meta)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_network_failure_outside_the_database_is_not_reported_as_a_database_outage() -> None:
|
||||
callback_unreachable: Final = httpx.ConnectError("[Errno 111] Connection refused")
|
||||
|
||||
wrapped: Final = proxy_exception_for(callback_unreachable, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
def _add_member_failing_with(db_error: Exception) -> None:
|
||||
try:
|
||||
raise db_error
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error adding member to organization: {e}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"underlying_error, expected",
|
||||
[
|
||||
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
|
||||
(httpx.ConnectError("[Errno 111] Connection refused"), ("500", "internal_server_error")),
|
||||
],
|
||||
)
|
||||
def test_a_route_that_rewraps_its_error_keeps_the_database_outage_it_wrapped(
|
||||
underlying_error: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="Error adding member") as raised:
|
||||
_add_member_failing_with(underlying_error)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
def test_a_proxy_without_prisma_installed_still_answers_with_an_error_object(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(sys.modules, "prisma", None)
|
||||
|
||||
wrapped: Final = proxy_exception_for(httpx.ConnectError("[Errno 111] Connection refused"), default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
def _look_up_team_failing_with(lookup_error: Exception) -> None:
|
||||
try:
|
||||
raise lookup_error
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Team doesn't exist in db")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"lookup_error, expected",
|
||||
[
|
||||
(DBLookupDeadlineExceeded("team", 10.0), ("503", "no_db_connection")),
|
||||
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
|
||||
(KeyError("team_id"), ("404", "invalid_request_error")),
|
||||
],
|
||||
)
|
||||
def test_a_not_found_answer_raised_while_the_database_was_down_reports_the_outage(
|
||||
lookup_error: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
with pytest.raises(HTTPException) as raised:
|
||||
_look_up_team_failing_with(lookup_error)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
def test_a_lookup_that_missed_its_deadline_answers_503() -> None:
|
||||
wrapped: Final = proxy_exception_for(DBLookupDeadlineExceeded("team", 10.0), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_database_outage_answer_does_not_reveal_where_the_database_lives() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
prisma.errors.PrismaError("Can't reach database server at `prod-db.internal:5432`"), default_status_code=400
|
||||
)
|
||||
|
||||
assert wrapped.code == "503"
|
||||
assert "prod-db.internal" not in wrapped.message
|
||||
|
||||
|
||||
def test_an_upstream_failure_answers_with_its_own_status_not_the_routes_default() -> None:
|
||||
upstream_overloaded: Final = litellm.ServiceUnavailableError(
|
||||
message="upstream overloaded", llm_provider="openai", model="gpt-5.4-mini"
|
||||
)
|
||||
|
||||
wrapped: Final = proxy_exception_for(upstream_overloaded, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "internal_server_error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_rejection, expected",
|
||||
[
|
||||
(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-5.4-mini", ("401", "auth_error")),
|
||||
(f"{RouterErrors.no_deployments_available.value}. Passed model=gpt-5.4-mini", ("429", "rate_limit_error")),
|
||||
],
|
||||
)
|
||||
def test_a_router_rejection_is_labelled_by_the_status_it_answers_with(
|
||||
router_rejection: str, expected: tuple[str, str]
|
||||
) -> None:
|
||||
wrapped: Final = proxy_exception_for(ValueError(router_rejection), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@
|
|||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from functools import reduce
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -12,14 +13,16 @@ from cryptography.hazmat.primitives import serialization
|
|||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
||||
from litellm.llms.github_copilot.per_user_auth import _SESSION_CACHE, _session_cache_key
|
||||
from litellm.models.credentials import CredentialSource, UserConnectionFlowHandle
|
||||
from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
|
||||
from litellm.proxy.credential_endpoints.endpoints import _decode_flow_handle, get_llm_router
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.utils import CredentialItem
|
||||
|
|
@ -460,6 +463,264 @@ def test_update_credential_never_syncs_a_null_into_the_in_memory_credential(rest
|
|||
assert synced.credential_values["api_key"] == "sk-rotated"
|
||||
|
||||
|
||||
_WRITTEN_JSON: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class _InstallCredentialStore(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
connected: bool = True,
|
||||
in_memory: tuple[object, ...] = (),
|
||||
llm_router: object | None = None,
|
||||
**repository_calls: AsyncMock,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def _encrypted_row(served: CredentialItem) -> CredentialItem:
|
||||
return CredentialItem(
|
||||
credential_name=served.credential_name,
|
||||
credential_values={key: encrypt_value_helper(value) for key, value in served.credential_values.items()},
|
||||
credential_info=dict(served.credential_info),
|
||||
)
|
||||
|
||||
|
||||
def _written_values(update_by_name: AsyncMock) -> dict[str, object]:
|
||||
written: Final = _WRITTEN_JSON.validate_json(update_by_name.await_args.kwargs["data"]["credential_values"])
|
||||
return {
|
||||
key: decrypt_value_helper(value, key) if isinstance(value, str) else value for key, value in written.items()
|
||||
}
|
||||
|
||||
|
||||
def _written_info(update_by_name: AsyncMock) -> dict[str, object]:
|
||||
return _WRITTEN_JSON.validate_json(update_by_name.await_args.kwargs["data"]["credential_info"])
|
||||
|
||||
|
||||
def _serve_credential(
|
||||
credential_store: _InstallCredentialStore,
|
||||
served: CredentialItem,
|
||||
in_memory: tuple[CredentialItem, ...] | None = None,
|
||||
) -> AsyncMock:
|
||||
update_by_name = AsyncMock(return_value=None)
|
||||
credential_store(
|
||||
in_memory=(served,) if in_memory is None else in_memory,
|
||||
find_by_name=AsyncMock(side_effect=lambda _name: _encrypted_row(served)),
|
||||
update_by_name=update_by_name,
|
||||
)
|
||||
return update_by_name
|
||||
|
||||
|
||||
def _service_account(
|
||||
private_key: str = "pk-1234567890", client_email: str = "svc@example.com"
|
||||
) -> dict[str, dict[str, str]]:
|
||||
return {"vertex_credentials": {"private_key": private_key, "client_email": client_email}}
|
||||
|
||||
|
||||
def test_update_credential_keeps_the_stored_secret_when_the_patch_echoes_its_masked_read_back(
|
||||
credential_store: _InstallCredentialStore,
|
||||
):
|
||||
served = CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={"api_key": "sk-real-secret-1234"},
|
||||
credential_info={"custom_llm_provider": "openai", "description": "original"},
|
||||
)
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
|
||||
response = _patch_credential(
|
||||
"existing",
|
||||
{
|
||||
"credential_values": {"api_key": "sk****34"},
|
||||
"credential_info": {"custom_llm_provider": "openai", "description": "edited"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {"api_key": "sk-real-secret-1234"}
|
||||
assert _written_info(update_by_name)["description"] == "edited"
|
||||
assert CredentialAccessor.get_credential_values("existing") == {"api_key": "sk-real-secret-1234"}
|
||||
assert litellm.credential_list[0].credential_info["description"] == "edited"
|
||||
|
||||
|
||||
def test_update_credential_keeps_the_secret_when_the_patch_echoes_what_get_by_name_served(
|
||||
credential_store: _InstallCredentialStore,
|
||||
):
|
||||
served = CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={"api_key": "sk-real-secret-1234", "api_base": "https://api.example.com"},
|
||||
credential_info={"custom_llm_provider": "openai", "description": "original"},
|
||||
)
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
read_back = _call_as("GET", "/credentials/by_name/existing").json()
|
||||
assert read_back["credential_values"]["api_key"] != "sk-real-secret-1234"
|
||||
|
||||
response = _patch_credential(
|
||||
"existing", {**read_back, "credential_info": {**read_back["credential_info"], "description": "edited"}}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {"api_key": "sk-real-secret-1234", "api_base": "https://api.example.com"}
|
||||
assert _written_info(update_by_name)["description"] == "edited"
|
||||
|
||||
|
||||
def test_update_credential_keeps_a_short_secret_whose_read_back_is_all_asterisks(
|
||||
credential_store: _InstallCredentialStore,
|
||||
):
|
||||
served = CredentialItem(credential_name="existing", credential_values={"api_key": "abc"}, credential_info={})
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
read_back = _call_as("GET", "/credentials/by_name/existing").json()
|
||||
assert read_back["credential_values"]["api_key"] != "abc"
|
||||
|
||||
response = _patch_credential(
|
||||
"existing", {"credential_values": read_back["credential_values"], "credential_info": {}}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {"api_key": "abc"}
|
||||
assert CredentialAccessor.get_credential_values("existing") == {"api_key": "abc"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rotated_values",
|
||||
(
|
||||
{"api_key": "sk-rotated-key-5678"},
|
||||
{"api_key": "new**secret"},
|
||||
_service_account(private_key="pk**rotated"),
|
||||
),
|
||||
)
|
||||
def test_update_credential_still_rotates_a_secret_sent_in_full(
|
||||
credential_store: _InstallCredentialStore, rotated_values: dict[str, object]
|
||||
):
|
||||
served = CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={"api_key": "sk-old-key-000011", **_service_account()},
|
||||
credential_info={},
|
||||
)
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
|
||||
response = _patch_credential("existing", {"credential_values": rotated_values, "credential_info": {}})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {**served.credential_values, **rotated_values}
|
||||
assert CredentialAccessor.get_credential_values("existing") == {**served.credential_values, **rotated_values}
|
||||
|
||||
|
||||
def test_update_credential_saves_a_mask_shaped_value_on_a_key_get_never_masks(
|
||||
credential_store: _InstallCredentialStore,
|
||||
):
|
||||
served = CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={
|
||||
"api_key": "sk-real-secret-1234",
|
||||
"api_base": "https://api.example.com",
|
||||
"extra_headers": {"x-team": "platform"},
|
||||
},
|
||||
credential_info={},
|
||||
)
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
sent: Final = {"api_base": "ab****cd", "organization": "*****", "extra_headers": {"x-team": "*****"}}
|
||||
assert get_masked_values(sent, unmasked_length=4, number_of_asterisks=4) == sent
|
||||
|
||||
response = _patch_credential("existing", {"credential_values": sent, "credential_info": {}})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {"api_key": "sk-real-secret-1234", **sent}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_email", ("svc@example.com", "new@example.com"))
|
||||
def test_update_credential_keeps_a_masked_secret_inside_a_stored_object(
|
||||
credential_store: _InstallCredentialStore, client_email: str
|
||||
):
|
||||
served = CredentialItem(credential_name="existing", credential_values=_service_account(), credential_info={})
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
|
||||
response = _patch_credential(
|
||||
"existing",
|
||||
{
|
||||
"credential_values": _service_account(private_key="pk****90", client_email=client_email),
|
||||
"credential_info": {},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == _service_account(client_email=client_email)
|
||||
assert CredentialAccessor.get_credential_values("existing") == _service_account(client_email=client_email)
|
||||
|
||||
|
||||
def test_update_credential_drops_a_masked_leaf_the_stored_object_no_longer_has(
|
||||
credential_store: _InstallCredentialStore,
|
||||
):
|
||||
served = CredentialItem(credential_name="existing", credential_values=_service_account(), credential_info={})
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
|
||||
response = _patch_credential(
|
||||
"existing",
|
||||
{
|
||||
"credential_values": {
|
||||
"vertex_credentials": {
|
||||
"private_key": "pk****90",
|
||||
"refresh_token": "rt****89",
|
||||
"client_email": "svc@example.com",
|
||||
}
|
||||
},
|
||||
"credential_info": {},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == _service_account()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"in_memory",
|
||||
(
|
||||
pytest.param(
|
||||
(
|
||||
CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={"api_key": "sk-old-key-000011"},
|
||||
credential_info={"description": "original"},
|
||||
),
|
||||
),
|
||||
id="replica_not_resynced",
|
||||
),
|
||||
pytest.param((), id="replica_never_loaded_it"),
|
||||
),
|
||||
)
|
||||
def test_update_credential_keeps_the_table_secret_whatever_this_replica_last_rendered(
|
||||
credential_store: _InstallCredentialStore, in_memory: tuple[CredentialItem, ...]
|
||||
):
|
||||
served = CredentialItem(
|
||||
credential_name="existing",
|
||||
credential_values={"api_key": "sk-rotated-key-000099"},
|
||||
credential_info={"description": "original"},
|
||||
)
|
||||
update_by_name = _serve_credential(credential_store, served, in_memory=in_memory)
|
||||
|
||||
response = _patch_credential(
|
||||
"existing",
|
||||
{"credential_values": {"api_key": "sk****11"}, "credential_info": {"description": "edited"}},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == {"api_key": "sk-rotated-key-000099"}
|
||||
assert _written_info(update_by_name)["description"] == "edited"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("levels", (20, 21))
|
||||
def test_update_credential_keeps_a_secret_exactly_as_deep_as_the_masker_masks(
|
||||
credential_store: _InstallCredentialStore, levels: int
|
||||
):
|
||||
nested = reduce(lambda inner, _level: {"credentials": inner}, range(levels), {"api_key": "leaf-secret-1234"})
|
||||
served = CredentialItem(credential_name="existing", credential_values=nested, credential_info={})
|
||||
update_by_name = _serve_credential(credential_store, served)
|
||||
read_back = get_masked_values(served.credential_values, unmasked_length=4, number_of_asterisks=4)
|
||||
|
||||
response = _patch_credential("existing", {"credential_values": read_back, "credential_info": {}})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _written_values(update_by_name) == nested
|
||||
|
||||
|
||||
def _generate_es256_pem() -> str:
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
return key.private_bytes(
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from starlette.requests import Request
|
||||
|
||||
from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens
|
||||
|
|
@ -105,3 +105,91 @@ async def test_proxy_gemini_to_openai_like_model_token_counting():
|
|||
)
|
||||
|
||||
assert response.get("totalTokens") > 0
|
||||
|
||||
|
||||
_DECLARATION: Final = {"name": "lookup", "description": "Find a record"}
|
||||
_SYSTEM: Final = {"systemInstruction": {"parts": [{"text": "Follow these rules"}]}}
|
||||
_NO_SYSTEM_PARTS: Final = {"systemInstruction": {"parts": []}}
|
||||
_TOOLS: Final = {"tools": [{"functionDeclarations": [_DECLARATION]}]}
|
||||
_TURN: Final = {"role": "user", "parts": [{"text": "Count this request"}]}
|
||||
_CONTENTS: Final = {"contents": [_TURN]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("spelled", "canonical"),
|
||||
(
|
||||
({"systemInstruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
|
||||
({"system_instruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": [{"text": "Follow these rules"}, None]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": None}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": [{"text": None}, {"text": "Follow these rules"}]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": {"text": None}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": {"text": None, "inlineData": None}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": {"text": None, "thought": False}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": [{"thoughtSignature": "c2ln"}, {"text": "Follow these rules"}]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": [{}]}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {}}, _NO_SYSTEM_PARTS),
|
||||
({"tools": None}, {}),
|
||||
({"tools": [None]}, {}),
|
||||
({"tools": [{"functionDeclarations": None}]}, {}),
|
||||
({"tools": {"functionDeclarations": [_DECLARATION]}}, _TOOLS),
|
||||
({"tools": [{"functionDeclarations": _DECLARATION}]}, _TOOLS),
|
||||
({"tools": {"functionDeclarations": _DECLARATION}}, _TOOLS),
|
||||
({"contents": _TURN}, _CONTENTS),
|
||||
({"contents": [None, _TURN]}, _CONTENTS),
|
||||
({"contents": None}, {"contents": []}),
|
||||
({"contents": [{"role": "user", "parts": {"text": "Count this request"}}]}, _CONTENTS),
|
||||
({"contents": [{"role": "user", "parts": [None, {"text": "Count this request"}]}]}, _CONTENTS),
|
||||
({"contents": [{"role": "user", "parts": None}]}, {"contents": [{"role": "user", "parts": []}]}),
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_counts_every_google_spelling_of_a_repeated_field_like_its_list_form(
|
||||
spelled: dict[str, JsonValue], canonical: dict[str, JsonValue]
|
||||
) -> None:
|
||||
contents: Final = [{"role": "user", "parts": [{"text": "Count this request"}]}]
|
||||
|
||||
assert _google_count_tokens_request("counted", {"contents": contents, **spelled}) == _google_count_tokens_request(
|
||||
"counted", {"contents": contents, **canonical}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"malformed",
|
||||
(
|
||||
{"tools": "lookup"},
|
||||
{"tools": [{"functionDeclarations": "lookup"}]},
|
||||
{"systemInstruction": {"parts": 7}},
|
||||
{"contents": 5},
|
||||
{"contents": ["Count this request"]},
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_rejects_a_scalar_where_google_expects_an_object_or_list(
|
||||
malformed: dict[str, JsonValue],
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
_google_count_tokens_request("counted", {"contents": [_TURN], **malformed})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parts",
|
||||
(
|
||||
{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
|
||||
[{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}],
|
||||
{"text": None, "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
|
||||
[{"text": "Follow these rules"}, {"text": None, "fileData": {"fileUri": "gs://bucket/rules.pdf"}}],
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_rejects_media_only_system_parts(parts: JsonValue) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
_google_count_tokens_request(
|
||||
"counted",
|
||||
{"contents": [{"role": "user", "parts": [{"text": "Hello"}]}], "systemInstruction": {"parts": parts}},
|
||||
)
|
||||
|
||||
|
||||
def test_google_count_tokens_keeps_counting_system_text_that_sits_beside_media() -> None:
|
||||
text_with_media: Final = {"text": "Follow these rules", "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}
|
||||
|
||||
assert _google_count_tokens_request(
|
||||
"counted", {**_CONTENTS, "systemInstruction": {"parts": [text_with_media]}}
|
||||
) == _google_count_tokens_request("counted", {**_CONTENTS, **_SYSTEM})
|
||||
|
|
|
|||
|
|
@ -5480,6 +5480,15 @@ async def test_health_services_endpoint_langfuse_missing_keys_errors(monkeypatch
|
|||
assert "connection refused by lf.internal.example" in str(raised.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_services_reports_an_unknown_service_as_a_request_error_not_an_auth_error() -> None:
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await health_services_endpoint(service="totally_unknown_service_xyz")
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("admin", [False, True])
|
||||
async def test_unsaved_chatgpt_account_probe_requires_proxy_admin(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import time
|
|||
from collections.abc import AsyncGenerator, Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Final, List, Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -45,6 +46,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
||||
PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import tokenizer_models
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR
|
||||
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
|
|
@ -9172,3 +9174,42 @@ async def test_a_successful_extraction_counts_its_input_tokens_toward_the_key_tp
|
|||
call_type="aextraction",
|
||||
)
|
||||
assert refused.value.status_code == 429
|
||||
|
||||
|
||||
_MULTILINGUAL_REQUEST: Final[dict[str, object]] = {
|
||||
"messages": [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
}
|
||||
|
||||
|
||||
def _project_itpm_estimate(router: Router, model: str) -> int:
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()),
|
||||
tokenizer_models_resolver=partial(tokenizer_models, llm_router=router),
|
||||
)
|
||||
return handler._estimate_precise_input_tokens(_MULTILINGUAL_REQUEST, model, "acompletion")
|
||||
|
||||
|
||||
def test_project_itpm_counts_a_group_mixing_in_a_non_openai_model_by_its_name_as_written_and_its_openai_models():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-6.1-sol", "litellm_params": {"model": model, "api_key": "sk-test"}}
|
||||
for model in ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5")
|
||||
]
|
||||
)
|
||||
messages: Final = _MULTILINGUAL_REQUEST["messages"]
|
||||
|
||||
assert _project_itpm_estimate(router, "gpt-6.1-sol") == max(
|
||||
litellm.token_counter(model="my-model", messages=messages),
|
||||
litellm.token_counter(model="openai/gpt-6.1-sol", messages=messages),
|
||||
)
|
||||
|
||||
|
||||
def test_project_itpm_counts_a_model_group_alias_as_its_target_group():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-6.1-sol", "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-test"}}
|
||||
],
|
||||
model_group_alias={"prod": "gpt-6.1-sol"},
|
||||
)
|
||||
|
||||
assert _project_itpm_estimate(router, "prod") == _project_itpm_estimate(router, "gpt-6.1-sol")
|
||||
|
|
|
|||
|
|
@ -5211,3 +5211,18 @@ async def test_user_update_without_password_revokes_nothing(_admin_prisma, mocke
|
|||
await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
|
||||
|
||||
revoke_mock.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(_admin_prisma: MagicMock) -> None:
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await user_update(
|
||||
data=UpdateUserRequest(user_id="target-user", budget_duration="not-a-duration"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Invalid budget_duration" in exc_info.value.message
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
|
|
|||
|
|
@ -6538,7 +6538,7 @@ async def test_list_keys_with_invalid_status():
|
|||
)
|
||||
|
||||
# Verify ProxyException properties
|
||||
assert exc_info.value.code == "400"
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Invalid status value" in str(exc_info.value.message)
|
||||
assert "deleted" in str(exc_info.value.message)
|
||||
|
||||
|
|
@ -23377,3 +23377,73 @@ def test_key_info_database_not_connected_sanitized_500(
|
|||
assert "Database not connected" in caplog.text
|
||||
finally:
|
||||
proxy_server.app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
|
||||
|
||||
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
|
||||
"litellm.proxy.proxy_server.prisma_client", MagicMock()
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_key_fn(
|
||||
request=MagicMock(spec=Request),
|
||||
data=UpdateKeyRequest(key="sk-whatever", budget_duration="not-a-duration"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_key_passes_its_own_rejection_through_without_making_it_its_own_cause(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
|
||||
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
|
||||
"litellm.proxy.proxy_server.prisma_client", prisma_client
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_key_fn(
|
||||
request=MagicMock(spec=Request),
|
||||
data=UpdateKeyRequest(key="sk-missing", max_budget=5),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "not_found_error")
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_aliases_answers_503_when_the_database_is_unreachable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import prisma.errors
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import key_aliases
|
||||
|
||||
unreachable_db: Final = MagicMock()
|
||||
unreachable_db.db.query_raw = AsyncMock(
|
||||
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
|
||||
)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", unreachable_db)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await key_aliases(
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
page=1,
|
||||
size=50,
|
||||
search=None,
|
||||
team_id=None,
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ async def test_model_creation_fails_closed_when_authoritative_policy_is_unavaila
|
|||
Deployment(model_name="cannot-create", litellm_params={"model": "openai/test-model"}),
|
||||
UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
)
|
||||
assert error.value.code == "503"
|
||||
assert (error.value.code, error.value.type) == ("503", "internal_server_error")
|
||||
prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
prisma.db.litellm_teamtable.find_unique.assert_not_called()
|
||||
|
||||
|
|
@ -10913,3 +10913,76 @@ class TestNonAdminCannotSetPerUserOauthOnModel:
|
|||
user_api_key_dict=admin,
|
||||
)
|
||||
assert result is created_row
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ModelInfoDelete
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import delete_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await delete_model(
|
||||
model_info=ModelInfoDelete(id="no-such-model"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_model(
|
||||
model_params=updateDeployment(model_info=ModelInfo(id="no-such-model")),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert exc_info.value.message == "model not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import prisma.errors
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
unreachable_db: Final = MagicMock()
|
||||
unreachable_db.db.query_raw = AsyncMock(return_value=[])
|
||||
unreachable_db.db.litellm_proxymodeltable.create = AsyncMock(
|
||||
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", unreachable_db)
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-outage-salt")
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await add_new_model(
|
||||
model_params=Deployment(
|
||||
model_name="outage-model",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.4-mini", api_key="fake-key"),
|
||||
model_info={"id": "outage-model-1"},
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")
|
||||
|
|
|
|||
|
|
@ -1617,6 +1617,37 @@ async def test_delete_organization_is_refused_while_one_of_its_teams_holds_a_ptu
|
|||
prisma_client.db.litellm_organizationtable.delete.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_organization_member_add_reports_an_unknown_org_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import (
|
||||
LitellmUserRoles,
|
||||
OrganizationMemberAddRequest,
|
||||
OrgMember,
|
||||
ProxyException,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.organization_endpoints import organization_member_add
|
||||
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_organizationtable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await organization_member_add(
|
||||
data=OrganizationMemberAddRequest(
|
||||
organization_id="no-such-org",
|
||||
member=OrgMember(role="internal_user", user_id="someone"),
|
||||
),
|
||||
http_request=MagicMock(spec=Request),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("budget_duration", ["30d", None])
|
||||
async def test_new_organization_schedules_budget_reset_from_duration(monkeypatch, budget_duration):
|
||||
|
|
|
|||
|
|
@ -19372,3 +19372,48 @@ async def test_default_budget_cache_eviction_does_not_fail_after_redis_error() -
|
|||
)
|
||||
|
||||
user_api_key_cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:budget_456")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_info_reports_a_missing_team_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints import team_endpoints
|
||||
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await team_endpoints.team_info(
|
||||
http_request=MagicMock(spec=Request),
|
||||
team_id="no-such-team",
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_info_keeps_the_auth_label_for_a_caller_outside_the_team(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints import team_endpoints
|
||||
|
||||
other_team: Final = MagicMock()
|
||||
other_team.model_dump.return_value = {"team_id": "team-a", "members_with_roles": []}
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=other_team)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await team_endpoints.team_info(
|
||||
http_request=MagicMock(spec=Request),
|
||||
team_id="team-a",
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="outsider", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("403", "auth_error")
|
||||
|
|
|
|||
|
|
@ -189,7 +189,11 @@ _TURN_AFTER_COMPACTION: Final = {"role": "user", "content": "what is the cat's n
|
|||
|
||||
|
||||
async def _cursor_stream_failure_prompt_tokens(
|
||||
monkeypatch: pytest.MonkeyPatch, request_input: list[Mapping[str, str]], fails_after_output: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
request_input: list[Mapping[str, str]],
|
||||
fails_after_output: bool,
|
||||
model: str = "gpt-6-astra",
|
||||
model_info: Mapping[str, str] | None = None,
|
||||
) -> int:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -208,7 +212,6 @@ async def _cursor_stream_failure_prompt_tokens(
|
|||
usage: Final = request_data.get("combined_usage_object")
|
||||
self.prompt_tokens = usage.prompt_tokens if isinstance(usage, litellm.Usage) else None
|
||||
|
||||
model: Final = "gpt-6-astra"
|
||||
upstream_url: Final = "https://streaming.example/v1"
|
||||
created: Final = {
|
||||
"type": "response.created",
|
||||
|
|
@ -239,6 +242,7 @@ async def _cursor_stream_failure_prompt_tokens(
|
|||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"},
|
||||
"model_info": dict(model_info or {}),
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
|
|
@ -276,6 +280,26 @@ async def test_a_failed_cursor_stream_counts_its_input_from_the_newest_compactio
|
|||
assert resent == dispatched_after_trim
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fails_after_output", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("base_model", "counted_as"), [("azure/gpt-6.1-sol", "azure/gpt-6.1-sol"), ("claude-sonnet-4-5", "my-gpt-6")]
|
||||
)
|
||||
async def test_a_failed_cursor_stream_counts_a_custom_named_deployment_with_its_base_model_only_when_it_is_openai(
|
||||
monkeypatch: pytest.MonkeyPatch, base_model: str, counted_as: str, fails_after_output: bool
|
||||
) -> None:
|
||||
from litellm.proxy.utils import count_request_input_tokens
|
||||
|
||||
turn: Final = {"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}
|
||||
|
||||
counted: Final = await _cursor_stream_failure_prompt_tokens(
|
||||
monkeypatch, [turn], fails_after_output, model="my-gpt-6", model_info={"base_model": base_model}
|
||||
)
|
||||
|
||||
assert count_request_input_tokens(base_model, [turn], None) != count_request_input_tokens("my-gpt-6", [turn], None)
|
||||
assert counted == count_request_input_tokens(counted_as, [turn], None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_api_background_polling_rejects_missing_input():
|
||||
from fastapi import Response as FastAPIResponse
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.proxy.spend_tracking.budget_reservation import (
|
|||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
release_unbound_budget_reservation,
|
||||
count_request_input_tokens,
|
||||
reserve_budget_for_request,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
|
@ -426,3 +427,118 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
|
|||
)
|
||||
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
|
||||
assert get_budget_window_start(window) == expected
|
||||
|
||||
|
||||
_MULTILINGUAL_MESSAGES: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
|
||||
|
||||
def _router_with_groups(*groups: tuple[str, str]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}} for name, model in groups
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def _input_tokens(router: Router, model: str) -> dict[str, int]:
|
||||
return dict(
|
||||
await count_request_input_tokens(
|
||||
request_body={"model": model, "messages": _MULTILINGUAL_MESSAGES},
|
||||
route="/v1/chat/completions",
|
||||
llm_router=router,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count():
|
||||
router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol"))
|
||||
|
||||
by_name: Final = await _input_tokens(router, "gpt-6.1-sol")
|
||||
|
||||
assert await _input_tokens(router, "prod-gpt-6") == {"prod-gpt-6": by_name["gpt-6.1-sol"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_group_alias_of_an_openai_group_reserves_on_that_groups_token_count():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-6.1-sol", "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-test"}}
|
||||
],
|
||||
model_group_alias={"prod": "gpt-6.1-sol"},
|
||||
)
|
||||
|
||||
by_name: Final = await _input_tokens(router, "gpt-6.1-sol")
|
||||
|
||||
assert await _input_tokens(router, "prod") == {"prod": by_name["gpt-6.1-sol"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_group_alias_of_a_group_no_openai_model_serves_reserves_on_the_alias_as_written():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-sonnet-5-5",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
|
||||
}
|
||||
],
|
||||
model_group_alias={"claude-alias": "claude-sonnet-5-5"},
|
||||
)
|
||||
|
||||
assert await _input_tokens(router, "claude-alias") == {
|
||||
"claude-alias": litellm.token_counter(model="claude-alias", messages=_MULTILINGUAL_MESSAGES)
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["gpt-6-claude", "o4-alias"])
|
||||
async def test_release_named_group_or_alias_no_openai_model_serves_reserves_on_an_unknown_names_count(
|
||||
name: str,
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": group, "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"}}
|
||||
for group in ("claude-sonnet-5-5", "gpt-6-claude")
|
||||
],
|
||||
model_group_alias={"o4-alias": "claude-sonnet-5-5"},
|
||||
)
|
||||
|
||||
assert await _input_tokens(router, name) == {
|
||||
name: litellm.token_counter(model="my-model", messages=_MULTILINGUAL_MESSAGES)
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("group", "counted_as"),
|
||||
[("gpt-6.1-sol", "my-model"), ("openai/gpt-6.1-sol", "my-model"), ("mixed", "my-model"), ("gpt-4o", "gpt-4o")],
|
||||
)
|
||||
async def test_group_mixing_in_a_non_openai_model_reserves_on_its_name_as_written_and_its_openai_models(
|
||||
group: str, counted_as: str
|
||||
):
|
||||
served: Final = ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5")
|
||||
router: Final = _router_with_groups(*((group, model) for model in served))
|
||||
|
||||
reserved: Final = (await _input_tokens(router, group))[group]
|
||||
|
||||
assert reserved == max(
|
||||
litellm.token_counter(model=counted_as, messages=_MULTILINGUAL_MESSAGES),
|
||||
litellm.token_counter(model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_MESSAGES),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_reserves_on_its_current_deployments_after_another_vendors_deployment_joins():
|
||||
router: Final = _router_with_groups(("prod", "openai/gpt-6.1-sol"))
|
||||
openai_only: Final = (await _input_tokens(router, "prod"))["prod"]
|
||||
|
||||
router.add_deployment(
|
||||
Deployment(
|
||||
model_name="prod",
|
||||
litellm_params=LiteLLM_Params(model="anthropic/claude-sonnet-5-5", api_key="sk-test"),
|
||||
)
|
||||
)
|
||||
|
||||
by_own_name: Final = litellm.token_counter(model="prod", messages=_MULTILINGUAL_MESSAGES)
|
||||
assert openai_only != by_own_name
|
||||
assert (await _input_tokens(router, "prod"))["prod"] == by_own_name
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import sys
|
|||
import traceback
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal, NoReturn, cast
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -3817,6 +3818,48 @@ def test_live_session_preserves_pretty_printed_start_frame(
|
|||
assert routed_data["live_session_start"] == expected_start
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_reports_an_unsupported_router_setting_as_a_request_error_not_an_auth_error() -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ConfigYAML, ProxyException
|
||||
from litellm.proxy.proxy_server import update_config
|
||||
|
||||
body: Final = {"router_settings": {"not_a_router_setting": True}}
|
||||
request: Final = MagicMock(spec=Request)
|
||||
request.json = AsyncMock(return_value=body)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_config(
|
||||
config_info=ConfigYAML(**body),
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_config_callbacks_reports_an_unreadable_config_as_a_server_error_not_an_auth_error(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import get_config
|
||||
|
||||
empty_config: Final = tmp_path / "config.yaml"
|
||||
empty_config.write_text("")
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
monkeypatch.setattr(proxy_server, "user_config_file_path", str(empty_config))
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await get_config(user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN))
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
|
||||
assert exc_info.value.message == "Config cannot be None or Empty."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", [None, "expand"])
|
||||
@pytest.mark.parametrize(
|
||||
"general_settings, wildcard_query, wildcard_listed",
|
||||
|
|
@ -3861,6 +3904,62 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting(
|
|||
assert ("openai/*" in listed) is wildcard_listed, listed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_request_labels_an_upstream_failure_a_server_error_not_an_auth_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import async_queue_request
|
||||
|
||||
body: Final = json.dumps(
|
||||
{"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "hi"}], "priority": 0}
|
||||
).encode()
|
||||
|
||||
async def receive() -> dict[str, object]:
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/queue/chat/completions",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"query_string": b"",
|
||||
"scheme": "http",
|
||||
"server": ("testserver", 80),
|
||||
"root_path": "",
|
||||
},
|
||||
receive,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.4-mini",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
"api_key": "sk-test",
|
||||
"mock_response": "litellm.InternalServerError",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await async_queue_request(
|
||||
request=request,
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a"
|
||||
_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20
|
||||
_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}}
|
||||
|
|
|
|||
|
|
@ -3989,6 +3989,32 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r
|
|||
spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base_models_tokenizer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
azure_params: Final = {"api_key": "sk-test", "api_base": "https://example.openai.azure.com"}
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "named",
|
||||
"litellm_params": {"model": "azure/my-gpt-6", "base_model": "azure/gpt-6.1-sol", **azure_params},
|
||||
},
|
||||
{"model_name": "served", "litellm_params": {"model": "azure/gpt-6.1-sol", **azure_params}},
|
||||
]
|
||||
),
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
|
||||
named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages))
|
||||
served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages))
|
||||
|
||||
assert named.total_tokens == served.total_tokens
|
||||
|
||||
|
||||
_COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2308,6 +2308,30 @@ def test_a_failed_dispatch_is_estimated_as_input_only():
|
|||
assert usage.total_tokens == usage.prompt_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_model", "counted_as"), [("azure/gpt-6.1-sol", "azure/gpt-6.1-sol"), ("claude-sonnet-4-5", "my-gpt-6")]
|
||||
)
|
||||
def test_a_failed_dispatch_to_a_custom_named_deployment_is_counted_with_its_base_model_only_when_it_is_openai(
|
||||
base_model: str, counted_as: str
|
||||
):
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from litellm.proxy.utils import _estimated_prompt_usage, count_request_input_tokens
|
||||
|
||||
messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
logging_obj = _dispatched_failure_logging_obj(
|
||||
"my-gpt-6", get_litellm_params(metadata={"model_info": {"base_model": base_model}})
|
||||
)
|
||||
logging_obj.model_call_details["messages"] = messages
|
||||
|
||||
usage: Final = _estimated_prompt_usage(logging_obj.model_call_details, {}, "acompletion")
|
||||
|
||||
assert usage is not None
|
||||
assert count_request_input_tokens(base_model, messages, None) != count_request_input_tokens(
|
||||
"my-gpt-6", messages, None
|
||||
)
|
||||
assert usage.prompt_tokens == count_request_input_tokens(counted_as, messages, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_input", [[], object()])
|
||||
def test_nothing_is_estimated_when_there_is_nothing_to_count(request_input):
|
||||
from litellm.proxy.utils import _estimate_dispatched_failure_usage
|
||||
|
|
|
|||
|
|
@ -663,6 +663,46 @@ class TestLiteLLMCompletionResponsesConfig:
|
|||
assert "message" not in output_types
|
||||
assert "function_call" in output_types
|
||||
|
||||
def test_cited_tool_call_only_response_keeps_its_citations_in_a_message_item(self) -> None:
|
||||
citation: Final = {"start_index": 0, "end_index": 0, "url": "https://example.com", "title": "Example"}
|
||||
chat_completion_response: Final = ModelResponse(
|
||||
id="test-response-id",
|
||||
created=1234567890,
|
||||
model="gemini-3.8-flash",
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
index=0,
|
||||
message=Message(
|
||||
content=None,
|
||||
role="assistant",
|
||||
annotations=[{"type": "url_citation", "url_citation": citation}],
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_cited",
|
||||
type="function",
|
||||
function=Function(name="save_museum", arguments='{"name": "Louvre"}'),
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
responses_api_response: Final = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
|
||||
request_input="which museum?",
|
||||
responses_api_request={},
|
||||
chat_completion_response=chat_completion_response,
|
||||
)
|
||||
)
|
||||
|
||||
output: Final = responses_api_response.output
|
||||
assert [item.type for item in output] == ["message", "function_call"]
|
||||
assert output[0].content[0].text == ""
|
||||
assert [annotation.url for annotation in output[0].content[0].annotations] == [citation["url"]]
|
||||
|
||||
def test_content_bearing_response_still_emits_message_item(self):
|
||||
"""Turns with real text content must keep their message output item."""
|
||||
chat_completion_response = ModelResponse(
|
||||
|
|
@ -4306,6 +4346,7 @@ class TestEnsureOutputItemContentPartAdded:
|
|||
iterator._reasoning_active = False
|
||||
iterator._pending_response_events = []
|
||||
iterator._pending_tool_events = []
|
||||
iterator._annotations_awaiting_message = ()
|
||||
iterator._tool_output_index_by_call_id = {}
|
||||
iterator._tool_args_by_call_id = {}
|
||||
iterator._tool_item_id_by_call_id = {}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve
|
|||
import itertools
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from itertools import accumulate, chain, islice, product
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -989,6 +990,15 @@ def _with_reasoning(chunk: ModelResponseStream, reasoning: str) -> ModelResponse
|
|||
)
|
||||
|
||||
|
||||
def _with_citation(chunk: ModelResponseStream) -> ModelResponseStream:
|
||||
choice: Final = chunk.choices[0]
|
||||
annotations: Final = _annotation_only_chunk().choices[0].delta.annotations
|
||||
delta: Final = Delta(**{**choice.delta.model_dump(), "annotations": annotations})
|
||||
return chunk.model_copy(
|
||||
update={"choices": [StreamingChoices(index=0, delta=delta, finish_reason=choice.finish_reason)]}
|
||||
)
|
||||
|
||||
|
||||
def _reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_reasoning_chunk("let me think"), _chunk("Hello"), _chunk("!", finish_reason="stop"))
|
||||
|
||||
|
|
@ -1662,6 +1672,266 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item
|
|||
assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize("batches", [(5,), (2, 3)])
|
||||
@pytest.mark.parametrize("with_text", [True, False])
|
||||
@pytest.mark.parametrize("reasoning_first", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_citation_batches_stream_before_message_done(
|
||||
sync_mode: bool, batches: tuple[int, ...], with_text: bool, reasoning_first: bool
|
||||
) -> None:
|
||||
annotations: Final = [
|
||||
{
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
"start_index": 0,
|
||||
"end_index": 5,
|
||||
"url": f"https://example.com/{index}",
|
||||
"title": f"Source {index}",
|
||||
},
|
||||
}
|
||||
for index in range(sum(batches))
|
||||
]
|
||||
citation_chunks: Final = tuple(
|
||||
ModelResponseStream(
|
||||
id=CHAT_COMPLETION_ID,
|
||||
created=1748575031,
|
||||
model="claude-haiku-4-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
role="assistant",
|
||||
content="!" if with_text else "",
|
||||
annotations=annotations[end - size : end],
|
||||
),
|
||||
finish_reason="stop" if end == len(annotations) else None,
|
||||
)
|
||||
],
|
||||
)
|
||||
for size, end in zip(batches, accumulate(batches), strict=True)
|
||||
)
|
||||
reasoning_chunks: Final = _reasoning_then_text_chunks()[:1] if reasoning_first else ()
|
||||
chunks: Final = (*reasoning_chunks, _chunk("Claim"), *citation_chunks)
|
||||
|
||||
events: Final = await _collect_events(_build_iterator(chunks), sync_mode)
|
||||
|
||||
added: Final = [event for event in events if event.type == "response.output_text.annotation.added"]
|
||||
terminal: Final = events[-1]
|
||||
message: Final = next(item for item in terminal.response.output if item.type == "message")
|
||||
final_annotations: Final = [annotation.model_dump() for annotation in message.content[0].annotations]
|
||||
assert [event.annotation for event in added] == final_annotations
|
||||
assert len(added) == len(annotations)
|
||||
assert [event.annotation_index for event in added] == list(range(len(annotations)))
|
||||
message_index: Final = next(
|
||||
event.output_index
|
||||
for event in events
|
||||
if event.type == "response.output_item.added" and event.item.type == "message"
|
||||
)
|
||||
assert all(event.item_id == message.id and event.output_index == message_index for event in added)
|
||||
done_index: Final = next(index for index, event in enumerate(events) if event.type == "response.output_text.done")
|
||||
assert all(events.index(event) < done_index for event in added)
|
||||
assert message.content[0].text == "Claim" + ("!" * len(batches) if with_text else "")
|
||||
|
||||
|
||||
def _citation_after_reasoning_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_reasoning_chunk("let me think"), _annotation_only_chunk())
|
||||
|
||||
|
||||
def _citation_with_reasoning_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_with_reasoning(_annotation_only_chunk(), "let me think"),)
|
||||
|
||||
|
||||
def _citation_between_reasoning_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_reasoning_chunk("let me "), _annotation_only_chunk(), _reasoning_chunk("think"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"stream_chunks",
|
||||
[_citation_after_reasoning_chunks, _citation_with_reasoning_chunks, _citation_between_reasoning_chunks],
|
||||
ids=["citation_after_reasoning", "citation_with_reasoning", "citation_between_reasoning"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_citation_during_reasoning_streams_once_reasoning_closes(
|
||||
stream_chunks: StreamChunks, sync_mode: bool
|
||||
) -> None:
|
||||
chunks: Final = (*stream_chunks(), _chunk("Claim", finish_reason="stop"))
|
||||
events: Final = await _collect_events(_build_iterator(chunks), sync_mode)
|
||||
|
||||
output: Final = events[-1].response.output
|
||||
message_index: Final = next(index for index, item in enumerate(output) if item.type == "message")
|
||||
message: Final = output[message_index]
|
||||
reasoning: Final = next(item for item in output if item.type == "reasoning")
|
||||
reasoning_done: Final = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.type == "response.output_item.done" and event.item.type == "reasoning"
|
||||
)
|
||||
message_added: Final = next(index for index, event in enumerate(events) if _is_message_item(event))
|
||||
part_added: Final = next(index for index, event in enumerate(events) if event.type == "response.content_part.added")
|
||||
text_done: Final = next(index for index, event in enumerate(events) if event.type == "response.output_text.done")
|
||||
citations: Final = [event for event in events if event.type == "response.output_text.annotation.added"]
|
||||
reasoning_deltas: Final = [
|
||||
index for index, event in enumerate(events) if event.type == "response.reasoning_summary_text.delta"
|
||||
]
|
||||
summary_done: Final = next(event for event in events if event.type == "response.reasoning_summary_text.done")
|
||||
text_delta: Final = next(event for event in events if event.type == "response.output_text.delta")
|
||||
|
||||
assert len(citations) == 1
|
||||
assert reasoning_done < message_added < part_added < events.index(citations[0]) < text_done
|
||||
assert all(index < reasoning_done for index in reasoning_deltas)
|
||||
assert summary_done.text == reasoning.content[0].text == "let me think"
|
||||
assert citations[0].item_id == message.id
|
||||
assert citations[0].output_index == message_index
|
||||
assert citations[0].annotation_index == 0
|
||||
assert [event.annotation for event in citations] == [entry.model_dump() for entry in message.content[0].annotations]
|
||||
assert ("sequence_number" in citations[0].model_dump()) == ("sequence_number" in text_delta.model_dump())
|
||||
assert message.content[0].text == "Claim"
|
||||
|
||||
|
||||
def _citation_then_tool_call_finishing_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_annotation_only_chunk(), _tool_call_chunk(finish_reason="tool_calls"))
|
||||
|
||||
|
||||
def _reasoning_citation_then_tool_call_finishing_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_reasoning_chunk("let me think"), *_citation_then_tool_call_finishing_chunks())
|
||||
|
||||
|
||||
def _reasoning_citation_tool_call_then_empty_finish_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (
|
||||
_reasoning_chunk("let me think"),
|
||||
_annotation_only_chunk(),
|
||||
_tool_call_chunk(),
|
||||
_chunk("", finish_reason="tool_calls"),
|
||||
)
|
||||
|
||||
|
||||
def _citation_between_reasoning_then_empty_finish_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (*_citation_between_reasoning_chunks(), _chunk("", finish_reason="stop"))
|
||||
|
||||
|
||||
def _tool_call_then_citation_finishing_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_tool_call_chunk(), _annotation_only_chunk())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"stream_chunks",
|
||||
[
|
||||
_citation_then_tool_call_finishing_chunks,
|
||||
_reasoning_citation_then_tool_call_finishing_chunks,
|
||||
_reasoning_citation_tool_call_then_empty_finish_chunks,
|
||||
_citation_between_reasoning_then_empty_finish_chunks,
|
||||
_citation_between_reasoning_chunks,
|
||||
_tool_call_then_citation_finishing_chunks,
|
||||
],
|
||||
ids=[
|
||||
"citation_then_tool_call",
|
||||
"reasoning_citation_then_tool_call",
|
||||
"reasoning_citation_tool_call_then_empty_finish",
|
||||
"citation_between_reasoning_then_empty_finish",
|
||||
"citation_between_reasoning_without_finish",
|
||||
"tool_call_then_citation_without_text",
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_citations_match_the_completed_message(stream_chunks: StreamChunks, sync_mode: bool) -> None:
|
||||
events: Final = await _collect_events(_build_iterator(stream_chunks()), sync_mode)
|
||||
|
||||
output: Final = events[-1].response.output
|
||||
completed_message_indexes: Final = [index for index, item in enumerate(output) if item.type == "message"]
|
||||
completed_citations: Final = chain.from_iterable(
|
||||
output[index].content[0].annotations for index in completed_message_indexes
|
||||
)
|
||||
completed_annotations: Final = [annotation.model_dump() for annotation in completed_citations]
|
||||
streamed_message_indexes: Final = [event.output_index for event in events if _is_message_item(event)]
|
||||
citations: Final = [event for event in events if event.type == "response.output_text.annotation.added"]
|
||||
reasoning_done: Final = [
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.type == "response.output_item.done" and event.item.type == "reasoning"
|
||||
]
|
||||
message_added: Final = [index for index, event in enumerate(events) if _is_message_item(event)]
|
||||
|
||||
assert len(citations) == 1
|
||||
assert [event.annotation for event in citations] == completed_annotations
|
||||
assert sorted(set(streamed_message_indexes)) == completed_message_indexes
|
||||
assert all(event.output_index in completed_message_indexes for event in citations)
|
||||
assert len(reasoning_done) == sum(item.type == "reasoning" for item in output)
|
||||
assert all(index < added for index, added in product(reasoning_done, message_added))
|
||||
|
||||
|
||||
def _empty_text_citation_then_tool_call_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_chunk(""), *_citation_then_tool_call_finishing_chunks())
|
||||
|
||||
|
||||
def _reasoning_empty_text_citation_then_tool_call_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_reasoning_chunk("let me think"), *_empty_text_citation_then_tool_call_chunks())
|
||||
|
||||
|
||||
def _empty_text_then_cited_tool_call_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_chunk(""), _with_citation(_tool_call_chunk(finish_reason="tool_calls")))
|
||||
|
||||
|
||||
def _citation_tool_call_then_text_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (_annotation_only_chunk(), _tool_call_chunk(), _chunk("Claim", finish_reason="tool_calls"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
("stream_chunks", "expected_order"),
|
||||
[
|
||||
(_empty_text_citation_then_tool_call_chunks, ("message", "function_call")),
|
||||
(_reasoning_empty_text_citation_then_tool_call_chunks, ("reasoning", "message", "function_call")),
|
||||
(_empty_text_then_cited_tool_call_chunks, ("message", "function_call")),
|
||||
(_citation_tool_call_then_text_chunks, ("message", "function_call")),
|
||||
(_citation_then_tool_call_finishing_chunks, ("message", "function_call")),
|
||||
(_reasoning_citation_tool_call_then_empty_finish_chunks, ("reasoning", "message", "function_call")),
|
||||
],
|
||||
ids=[
|
||||
"empty_text_citation_then_tool_call",
|
||||
"reasoning_empty_text_citation_then_tool_call",
|
||||
"empty_text_then_cited_tool_call",
|
||||
"citation_tool_call_then_text",
|
||||
"citation_then_tool_call_without_text",
|
||||
"reasoning_citation_tool_call_then_empty_finish",
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_citation_before_a_tool_call_keeps_the_message_ahead_of_it(
|
||||
stream_chunks: StreamChunks, expected_order: tuple[str, ...], sync_mode: bool
|
||||
) -> None:
|
||||
events: Final = await _collect_events(_build_iterator(stream_chunks()), sync_mode)
|
||||
|
||||
added: Final = [event for event in events if event.type == "response.output_item.added"]
|
||||
output: Final = events[-1].response.output
|
||||
message_index: Final = expected_order.index("message")
|
||||
citations: Final = [event for event in events if event.type == "response.output_text.annotation.added"]
|
||||
|
||||
assert tuple(event.item.type for event in added) == expected_order
|
||||
assert [event.output_index for event in added] == list(range(len(expected_order)))
|
||||
assert tuple(item.type for item in output) == expected_order
|
||||
assert [event.output_index for event in citations] == [message_index]
|
||||
assert [event.annotation for event in citations] == [
|
||||
annotation.model_dump() for annotation in output[message_index].content[0].annotations
|
||||
]
|
||||
|
||||
|
||||
def test_disconnect_after_a_held_citation_still_bills_it() -> None:
|
||||
iterator: Final = _build_iterator((*_citation_between_reasoning_chunks(), _chunk("Claim", finish_reason="stop")))
|
||||
reasoning_deltas: Final = (event for event in iterator if event.type == "response.reasoning_summary_text.delta")
|
||||
assert len(list(islice(reasoning_deltas, 2))) == 2
|
||||
|
||||
incomplete: Final = iterator.partial_response_on_disconnect()
|
||||
|
||||
assert incomplete is not None
|
||||
messages: Final = [item for item in incomplete.response.output if item.type == "message"]
|
||||
annotations: Final = chain.from_iterable(message.content[0].annotations for message in messages)
|
||||
assert [annotation.url for annotation in annotations] == ["https://example.com"]
|
||||
|
||||
|
||||
def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None:
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None)
|
|||
("gemini/gemini-2.5-pro", "cl100k_base"),
|
||||
("mistral/mistral-large-latest", "cl100k_base"),
|
||||
("my-router-alias", "cl100k_base"),
|
||||
("azure/gpt-4o", "cl100k_base"),
|
||||
("azure/gpt-4o", "o200k_base"),
|
||||
("command-r-plus", "cl100k_base"),
|
||||
("gpt-4o", "o200k_base"),
|
||||
("gpt-4o-mini", "o200k_base"),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -51,12 +52,16 @@ def _serialize(cost_map: dict[str, object]) -> str:
|
|||
return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _snapshot(cost_map: dict[str, object], backup: str | None = None) -> object:
|
||||
NO_PINS: Final = "{}\n"
|
||||
|
||||
|
||||
def _snapshot(cost_map: dict[str, object], backup: str | None = None, pins: str = NO_PINS) -> object:
|
||||
text = _serialize(cost_map)
|
||||
return guard.Snapshot(cost_map=text, backup=text if backup is None else backup)
|
||||
return guard.Snapshot(cost_map=text, backup=text if backup is None else backup, pins=pins)
|
||||
|
||||
|
||||
BASE: Final = _snapshot(BASE_MAP)
|
||||
UP_TO_DATE: Final = object()
|
||||
|
||||
|
||||
def _failures(
|
||||
|
|
@ -64,10 +69,13 @@ def _failures(
|
|||
changed_files: tuple[str, ...] = MAP_FILES,
|
||||
bot: bool = True,
|
||||
evaluation_output_price_sources: Mapping[str, str] = guard.EVALUATION_OUTPUT_PRICE_SOURCES,
|
||||
base: object = BASE,
|
||||
merged: object = UP_TO_DATE,
|
||||
) -> tuple[str, ...]:
|
||||
return guard.guard_failures(
|
||||
BASE,
|
||||
base,
|
||||
head,
|
||||
head if merged is UP_TO_DATE else merged,
|
||||
changed_files,
|
||||
bot,
|
||||
evaluation_output_price_sources=evaluation_output_price_sources,
|
||||
|
|
@ -129,14 +137,14 @@ def test_bot_may_add_and_reprice_models() -> None:
|
|||
|
||||
|
||||
def test_broken_json_is_reported() -> None:
|
||||
head = guard.Snapshot(cost_map="{not json", backup="{not json")
|
||||
head = guard.Snapshot(cost_map="{not json", backup="{not json", pins=NO_PINS)
|
||||
assert _failures(head, bot=False) == (
|
||||
f"{guard.COST_MAP_PATH} is not valid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
|
||||
)
|
||||
|
||||
|
||||
def test_non_object_root_is_reported() -> None:
|
||||
head = guard.Snapshot(cost_map="[]", backup="[]")
|
||||
head = guard.Snapshot(cost_map="[]", backup="[]", pins=NO_PINS)
|
||||
assert _failures(head, bot=False) == (f"{guard.COST_MAP_PATH} must be a JSON object at the root",)
|
||||
|
||||
|
||||
|
|
@ -154,15 +162,15 @@ CODE_ONLY: Final = (
|
|||
|
||||
|
||||
def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None:
|
||||
unparseable: Final = guard.Snapshot(cost_map="{not json", backup="")
|
||||
unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", pins=NO_PINS)
|
||||
assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == ()
|
||||
assert _failures(STALE_HEAD, changed_files=(), bot=False) == ()
|
||||
assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS)
|
||||
def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None:
|
||||
failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False)
|
||||
@pytest.mark.parametrize("checked_path", guard.CHECKED_PATHS)
|
||||
def test_touching_any_cost_map_file_keeps_the_file_checks(checked_path: str) -> None:
|
||||
failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, checked_path), bot=False)
|
||||
assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)]
|
||||
|
||||
|
||||
|
|
@ -204,8 +212,155 @@ def test_bot_may_not_change_special_root_keys() -> None:
|
|||
assert _failures(head) == ("bot PRs may not change fallback_generalizations",)
|
||||
|
||||
|
||||
def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str:
|
||||
PINNED_SOURCE: Final = "a 512x512 probe image was read back over the provider's chat endpoint"
|
||||
PINS: Final = _serialize(
|
||||
{"openrouter/a": {"supports_vision": {"value": True, "checked": "2026-09-26", "source": PINNED_SOURCE}}}
|
||||
)
|
||||
|
||||
|
||||
def test_a_pinned_capability_that_holds_passes_for_humans_and_bots() -> None:
|
||||
head = _snapshot(BASE_MAP, pins=PINS)
|
||||
assert _failures(head, bot=False) == ()
|
||||
assert _failures(head, bot=True) == ()
|
||||
|
||||
|
||||
LOWERED: Final = (
|
||||
f"{guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: {PINNED_SOURCE}); "
|
||||
f"change {guard.PINS_PATH} with new evidence first"
|
||||
)
|
||||
NO_VISION_MAP: Final = {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", [False, True])
|
||||
def test_lowering_a_pinned_capability_fails_with_its_evidence(bot: bool) -> None:
|
||||
assert _failures(_snapshot(NO_VISION_MAP, pins=PINS), bot=bot) == (LOWERED,)
|
||||
|
||||
|
||||
def test_dropping_a_pinned_field_or_model_is_reported() -> None:
|
||||
no_field = _snapshot({**BASE_MAP, "openrouter/a": _entry()}, pins=PINS)
|
||||
assert [failure for failure in _failures(no_field, bot=False) if "must stay true" in failure]
|
||||
no_model = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/a"}, pins=PINS)
|
||||
assert _failures(no_model, bot=False) == (
|
||||
f"{guard.PINS_PATH} pins openrouter/a.supports_vision but {guard.COST_MAP_PATH} has no openrouter/a entry",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pins", "expected"),
|
||||
[
|
||||
(
|
||||
"{not json",
|
||||
f"{guard.PINS_PATH} is not valid JSON: Expecting property name enclosed in double quotes: "
|
||||
"line 1 column 2 (char 1)",
|
||||
),
|
||||
(
|
||||
_serialize({"openrouter/a": {"supports_vision": {"value": True}}}),
|
||||
f"{guard.PINS_PATH}: openrouter/a.supports_vision needs value, checked, and source",
|
||||
),
|
||||
(_serialize({"openrouter/a": True}), f"{guard.PINS_PATH}: openrouter/a must map field names to pins"),
|
||||
],
|
||||
)
|
||||
def test_malformed_pins_are_reported(pins: str, expected: str) -> None:
|
||||
assert _failures(_snapshot(BASE_MAP, pins=pins), bot=False) == (expected,)
|
||||
|
||||
|
||||
MISSING_PINS: Final = f"{guard.PINS_PATH} is missing; restore it, its pins were verified against live provider calls"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", [False, True])
|
||||
def test_a_pr_that_deletes_the_pins_file_fails(bot: bool) -> None:
|
||||
head: Final = _snapshot(BASE_MAP, pins="")
|
||||
assert _failures(head, bot=bot, base=_snapshot(BASE_MAP, pins=PINS)) == (MISSING_PINS,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", [False, True])
|
||||
def test_pin_checks_are_skipped_when_no_side_of_the_merge_has_the_pins_file(bot: bool) -> None:
|
||||
assert _failures(_snapshot(NO_VISION_MAP, pins=""), bot=bot, base=_snapshot(NO_VISION_MAP, pins="")) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", [False, True])
|
||||
def test_a_branch_cut_before_a_pin_was_added_cannot_lower_it(bot: bool) -> None:
|
||||
head: Final = _snapshot(NO_VISION_MAP, pins="")
|
||||
merged: Final = _snapshot(NO_VISION_MAP, pins=PINS)
|
||||
assert _failures(head, bot=bot, base=_snapshot(BASE_MAP, pins=""), merged=merged) == (LOWERED,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bot", [False, True])
|
||||
def test_a_branch_cut_before_a_pin_was_added_passes_when_merging_keeps_the_pinned_value(bot: bool) -> None:
|
||||
repriced: Final = {"openrouter/b": _entry(3e-06)}
|
||||
head: Final = _snapshot({**NO_VISION_MAP, **repriced}, pins="")
|
||||
merged: Final = _snapshot({**BASE_MAP, **repriced}, pins=PINS)
|
||||
assert _failures(head, bot=bot, base=_snapshot(NO_VISION_MAP, pins=""), merged=merged) == ()
|
||||
|
||||
|
||||
def test_a_pr_that_does_not_merge_cleanly_is_told_to_merge_the_base_branch_in() -> None:
|
||||
assert _failures(BASE, bot=False, merged=None) == (
|
||||
"the pull request does not merge cleanly into its base branch; merge the base branch in first",
|
||||
)
|
||||
|
||||
|
||||
def test_a_merge_result_that_is_not_valid_json_is_reported() -> None:
|
||||
merged: Final = guard.Snapshot(cost_map="{broken", backup="{broken", pins=PINS)
|
||||
assert _failures(BASE, bot=False, merged=merged) == (
|
||||
f"after merging into the base branch, {guard.COST_MAP_PATH} is not valid JSON: "
|
||||
"Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
|
||||
)
|
||||
|
||||
|
||||
def test_a_pr_that_adds_the_pins_file_gets_its_pins_checked() -> None:
|
||||
head: Final = _snapshot({**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, pins=PINS)
|
||||
changed: Final = (*MAP_FILES, guard.PINS_PATH)
|
||||
failures: Final = _failures(head, changed_files=changed, bot=False, base=_snapshot(BASE_MAP, pins=""))
|
||||
assert [failure for failure in failures if "openrouter/a.supports_vision must stay true" in failure]
|
||||
|
||||
|
||||
def test_bot_may_not_touch_the_pins_file() -> None:
|
||||
head = _snapshot(BASE_MAP, pins=PINS)
|
||||
changed = (*guard.GUARDED_PATHS, guard.PINS_PATH)
|
||||
assert _failures(head, changed_files=changed, bot=False) == ()
|
||||
assert _failures(head, changed_files=changed) == (
|
||||
f"bot PRs may only change the cost map files, not {guard.PINS_PATH}",
|
||||
)
|
||||
|
||||
|
||||
PRICE_SYNC_PIN_FAILURE: Final = (
|
||||
f"price sync PRs may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence"
|
||||
)
|
||||
BOT_AUTHOR_PIN_FAILURE: Final = (
|
||||
f"PRs opened by a bot may not change {guard.PINS_PATH}; a pin moves only in a human PR with new evidence"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("head_ref", "author_type", "changed", "expected"),
|
||||
[
|
||||
("litellm-providers/price-sync-fireworks-ai", "User", (*MAP_FILES, guard.PINS_PATH), (PRICE_SYNC_PIN_FAILURE,)),
|
||||
("litellm-providers/price-sync-fireworks-ai", "Bot", MAP_FILES, ()),
|
||||
("litellm_azure_ai_mai_image_2_5_pro", "Bot", (*MAP_FILES, guard.PINS_PATH), (BOT_AUTHOR_PIN_FAILURE,)),
|
||||
("litellm_azure_ai_mai_image_2_5_pro", "Bot", MAP_FILES, ()),
|
||||
("fix-openrouter-price-sync", "User", (*MAP_FILES, guard.PINS_PATH), ()),
|
||||
("fix-openrouter-price-sync", "", (*MAP_FILES, guard.PINS_PATH), ()),
|
||||
],
|
||||
)
|
||||
def test_only_human_opened_non_price_sync_prs_may_touch_the_pins_file(
|
||||
head_ref: str, author_type: str, changed: tuple[str, ...], expected: tuple[str, ...]
|
||||
) -> None:
|
||||
assert guard.pin_author_failures(head_ref, author_type, changed) == expected
|
||||
|
||||
|
||||
def test_checked_in_pins_hold_in_the_checked_in_cost_map() -> None:
|
||||
cost_map = json.loads((ROOT / guard.COST_MAP_PATH).read_text())
|
||||
pins = (ROOT / guard.PINS_PATH).read_text()
|
||||
assert json.loads(pins), "the pins file must pin at least one capability"
|
||||
assert guard.pin_failures(pins, cost_map) == ()
|
||||
|
||||
|
||||
def _commit(repo: Path, cost_map: dict[str, object], message: str, pins: str | None = None) -> str:
|
||||
text = _serialize(cost_map)
|
||||
pins_file = repo / guard.PINS_PATH
|
||||
if pins is not None or not pins_file.exists():
|
||||
pins_file.parent.mkdir(exist_ok=True)
|
||||
pins_file.write_text(NO_PINS if pins is None else pins)
|
||||
(repo / guard.COST_MAP_PATH).write_text(text)
|
||||
(repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True)
|
||||
(repo / guard.BACKUP_PATH).write_text(text)
|
||||
|
|
@ -230,13 +385,38 @@ def _git_commit(repo: Path, message: str) -> str:
|
|||
).stdout.strip()
|
||||
|
||||
|
||||
def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.CompletedProcess[str]:
|
||||
def _run_guard(
|
||||
repo: Path,
|
||||
base: str,
|
||||
head: str,
|
||||
head_ref: str,
|
||||
*,
|
||||
target: str | None = None,
|
||||
target_env: str | None = None,
|
||||
author_type: str | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
target_args: Final = () if target_env is not None else ("--target", target or base)
|
||||
env_overrides: Final = {
|
||||
**({} if target_env is None else {guard.TARGET_ENV: target_env}),
|
||||
**({} if author_type is None else {guard.AUTHOR_TYPE_ENV: author_type}),
|
||||
}
|
||||
return subprocess.run(
|
||||
(sys.executable, str(CI_CD / "cost_map_guard.py"), "--base", base, "--head", head, "--head-ref", head_ref),
|
||||
(
|
||||
sys.executable,
|
||||
str(CI_CD / "cost_map_guard.py"),
|
||||
"--base",
|
||||
base,
|
||||
"--head",
|
||||
head,
|
||||
*target_args,
|
||||
"--head-ref",
|
||||
head_ref,
|
||||
),
|
||||
cwd=repo,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
env={**os.environ, **env_overrides},
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -298,3 +478,69 @@ def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None:
|
|||
head = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head")
|
||||
assert _run_guard(tmp_path, base, head, BOT_REF).returncode == 1
|
||||
assert _run_guard(tmp_path, base, head, "litellm_fix_pricing").returncode == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"])
|
||||
def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: str) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS)
|
||||
head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/a": _entry(supports_vision=False)}, "sync lowers it")
|
||||
result: Final = _run_guard(tmp_path, base, head, head_ref)
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: "
|
||||
assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"])
|
||||
def test_main_holds_a_pin_the_base_branch_added_after_the_pr_branched(tmp_path: Path, head_ref: str) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base")
|
||||
target: Final = _commit(tmp_path, BASE_MAP, "pin on the base branch", pins=PINS)
|
||||
subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True)
|
||||
head: Final = _commit(tmp_path, NO_VISION_MAP, "branch cut before the pin lowers it", pins=NO_PINS)
|
||||
result: Final = _run_guard(tmp_path, merge_base, head, head_ref, target_env=target)
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: "
|
||||
assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)]
|
||||
|
||||
|
||||
def test_main_reports_a_failed_merge_apart_from_a_conflict(tmp_path: Path) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
base: Final = _commit(tmp_path, BASE_MAP, "base")
|
||||
head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head")
|
||||
result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing", target="0" * 40)
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
assert result.stdout.startswith(f"cost map guard failed: git merge-tree into {'0' * 40} failed: ")
|
||||
|
||||
|
||||
def test_main_tells_a_conflicting_pr_to_merge_the_base_branch_in(tmp_path: Path) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base")
|
||||
target: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(3e-06)}, "base branch reprices b")
|
||||
subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True)
|
||||
head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(4e-06)}, "pr reprices b differently")
|
||||
result: Final = _run_guard(tmp_path, merge_base, head, "litellm_fix_pricing", target=target)
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
conflict_line: Final = "- the pull request does not merge cleanly into its base branch; merge the base branch in first"
|
||||
assert conflict_line in result.stdout.splitlines()
|
||||
|
||||
|
||||
def test_main_skips_the_merge_for_a_pr_that_leaves_the_cost_map_alone(tmp_path: Path) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
base: Final = _commit(tmp_path, BASE_MAP, "base")
|
||||
head: Final = _commit_code_only(tmp_path, "code change")
|
||||
result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing", target="0" * 40)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert result.stdout.splitlines() == ["cost map guard passed (human PR, cost map untouched)"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("author_type", "expected_code"), [("Bot", 1), ("User", 0)])
|
||||
def test_main_lets_only_a_human_opened_pr_change_the_pins(tmp_path: Path, author_type: str, expected_code: int) -> None:
|
||||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
base: Final = _commit(tmp_path, BASE_MAP, "base", pins=PINS)
|
||||
head: Final = _commit(tmp_path, NO_VISION_MAP, "lower the flag and drop its pin", pins=NO_PINS)
|
||||
result: Final = _run_guard(
|
||||
tmp_path, base, head, "litellm_azure_ai_mai_image_2_5_pro", author_type=author_type
|
||||
)
|
||||
assert result.returncode == expected_code, result.stdout + result.stderr
|
||||
assert (f"- {BOT_AUTHOR_PIN_FAILURE}" in result.stdout.splitlines()) == (author_type == "Bot")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits).
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -1039,3 +1040,34 @@ class TestContextSlotRetention:
|
|||
assert deployment is not None
|
||||
router._update_kwargs_with_deployment(deployment=deployment.model_dump(), kwargs=kwargs)
|
||||
assert get_io_token_rate_limit_request_kwargs() is kwargs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_token_count() -> None:
|
||||
async def reserved_input(deployment: dict[str, object]) -> object:
|
||||
metadata: Final[dict[str, object]] = {}
|
||||
request_kwargs: Final[dict[str, object]] = {
|
||||
"messages": [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}],
|
||||
"max_tokens": 5,
|
||||
"metadata": metadata,
|
||||
}
|
||||
set_io_token_rate_limit_request_kwargs(request_kwargs)
|
||||
await async_io_token_pre_call_check(DualCache(), deployment)
|
||||
return metadata[ITPM_RESERVED_KEY]
|
||||
|
||||
named = await reserved_input(
|
||||
{
|
||||
"litellm_params": {"model": "azure/my-gpt-6", "base_model": "azure/gpt-6.1-sol", "itpm": 100_000},
|
||||
"model_info": {"id": "named-deployment"},
|
||||
"model_name": "named",
|
||||
}
|
||||
)
|
||||
served = await reserved_input(
|
||||
{
|
||||
"litellm_params": {"model": "azure/gpt-6.1-sol", "itpm": 100_000},
|
||||
"model_info": {"id": "served-deployment"},
|
||||
"model_name": "served",
|
||||
}
|
||||
)
|
||||
|
||||
assert named == served
|
||||
|
|
|
|||
|
|
@ -7938,6 +7938,86 @@ def test_pre_call_checks_counts_once_and_filters_on_max_input_tokens(monkeypatch
|
|||
assert calls == [1]
|
||||
|
||||
|
||||
_MULTILINGUAL_PROMPT: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}]
|
||||
|
||||
|
||||
def _context_limited_router(*served: str) -> litellm.Router:
|
||||
limit: Final = (
|
||||
litellm.token_counter(model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT)
|
||||
+ litellm.token_counter(model="prod", messages=_MULTILINGUAL_PROMPT)
|
||||
) // 2
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "prod",
|
||||
"litellm_params": {"model": model, "api_key": "sk-test"},
|
||||
"model_info": {"max_input_tokens": limit},
|
||||
}
|
||||
for model in served
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
|
||||
def test_pre_call_checks_count_a_group_of_one_openai_model_with_that_models_tokenizer():
|
||||
router: Final = _context_limited_router("openai/gpt-6.1-sol")
|
||||
|
||||
kept: Final = router._pre_call_checks(
|
||||
model="prod", healthy_deployments=router.get_model_list(model_name="prod"), messages=_MULTILINGUAL_PROMPT
|
||||
)
|
||||
|
||||
assert len(kept) == 1
|
||||
|
||||
|
||||
def test_pre_call_checks_count_a_group_mixing_in_another_vendor_by_its_own_name():
|
||||
router: Final = _context_limited_router("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5")
|
||||
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
router._pre_call_checks(
|
||||
model="prod", healthy_deployments=router.get_model_list(model_name="prod"), messages=_MULTILINGUAL_PROMPT
|
||||
)
|
||||
|
||||
|
||||
async def test_acount_pre_call_check_tokens_counts_a_group_of_one_openai_model_with_that_models_tokenizer():
|
||||
router: Final = _context_limited_router("openai/gpt-6.1-sol")
|
||||
|
||||
count: Final = await router._acount_pre_call_check_tokens(
|
||||
model="prod",
|
||||
healthy_deployments=router.get_model_list(model_name="prod"),
|
||||
messages=_MULTILINGUAL_PROMPT,
|
||||
input=None,
|
||||
request_kwargs=None,
|
||||
)
|
||||
|
||||
assert count == litellm.token_counter(model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT)
|
||||
|
||||
|
||||
async def test_a_wildcard_route_checks_the_context_window_with_the_requested_openai_models_tokenizer():
|
||||
limit: Final = (
|
||||
litellm.token_counter(model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT)
|
||||
+ litellm.token_counter(model="my-model", messages=_MULTILINGUAL_PROMPT)
|
||||
) // 2
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "sk-test"},
|
||||
"model_info": {"max_input_tokens": limit},
|
||||
}
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
sync_response: Final = router.completion(
|
||||
model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT, mock_response="ok"
|
||||
)
|
||||
async_response: Final = await router.acompletion(
|
||||
model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT, mock_response="ok"
|
||||
)
|
||||
|
||||
assert (sync_response.choices[0].message.content, async_response.choices[0].message.content) == ("ok", "ok")
|
||||
|
||||
|
||||
def test_pre_call_checks_uses_precounted_tokens(monkeypatch):
|
||||
"""
|
||||
An async caller counts off the event loop and passes the result in. _pre_call_checks
|
||||
|
|
|
|||
|
|
@ -519,6 +519,40 @@ def test_vector_store_tools_pop_exactly_the_store_ids_the_registry_reports_to_ru
|
|||
assert ran == checked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vector_stores_to_run_ignore_a_vector_store_ids_object() -> None:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
registry: Final = VectorStoreRegistry(
|
||||
[LiteLLM_ManagedVectorStore(vector_store_id="vs_registered", created_at=now, updated_at=now)]
|
||||
)
|
||||
|
||||
ran: Final = (
|
||||
registry.pop_vector_stores_to_run(non_default_params={"vector_store_ids": {"vs_registered": True}}),
|
||||
await registry.pop_vector_stores_to_run_with_db_fallback(
|
||||
non_default_params={"vector_store_ids": {"vs_registered": True}}
|
||||
),
|
||||
)
|
||||
|
||||
assert ran == ([], [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vector_stores_to_run_read_a_tuple_of_vector_store_ids() -> None:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
registry: Final = VectorStoreRegistry(
|
||||
[LiteLLM_ManagedVectorStore(vector_store_id="vs_registered", created_at=now, updated_at=now)]
|
||||
)
|
||||
|
||||
ran: Final = (
|
||||
registry.pop_vector_stores_to_run(non_default_params={"vector_store_ids": ("vs_registered",)}),
|
||||
await registry.pop_vector_stores_to_run_with_db_fallback(
|
||||
non_default_params={"vector_store_ids": ("vs_registered",)}
|
||||
),
|
||||
)
|
||||
|
||||
assert [[store["vector_store_id"] for store in stores] for stores in ran] == [["vs_registered"], ["vs_registered"]]
|
||||
|
||||
|
||||
class TestRegistriesDoNotShareDefaultState:
|
||||
def test_adding_to_one_registry_leaves_the_next_one_empty(self):
|
||||
VectorStoreRegistry().add_vector_store_to_registry(
|
||||
|
|
|
|||
84
uv.lock
generated
84
uv.lock
generated
|
|
@ -25,11 +25,13 @@ members = [
|
|||
]
|
||||
constraints = [
|
||||
{ name = "aiohttp", specifier = ">=3.14.2,<4.0" },
|
||||
{ name = "dnspython", marker = "python_full_version >= '3.11'", specifier = ">=2.9.0" },
|
||||
{ name = "httplib2", specifier = ">=0.32.0" },
|
||||
{ name = "packaging", specifier = ">=24.0" },
|
||||
{ name = "setuptools", specifier = ">=84.1.0" },
|
||||
{ name = "soupsieve", specifier = ">=2.8.4" },
|
||||
{ name = "tornado", specifier = ">=6.5.8" },
|
||||
{ name = "tornado", marker = "python_full_version >= '3.11'", specifier = ">=6.6" },
|
||||
]
|
||||
overrides = [
|
||||
{ name = "cryptography", specifier = ">=50.0.0,<51.0" },
|
||||
|
|
@ -579,7 +581,8 @@ dependencies = [
|
|||
{ name = "python-dotenv" },
|
||||
{ name = "requests" },
|
||||
{ name = "requests-toolbelt" },
|
||||
{ name = "tornado" },
|
||||
{ name = "tornado", version = "6.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" },
|
||||
{ name = "tornado", version = "6.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/27/0e/c2e369ad173fb3d76448e46d10beb3dcc53388318933ddf8169a3f21a810/aurelio_sdk-0.0.19.tar.gz", hash = "sha256:14107e7440ff2efd0b4a08c52fb595e7680bd4bc973a0ddfb3b64157c6666b91", size = 15258, upload-time = "2025-03-24T14:37:32.203Z" }
|
||||
wheels = [
|
||||
|
|
@ -4702,7 +4705,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "langgraph"
|
||||
version = "1.2.6"
|
||||
version = "1.2.17"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "langchain-core" },
|
||||
|
|
@ -4713,22 +4716,22 @@ dependencies = [
|
|||
{ name = "pydantic", version = "2.14.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
|
||||
{ name = "xxhash" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/02/7a/ea09b05bb0cbddfa43bd34fc581357e87fc3f21a751cc0d419688c3106da/langgraph-1.2.6.tar.gz", hash = "sha256:f9b45a34f13930c94d96cdb76277447ad2cc70ec2d18cd2764d7fdadb36cdc1b", size = 714400, upload-time = "2026-06-18T20:58:21.514Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/75/39/38b60bfc6fd064bc172693e19b3a6dccdaf9141bc9b984a9f83cdb06d014/langgraph-1.2.17.tar.gz", hash = "sha256:311516e313abd5424596d6c5250507c89be2e1973cbe813061696728803b0e10", size = 774002, upload-time = "2026-10-24T01:07:50.567Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/89/32/772db1b00a9fe42f50320d1aa20caefb76e621eff1f7218b9918093d631d/langgraph-1.2.6-py3-none-any.whl", hash = "sha256:1cf94d3ca124f84f77ce408fa1b06c3dee680a8aafffe364a8fd5d7d03eb8695", size = 246132, upload-time = "2026-06-18T20:58:20.335Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/d9/2e5b99319e2e8d39bc44d84836497c2e8f616554bb47843c8c819c8fed28/langgraph-1.2.17-py3-none-any.whl", hash = "sha256:e022a9fea64631fed946557a23f3fa5ac2ff38c72dcb999f426fc77d55ccde60", size = 260489, upload-time = "2026-10-24T01:07:49.155Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "langgraph-checkpoint"
|
||||
version = "4.1.1"
|
||||
version = "4.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "langchain-core" },
|
||||
{ name = "ormsgpack" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/83/47/886af6f886f0bff2273164a45f008694e48a96ff3cd25ff0228f2aa9480e/langgraph_checkpoint-4.1.1.tar.gz", hash = "sha256:6c2bdb530c91f91d7d9c1bd100925d0fc4f498d418c17f3587d1526279482a25", size = 184020, upload-time = "2026-05-22T16:57:38.503Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0f/69/31fdbdc65a85bbd6178afa193c772bb926620f47b4869638bc2bc80afaaa/langgraph_checkpoint-4.3.0.tar.gz", hash = "sha256:c75965d84cc2c1d549163e910a15bcb577758001b141619d05297c463280b018", size = 182652, upload-time = "2026-10-12T22:26:31.478Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/b4/71425e3e38be92611300b9cc5e46a5bf98ab23f5ea8a75b73d02a2f1413c/langgraph_checkpoint-4.1.1-py3-none-any.whl", hash = "sha256:25d29144b082827218e7bc3f1e9b0566a4bb007895cd6cc26f66a8428739f56e", size = 56212, upload-time = "2026-05-22T16:57:37.203Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/0c/84747e340bf4f29291c84cdd5733fc8d0a822f3d33bb24e664a18afa4a7c/langgraph_checkpoint-4.3.0-py3-none-any.whl", hash = "sha256:bedfafe2f997ded60e4fa593e79f56f436a6e45586392dc382aa810d0c751c64", size = 58063, upload-time = "2026-10-12T22:26:30.429Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4746,7 +4749,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "langgraph-sdk"
|
||||
version = "0.4.4"
|
||||
version = "0.4.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "httpx" },
|
||||
|
|
@ -4755,9 +4758,9 @@ dependencies = [
|
|||
{ name = "orjson" },
|
||||
{ name = "websockets" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3f/ae/91446c1fffa04a2dc1f81afbfb5bfff3590452891a72bd9098a656ab2657/langgraph_sdk-0.4.4.tar.gz", hash = "sha256:4e651ffa09de695681579396375377bdde23bedbc8e35b070e615cbd5af7da8b", size = 345789, upload-time = "2026-08-27T21:25:22.593Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4e/ef/e8f63b62dea6ecb75bef9d336f62e3e8f853b17a9943e4727c1bbeb09b11/langgraph_sdk-0.4.9.tar.gz", hash = "sha256:24d775ac1382b94c2595b8eb4eb4e0aa8dde95c905f18e017dc6574e1d79070d", size = 358868, upload-time = "2026-10-23T14:19:05.963Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/43/d5/2ad7f3a835c6fcebf58b6669552536ecd52d2dfe4cf49c6c36648fd83572/langgraph_sdk-0.4.4-py3-none-any.whl", hash = "sha256:39afe416c91742925e6f8a93715f566d499b36e1b636b804a4ffe3190e4f4e64", size = 162270, upload-time = "2026-08-27T21:25:21.072Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/7f/3102130eab099da9653ebe4a49123087a5889f4070eeee99b402255b1786/langgraph_sdk-0.4.9-py3-none-any.whl", hash = "sha256:c009243f4784758a07287fb2fd21fe47381c3d85f30479df2ae7f345c8703bb1", size = 167613, upload-time = "2026-10-23T14:19:04.82Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -10228,7 +10231,8 @@ dependencies = [
|
|||
{ name = "pyyaml" },
|
||||
{ name = "regex" },
|
||||
{ name = "tiktoken" },
|
||||
{ name = "tornado" },
|
||||
{ name = "tornado", version = "6.5.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or python_full_version >= '3.14'" },
|
||||
{ name = "tornado", version = "6.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/dc/a9/1a689e916e8b280f1fd8fb335cc059be626a22fe4533baa045d32fcd6de5/semantic_router-0.1.15.tar.gz", hash = "sha256:328256ddc3c2b713101ec69561d6585aecbf1198ea3461e1486289d8c3a35288", size = 95605, upload-time = "2026-05-23T12:58:15.444Z" }
|
||||
|
|
@ -10920,6 +10924,9 @@ wheels = [
|
|||
name = "tornado"
|
||||
version = "6.5.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/61/53d562a57b28c08eda40b258c0f975e360541943ad7c7bef897a40caafda/tornado-6.5.10.tar.gz", hash = "sha256:a6b1ccd08c04b4a06fb5aeb381be99de5ad1e5375c1785e31d78c880feb57687", size = 537910, upload-time = "2026-09-15T13:47:48.73Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/5b/ff5fc58fa2427c30dea74c90053f4fc5eda1e7f3833ed3ecc7147fe2b311/tornado-6.5.10-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9261783640e23258694a9ff0795df430a5a7b0a651d3dd53dd0969ad6be16da7", size = 465883, upload-time = "2026-09-15T13:47:35.463Z" },
|
||||
|
|
@ -10933,6 +10940,61 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/f3/1a/fd497f3a7f7b74bb04f4b94536b5c9f80742b5d50501fd27977652ddec16/tornado-6.5.10-cp39-abi3-win_arm64.whl", hash = "sha256:ce045d3c298fddd30e89a2777f97039d1b641eb9518ac7b26a4721903539c694", size = 467847, upload-time = "2026-09-15T13:47:47.283Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tornado"
|
||||
version = "6.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version == '3.13.*'",
|
||||
"python_full_version == '3.12.*'",
|
||||
"python_full_version == '3.11.*'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d0/02/5f549b1ad5da6267ed1e1fac2eef69257b1979cc812497002499b7895ff9/tornado-6.6.tar.gz", hash = "sha256:34c4aa916d7f2b2d5cadee2c6850f5a4a7f3d688cbf58399ea9bd3057af4abd5", size = 617998, upload-time = "2026-10-21T15:31:04.733Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/df/f5/d3e6f76b945f9812842db742d0708a3b6287e850e81be3ca10e290c103c6/tornado-6.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:01abf15429d64f04ccfafc238b191cc2c177108b7a331f4461707ad77198c8c2", size = 528326, upload-time = "2026-10-21T15:30:05.528Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/69/b20cf3a8bf641ec792a5a6050aa153e8f120604f9c23471142de535113e5/tornado-6.6-cp311-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:11bb97c00981eb8254e55bd6a5ad960c983ae343fef8ae3ae5584cd1fb70236a", size = 529544, upload-time = "2026-10-21T15:30:07.126Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/f4/d9d8d3ac345aa9df3a132ad03bd443160c2ab24ad4003b4e8d6d5af06371/tornado-6.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e152a3b2d3eae113b836d979823f7bb2320f7e334a84449729964921f374a4c", size = 530523, upload-time = "2026-10-21T15:30:08.478Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ad/82/1296ade1c14dee5a4906e5ca092997622ece7c4dc768c68ed9caaf895d1b/tornado-6.6-cp311-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e912d4c4babd04db6f34b25b748174a4a176debe86c8610beedf1815f9af7fdd", size = 530739, upload-time = "2026-10-21T15:30:09.98Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/66/30/6ac1eab66c81af46dafd93c9b24bd311f82e199d9d7b8ce2a64a91b6a200/tornado-6.6-cp311-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33aabe8a03d426cef4e0fe5669f0643cadd983b54957f1a7ac2789ec326e9644", size = 529738, upload-time = "2026-10-21T15:30:11.498Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/b4/6c1f6aa1fd148e4d0f500a10cd0c9949e7bf385e161ff497c81de5b3ae4f/tornado-6.6-cp311-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9aaf0e1c2fb9f046f30e2036b8d2060666399e8464d0a72769e43c754089567b", size = 527908, upload-time = "2026-10-21T15:30:13.298Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/5b/67d3d452b1179bc9e265cebd381a56ab4cf2d7b2a54607aedd8d6e1ebc2e/tornado-6.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61fc5513cea97720ab9bf54b04827ac0f4500c63f0b072b963d6145c798b4661", size = 530367, upload-time = "2026-10-21T15:30:15.183Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/14/4e8e9a14f77bcafb2408634b2d4c8c676ad7a43bf8dac4ee2d48370698f9/tornado-6.6-cp311-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:31e1a0d5cd704d2fe8a008c1cc27a0e2b67269ca263630a27985dddc160b5537", size = 530456, upload-time = "2026-10-21T15:30:16.741Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/12/e3ddef99a2089ccfc86f32bd3f39caaad87bee4868be1bacbf213a0b07a2/tornado-6.6-cp311-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:74416e41e9bdf70fbc2fb242f6fbbc7a58ec11e59d39f15fc5a421ecb6845118", size = 529610, upload-time = "2026-10-21T15:30:18.031Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/3d/62a692343d46b37e852e5078e0f6513534d925f70a04580072e3c127b129/tornado-6.6-cp311-abi3-musllinux_1_2_s390x.whl", hash = "sha256:8cdd69104ae9d705b3f04d2563aa96e357043e267a455802daae9a1e633a37c8", size = 529803, upload-time = "2026-10-21T15:30:19.326Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/04/6c59a9413adfe4810f648a4dd9fdf476f13b0495f9cc25ef856b71f9ce1e/tornado-6.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fc7c8a3263501fbe97518cd699101bc3c5f499a7dd4f75fc6cbb3bd9ee1db1f1", size = 529751, upload-time = "2026-10-21T15:30:20.826Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/36/a1b3b84cd347ae43f982d3bf95628de6c6365668dba2543af47c6a1c888e/tornado-6.6-cp311-abi3-win32.whl", hash = "sha256:b54ca69ff93d0374efb083a12a1e9ec2dd2036e8496d197e0e8ef77b8d9135c3", size = 530643, upload-time = "2026-10-21T15:30:22.428Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/50/dc813fe35a672e14ee8bed8b5c634eeb343a05c98eb455318a169de24567/tornado-6.6-cp311-abi3-win_amd64.whl", hash = "sha256:48f26ed9d7beb64e5f06adc864524e581ab6d62ba8ff46eaaa1ac2d0e6d753fd", size = 530886, upload-time = "2026-10-21T15:30:23.975Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/71/62a9b9e980026d38874740d0a7c162f53eb0563f6a441e3aff84c48db7e8/tornado-6.6-cp311-abi3-win_arm64.whl", hash = "sha256:29114ffd4e4de869d390393df9e9a0f191065aae1a96f38a58d35e464c04a867", size = 530899, upload-time = "2026-10-21T15:30:25.264Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/4c/633e3568eb574df555d6ddd2dac44b7793a5c26cd63b444e653fcfb1d6ec/tornado-6.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:63fc8d243b52fd8dd9ed4ad02d0df08f2024f9aea632a433a8a6c91075076d44", size = 528432, upload-time = "2026-10-21T15:30:26.578Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/eb/b525d1870ad28a56f47b664dd8fd3fab4f7ddd237c87c14e97248bf4762f/tornado-6.6-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d4b85cac795bfc6a7f20ba81871abb8c2ae8fba019061b8a9ae1c2f97e29637", size = 532586, upload-time = "2026-10-21T15:30:28.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/35/909a33c253352346ca081a27ca66b45c1398468adf784b767597ca773c28/tornado-6.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e035c065edadfa696529d05eebcbeb25154c129b5aee6dbfb697a6abf5d4f697", size = 533641, upload-time = "2026-10-21T15:30:29.318Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/64/100202666d6e3d96288ab184827762415456c139798a15c962547f9fabf9/tornado-6.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5b94e589ab855090c448b5c0937b3a75f1c1a604b7b36207e8236d209f720b9c", size = 533869, upload-time = "2026-10-21T15:30:30.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/e0/af3ed62bc15830d60328d5b4a1a9076b35ea0ba142cb17b1c1772cb15c59/tornado-6.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a394c8c9db9137f2a2b42869d9f7b91398fcf19da120d0dde4c4546172710f7", size = 532767, upload-time = "2026-10-21T15:30:32.273Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/d7/1e2e1a6932eb62d63324b64dba2f62e6eaf79ad883dd209352c297b7848c/tornado-6.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0215d1df6e4b5e736ddad5adf0af4138cf7437fdd39c77c5e69768e37850a632", size = 530956, upload-time = "2026-10-21T15:30:33.746Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/65/3b9a0e53d7a4c39f2ebe95486540fa8eeb67bfb6e8c437d603c00d35b128/tornado-6.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4d711790e0a9576ad924d942c4607e4e401d697588ea6f30f6662824008a7add", size = 533372, upload-time = "2026-10-21T15:30:35.085Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/48/90979cbfaa3ff941324a2605eb794af0f087c670cd99f18a69ab40498073/tornado-6.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:034a66bae0f55421d007803e2385d2a59707d8829b2f026373d42f08256e9922", size = 533425, upload-time = "2026-10-21T15:30:36.425Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/c4/ce743a86d91f85c255b77a4282eb5b2716b61e74430eb9d1df8578a539d2/tornado-6.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5cb38483b077129056a94dd415a71d29ab4e42b46da894fee8b4657003f10b21", size = 532545, upload-time = "2026-10-21T15:30:37.789Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/5e/a82131e35778540fc19139229d02aa8cf2453a8022e202be50c88f0f8ac3/tornado-6.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:46bbbd107efa37777ac4b83601866d19b4dbd5b770a7a9123582633d95959ad3", size = 532700, upload-time = "2026-10-21T15:30:39.336Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/09/6dac8c2c677c65d60b3d7dae94176dbfe392f314e7c8e2d709623edd6a26/tornado-6.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b85e9d0263cdbc027b95947c8b873a7bc7fd6a08d16bfb4f5047350b0808e464", size = 532712, upload-time = "2026-10-21T15:30:40.698Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/ac/71e5776cbcdc3264e81aa6db28c69717ba50607a9eb8d5b766f11a4b020e/tornado-6.6-cp314-cp314t-win32.whl", hash = "sha256:4276eef0880e5fe4488e6799c2842f59af3b06637e9313ab4fb506f9c9c2d6cb", size = 530491, upload-time = "2026-10-21T15:30:42.115Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d8/0f/0b6a6908a5b5aa8320009dfdb84434b1cf160b577a00456e59ab9e6ea598/tornado-6.6-cp314-cp314t-win_amd64.whl", hash = "sha256:10d4af98c6df09f8429b1340ccaea3a72887b86f695dfb8056bda517c8115a00", size = 530780, upload-time = "2026-10-21T15:30:43.414Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/39/c152e1cc5371f75f5f9f0a4f44bf2dc9396be9407d384de7428446fb4651/tornado-6.6-cp314-cp314t-win_arm64.whl", hash = "sha256:b1f0ef20a5278854f5fdac31637bafadb055405df28b7fc945e54add12b172e3", size = 530781, upload-time = "2026-10-21T15:30:44.741Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/c5/f96c7d7a8895745bb37dfb6e6107e1ee40ad02a4b162d7d6c143644455ab/tornado-6.6-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8f8e382da30fe042f27f27a871e21aefc2613870e5a41648de18446c72d6ea59", size = 528446, upload-time = "2026-10-21T15:30:46.112Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/dd/ab8f6500d274b9c55b025798eb54e92591bcff27515799bec443bd54e35a/tornado-6.6-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8c7a8a328396c2195454a2ecd9499512441bbce99505544a717e6a87d16ff6f9", size = 532774, upload-time = "2026-10-21T15:30:47.393Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/23/b78e7d1bf329160b8157284005395421ea941864c16c7f04c3e48c98bcf9/tornado-6.6-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca025bb0da81c294aef26f27427316df1525220820315f4bf32dc1e84a4d2232", size = 533872, upload-time = "2026-10-21T15:30:48.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/4f/ab66b614ad0bc9a219484be6bc2c002c8de9916a7b6c7b27fe6273838e5c/tornado-6.6-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:59c22ebc905ac2a267f8d9d8a878f86afbcbbde329d75c485f1422fc45a81b83", size = 534027, upload-time = "2026-10-21T15:30:50.157Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/63/501fa15a36bace2f4b9158228cf922d606403261bf71c960c433ec3ca0a4/tornado-6.6-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f89802bc0bcd15f988d9325b4e6d2171bb04ea68d188732556335e10d3fd00bd", size = 532936, upload-time = "2026-10-21T15:30:51.45Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/b7/82d643f635f0300bda98613ca2660b4f3134b6fb594c6fddfa4de706806c/tornado-6.6-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:70043256d5958f9c655066153d23d60ea3e29bf9854f7fb75850e223c477758a", size = 531254, upload-time = "2026-10-21T15:30:52.865Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/53/92c40891a94750bc09257310af964bf25c59e3531d6199736c318baa864f/tornado-6.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:82b49495e3849d644b841ef33b35491160055997deaa93720ff647ef019421db", size = 533566, upload-time = "2026-10-21T15:30:54.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/88/b82f5c5efde731b88d839ad99cb6ace2a521f03bb6d7346d2de089c76da8/tornado-6.6-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:07b2bcaa4bf447bdbfb664e4c28344226540db3f0ebccd74886528338a240765", size = 533567, upload-time = "2026-10-21T15:30:55.451Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/2c/e9a7f5d0fca0bdbcd48b2d9c7884dd0b32669c9945d98bc6d303a80ecfea/tornado-6.6-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:c36b921374c302e54909bae17d9f354061d80e7ae7d8c94f73abbcc217866460", size = 532849, upload-time = "2026-10-21T15:30:56.729Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/f5/4ade737340ca6ce9c677ea0c2843591c8ad065d07f0913c28d922c7e078e/tornado-6.6-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:7412d9e37aa9f7048e4b591f520a95f7a097278ffab15f1ddf3f5c2320cf6648", size = 532873, upload-time = "2026-10-21T15:30:58.067Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/04/6acdc937d0bb6189ddb34ceded517d7bd1c7b3bded5d4e1b3442160e6b6f/tornado-6.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:15bc95112b2d24b0bfaee9a004e3f974da929e33b4244a7f695a22f45b08a5e5", size = 532883, upload-time = "2026-10-21T15:30:59.395Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/27/0a766fac6b1cc629d96346329a0485287fd2857c51a887a3a154f2a69938/tornado-6.6-cp315-cp315t-win32.whl", hash = "sha256:f661c21117b875d612371f3eb243d0c69f56d9bdd6aee38b7c24453419dfe554", size = 530488, upload-time = "2026-10-21T15:31:00.77Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/80/3e4142570883da27b3f60ae82c638ac96e60997eac684740613557519c96/tornado-6.6-cp315-cp315t-win_amd64.whl", hash = "sha256:ee8853ab35da510dc952879ebd9995c82bc10ffb7925fd87930dc0d6916f4b39", size = 530777, upload-time = "2026-10-21T15:31:02.087Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/57/4a85c617b6349962320b4afbe1386d5b05fd57218ee8ad6e82e0110ec9ee/tornado-6.6-cp315-cp315t-win_arm64.whl", hash = "sha256:7de62e2dfdf559fd7154d2b9aa58ff3d9cff8304ae0519341feb082618bc2a33", size = 530780, upload-time = "2026-10-21T15:31:03.425Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.67.3"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue