mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts: # litellm/router.py
This commit is contained in:
commit
fabc8e348d
882 changed files with 67325 additions and 9210 deletions
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GITHUB_ACTIONS:-}" = true ]; then
|
||||
echo "Integration contracts are owned by CircleCI" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
suite="${1:?integration suite required}"
|
||||
mode="${2:-standard}"
|
||||
side="${3:-}"
|
||||
requested_workers="${INTEGRATION_WORKERS:-}"
|
||||
license_env=()
|
||||
if [ -n "${LITELLM_LICENSE:-}" ]; then
|
||||
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
|
||||
fi
|
||||
if [ "$mode" = replica ]; then
|
||||
results="test-results/integration-${suite}-replica"
|
||||
elif [ "$mode" = parity ]; then
|
||||
|
|
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
|
|||
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
|
||||
test -f "$LITELLM_UI_PATH/index.html"
|
||||
fi
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
|
||||
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
|
||||
|
|
@ -107,7 +107,7 @@ sudo iptables -N integration_only
|
|||
guard_created=true
|
||||
sudo iptables -A integration_only -o lo -j ACCEPT
|
||||
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
for service in postgres-db redis-cache; do
|
||||
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
|
||||
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
|
||||
sudo iptables -A integration_only -d "$address" -j ACCEPT
|
||||
done
|
||||
|
|
@ -145,6 +145,9 @@ fi
|
|||
if [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
|
||||
fi
|
||||
if [ -n "$requested_workers" ]; then
|
||||
export INTEGRATION_WORKERS="$requested_workers"
|
||||
fi
|
||||
isolated_workers=false
|
||||
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
|
||||
isolated_workers=true
|
||||
|
|
@ -185,7 +188,7 @@ start_proxy() {
|
|||
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
|
||||
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
|
||||
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
|
||||
|
|
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|||
"${shared_services[@]}" "${worker_coverage[@]}" \
|
||||
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
||||
LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
||||
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
|
||||
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
|
||||
|
|
|
|||
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
|
@ -1,5 +1,6 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
/.github/workflows/zizmor.yml @yuneng-berri
|
||||
|
|
|
|||
7
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
7
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
|
|||
## Screenshots / Proof of Fix
|
||||
|
||||
<!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
|
||||
The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
|
||||
Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
|
||||
Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
|
||||
Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
|
||||
Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
|
||||
Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
|
||||
Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
|
||||
|
||||
### Before (<hash>)
|
||||
|
||||
|
|
|
|||
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TextIO
|
||||
|
||||
SECRET_VARIABLE: Final = "SCANNED_SECRET"
|
||||
MIN_ENCODED_LENGTH: Final = 12
|
||||
MAX_ARCHIVE_DEPTH: Final = 3
|
||||
MASK: Final = "***"
|
||||
|
||||
Forms = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hit:
|
||||
location: str
|
||||
form: str
|
||||
|
||||
|
||||
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
|
||||
for offset in range(3):
|
||||
chunk: Final = raw[offset:]
|
||||
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
|
||||
|
||||
|
||||
def secret_forms(secret: str) -> Forms:
|
||||
encoded: Final = (
|
||||
("url-encoded", urllib.parse.quote(secret, safe="")),
|
||||
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
|
||||
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
|
||||
)
|
||||
by_value: Final = MappingProxyType(
|
||||
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
|
||||
)
|
||||
return (("raw", secret), *((name, value) for value, name in by_value.items()))
|
||||
|
||||
|
||||
def mask(text: str, forms: Forms) -> str:
|
||||
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
|
||||
|
||||
|
||||
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value in name:
|
||||
yield Hit(mask(location, forms), f"{form} in name")
|
||||
|
||||
|
||||
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value.encode() in data:
|
||||
yield Hit(mask(location, forms), form)
|
||||
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
|
||||
return
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
for member in archive.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
|
||||
|
||||
|
||||
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
|
||||
for directory, subdirectories, files in os.walk(root):
|
||||
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
|
||||
yield from ((Path(directory, name), False) for name in files)
|
||||
|
||||
|
||||
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
|
||||
location: Final = str(path.relative_to(root))
|
||||
if linked_directory:
|
||||
yield Hit(mask(location, forms), "directory symlink, not scanned")
|
||||
return
|
||||
yield from _name_hits(location, location, forms)
|
||||
yield from _content_hits(location, path.read_bytes(), forms, 0)
|
||||
|
||||
|
||||
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
|
||||
forms: Final = secret_forms(secret)
|
||||
entries: Final = tuple(_walk(root))
|
||||
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
|
||||
return len(entries), hits
|
||||
|
||||
|
||||
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
|
||||
if len(argv) != 1:
|
||||
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
|
||||
return 2
|
||||
secret: Final = environ.get(SECRET_VARIABLE, "")
|
||||
root: Final = Path(argv[0])
|
||||
if not secret:
|
||||
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
|
||||
return 0
|
||||
if not root.is_dir():
|
||||
print(f"{root} is not a directory, nothing to scan", file=out)
|
||||
return 0
|
||||
try:
|
||||
scanned, hits = scan(root, secret)
|
||||
except Exception as error:
|
||||
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
|
||||
return 1
|
||||
for hit in hits:
|
||||
print(f"secret found: {hit.location} ({hit.form})", file=out)
|
||||
if hits:
|
||||
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
|
||||
return 1
|
||||
print(f"no copy of the secret in {scanned} files under {root}", file=out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))
|
||||
9
.github/scripts/smoke_test_native_wheel.py
vendored
9
.github/scripts/smoke_test_native_wheel.py
vendored
|
|
@ -7,6 +7,8 @@ import zipfile
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json"
|
||||
|
||||
CHILD_SCRIPT: Final = """
|
||||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from pathlib import Path
|
||||
|
|
@ -34,6 +36,9 @@ else:
|
|||
after = module.gil_stats()
|
||||
if not isinstance(after.get("releases"), int):
|
||||
raise AssertionError(f"native module unusable after panic: {after!r}")
|
||||
|
||||
if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes():
|
||||
raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout")
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -55,7 +60,9 @@ def main() -> int:
|
|||
|
||||
native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
|
||||
native_path.write_bytes(archive.read(native_members[0]))
|
||||
result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
|
||||
result: Final = subprocess.run(
|
||||
(sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")
|
||||
|
|
|
|||
33
.github/workflows/cost-map-guard.yml
vendored
33
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -21,16 +21,18 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head and its merge base
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
|
|
@ -38,7 +40,30 @@ jobs:
|
|||
- name: Run the guard
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
|
||||
COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
|
||||
- name: Check whether the pull request changes the cost map
|
||||
id: changed
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
|
||||
echo "cost_map=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "cost_map=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- if: steps.changed.outputs.cost_map == 'true'
|
||||
run: rustup toolchain install --no-self-update
|
||||
- name: Load the cost map through model-catalog
|
||||
if: steps.changed.outputs.cost_map == 'true'
|
||||
env:
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
|
||||
cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
|
||||
-- "$RUNNER_TEMP/model_prices_and_context_window.json"
|
||||
|
|
|
|||
91
.github/workflows/cost-map-pins-recheck.yml
vendored
Normal file
91
.github/workflows/cost-map-pins-recheck.yml
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: Cost map pins recheck
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- model_prices_and_context_window.json
|
||||
- litellm/model_prices_and_context_window_backup.json
|
||||
- ci_cd/cost_map_pins.json
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
rerun-cost-map-guard:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Re-run the cost map guard on open pull requests the push could break
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.ref_name }}
|
||||
BEFORE_SHA: ${{ github.event.before }}
|
||||
AFTER_SHA: ${{ github.event.after }}
|
||||
FORCED: ${{ github.event.forced }}
|
||||
COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
|
||||
PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
|
||||
run: |
|
||||
if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
|
||||
|| ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
|
||||
--jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
|
||||
| grep -qx true; then
|
||||
export RECHECK_PATHS="$COST_MAP_PATHS"
|
||||
else
|
||||
export RECHECK_PATHS="$PINS_PATHS"
|
||||
fi
|
||||
rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
|
||||
candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
|
||||
-f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
|
||||
nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
|
||||
pageInfo { hasNextPage endCursor }
|
||||
}
|
||||
}
|
||||
}' \
|
||||
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
|
||||
| {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
|
||||
| select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
|
||||
failed=0
|
||||
declare -A rerun_heads
|
||||
while read -r number head_sha listed; do
|
||||
[ -n "$number" ] || continue
|
||||
if [ "$listed" = false ]; then
|
||||
if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
|
||||
--jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
|
||||
echo "::error::could not list the files of #$number"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
[ -n "$changed" ] || continue
|
||||
fi
|
||||
[ -z "${rerun_heads[$head_sha]:-}" ] || continue
|
||||
rerun_heads[$head_sha]=1
|
||||
if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
|
||||
echo "::error::could not list the cost map guard runs of #$number"
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
if [ -z "$runs" ]; then
|
||||
echo "::warning::no cost map guard run found for #$number"
|
||||
continue
|
||||
fi
|
||||
while read -r run_id created_at; do
|
||||
if [[ "$created_at" < "$rerun_cutoff" ]]; then
|
||||
echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
|
||||
continue
|
||||
fi
|
||||
gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
|
||||
if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
|
||||
echo "::error::could not re-run cost map guard run $run_id for #$number"
|
||||
failed=1
|
||||
fi
|
||||
done <<< "$runs"
|
||||
done <<< "$candidates"
|
||||
exit "$failed"
|
||||
2
.github/workflows/image-scan.yml
vendored
2
.github/workflows/image-scan.yml
vendored
|
|
@ -13,8 +13,10 @@ on:
|
|||
- migrations/Dockerfile
|
||||
- migrations/run.py
|
||||
- gateway/Dockerfile
|
||||
- gateway/launch.py
|
||||
- gateway/main.py
|
||||
- gateway/routes/allowlist.py
|
||||
- litellm_cli/**
|
||||
- backend/Dockerfile
|
||||
- backend/main.py
|
||||
- litellm-rust/**
|
||||
|
|
|
|||
199
.github/workflows/test-integration.yml
vendored
Normal file
199
.github/workflows/test-integration.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
name: integration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
schedule:
|
||||
- cron: "5 8-20/3 * * 1-6"
|
||||
timezone: America/Los_Angeles
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
suite:
|
||||
description: Integration group to run
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- providers
|
||||
- extensions
|
||||
- accounting
|
||||
- management
|
||||
- management-replica
|
||||
- database
|
||||
- database-replica
|
||||
- mcp
|
||||
- security
|
||||
- cost
|
||||
- sdk
|
||||
- browser
|
||||
default: all
|
||||
workers:
|
||||
description: Workers per group, blank keeps each group's default
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
integration:
|
||||
name: integration ${{ matrix.suite }}
|
||||
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
|
||||
environment: integration-license
|
||||
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: circle_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--shm-size 1g
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
|
||||
|
||||
- name: Install redis-server 7.2.16
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/redis-bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
|
||||
docker rm "$cid"
|
||||
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
|
||||
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install MCP SDK1 peer
|
||||
if: matrix.suite == 'mcp'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: |
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
tests/e2e/ui/package-lock.json
|
||||
|
||||
- name: Install browser dependencies and build the dashboard
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
(cd ui/litellm-dashboard && npm ci)
|
||||
(cd tests/e2e/ui && npm ci)
|
||||
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
|
||||
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
|
||||
|
||||
- name: Run integration contracts
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SUITE: ${{ matrix.suite }}
|
||||
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
|
||||
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: |
|
||||
case "$SUITE" in
|
||||
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
|
||||
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
|
||||
esac
|
||||
|
||||
- name: Verify every case ran exactly once
|
||||
if: ${{ !cancelled() && matrix.suite != 'browser' }}
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
|
||||
test -n "$record"
|
||||
mkdir -p "$RUNNER_TEMP/shards/0"
|
||||
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
|
||||
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
|
||||
|
||||
- name: Report MCP coverage
|
||||
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
coverage_dir=test-results/integration-mcp/coverage
|
||||
test -f "$coverage_dir/coverage.txt" && exit 0
|
||||
export COVERAGE_FILE="$PWD/$coverage_dir/data"
|
||||
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
|
||||
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
|
||||
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
|
||||
tail -n 1 "$coverage_dir/coverage.txt"
|
||||
|
||||
- name: Check results for the license
|
||||
id: secret-scan
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: python .github/scripts/assert_secret_absent.py test-results
|
||||
|
||||
- name: Upload results
|
||||
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
|
||||
timeout-minutes: 10
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: test-results/
|
||||
retention-days: 14
|
||||
4
.github/workflows/test-linting.yml
vendored
4
.github/workflows/test-linting.yml
vendored
|
|
@ -376,10 +376,10 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
files="$(git diff --name-only "$base" HEAD)"
|
||||
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
|
||||
if grep -Eq '^(litellm/|enterprise/|(pyproject\.toml|uv\.lock)$|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No proxy, types or generator changes in this pull request, nothing to verify."
|
||||
echo "No litellm, enterprise, dependency or generator changes in this pull request, nothing to verify."
|
||||
echo "relevant=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ on:
|
|||
paths:
|
||||
- "terraform/provider/**"
|
||||
- "litellm/proxy/**"
|
||||
- "enterprise/**"
|
||||
- ".github/workflows/test-terraform-provider.yml"
|
||||
|
||||
permissions:
|
||||
|
|
|
|||
2
.github/workflows/test-unit.yml
vendored
2
.github/workflows/test-unit.yml
vendored
|
|
@ -236,6 +236,7 @@ jobs:
|
|||
tests/unit/llms/azure_ai
|
||||
tests/unit/llms/base_llm
|
||||
tests/unit/llms/baseten
|
||||
tests/unit/llms/bigmodel
|
||||
tests/unit/llms/black_forest_labs
|
||||
tests/unit/llms/brave
|
||||
tests/unit/llms/bytez
|
||||
|
|
@ -321,6 +322,7 @@ jobs:
|
|||
tests/unit/llms/openai_like
|
||||
tests/unit/llms/opencode
|
||||
tests/unit/llms/opensandbox
|
||||
tests/unit/llms/opensearch
|
||||
tests/unit/llms/openrouter
|
||||
tests/unit/llms/ovhcloud
|
||||
tests/unit/llms/parallel_ai
|
||||
|
|
|
|||
|
|
@ -11,3 +11,12 @@ ignore:
|
|||
package:
|
||||
name: zlib
|
||||
type: apk
|
||||
- vulnerability: GHSA-v4x8-3wr5-3wq8
|
||||
reason: >-
|
||||
filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
|
||||
the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
|
||||
and the image then moves off 3.32.7, which retires this version-scoped rule
|
||||
package:
|
||||
name: filelock
|
||||
version: 3.32.7
|
||||
type: python
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
|
|||
|
||||
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
|
||||
|
||||
Before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging, read the architecture decision records in `docs/adrs/`, which say why those layers are shaped the way they are
|
||||
|
||||
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
|
||||
|
||||
When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
|
||||
|
|
@ -39,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
|
|||
|
||||
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
|
||||
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
|
||||
Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
|
||||
|
||||
For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
|
||||
|
||||
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
|
|
@ -105,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
|
|||
- No file sprawl: deliberate file and folder structure
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
|
||||
|
||||
Follow conventional commits for commit names and PR titles
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
This document helps contributors understand where to make changes in LiteLLM.
|
||||
|
||||
For why a layer is shaped the way it is, read the architecture decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) (Python SDK and proxy) and [`litellm-rust/docs/adrs/`](litellm-rust/docs/adrs/AGENTS.md) (Rust core)
|
||||
|
||||
---
|
||||
|
||||
## How It Works
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
|
|||
|
||||
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
|
||||
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
|
||||
- [ ] **Prove it works end to end** - If your change affects what the proxy or SDK does, paste the commands you ran against a live proxy and their output in the PR - [see details](#proving-your-change-works-end-to-end)
|
||||
|
||||
#### Proxy (Backend) PRs
|
||||
|
||||
|
|
@ -248,6 +249,41 @@ If `make test-unit` fails:
|
|||
- **Test edge cases**: Don't just test the happy path
|
||||
- **Update documentation**: If you change APIs, update docs
|
||||
|
||||
## Proving Your Change Works End to End
|
||||
|
||||
Tests are required, but they do not prove that your change works for a user. A PR that changes what the proxy or SDK does shows a real run in its "Screenshots / Proof of Fix" section, meaning the exact commands you sent to a live proxy and the output that came back, against real provider APIs and not mocks. `pytest` output does not count as proof, because a passing test only shows that the code does what its mocks expect. A PR that changes only docs or CI shows the rendered page or the CI run instead
|
||||
|
||||
A good proof looks like this:
|
||||
|
||||
1. Install the proxy dependencies with `make install-proxy-dev`, then start the proxy from your branch with a config that has the model you changed, `uv run litellm --config your_config.yaml --detailed_debug`
|
||||
2. Send the request a user would send with `curl`, and paste both the command and the response. When cost or routing is involved, include the response headers, since `x-litellm-response-cost` is what the user sees
|
||||
3. For a bug fix, do this twice, once at the merge base with BerriAI/litellm's main (`git merge-base origin/main HEAD`, with your remote for BerriAI/litellm in place of `origin` in a fork clone) to show the bug and once at your branch's latest commit to show the fix. Name both commit hashes
|
||||
4. If your change is reachable from more than one endpoint (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`), show each one
|
||||
5. If the request can stream, show a `"stream": true` run too, because streaming builds usage and cost on a different path. A stream sends its headers before its usage exists, so show the `usage.cost` on the last chunk, which `/v1/chat/completions` sends when the request has `"stream_options": {"include_usage": true}` and other endpoints send with `include_cost_in_streaming_usage: true` under `litellm_settings`, or, when the proxy has a database, the request's row on the Admin UI's Logs page or `GET /spend/logs`
|
||||
6. For UI changes, add before and after screenshots and name the page
|
||||
7. If users reach your change through a coding tool such as Claude Code or Codex, drive that tool the way a user does and add before and after screenshots of it, because a `curl` or a headless run alone does not show what the user sees
|
||||
|
||||
```bash
|
||||
curl -sD - http://localhost:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "your-model", "messages": [{"role": "user", "content": "hi"}]}'
|
||||
```
|
||||
|
||||
When the provider reports its own usage or spend numbers, compare the proxy's numbers against them in the proof
|
||||
|
||||
## Contributing an LLM Provider Integration
|
||||
|
||||
Provider work (a new provider, a new parameter, a new usage or cost field) is the most common kind of contribution, and it is where PRs get reworked most often. Follow these rules
|
||||
|
||||
**Find the existing machinery before you add plumbing.** LiteLLM already has generic paths that move data between the endpoints, the provider transformations, and cost tracking. A second, hand-rolled path for the same data is a common reason a provider PR gets rewritten before merge. [`ARCHITECTURE.md`](ARCHITECTURE.md) shows where the layers live, and the decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) explain why they are shaped that way. For example, [the ADR on provider usage fields](docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md) covers how provider-specific usage fields reach cost tracking. If you cannot find a mechanism for what you need, say so in the PR or ask in [#pr-review on Slack](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) before you build one
|
||||
|
||||
**Bill a new provider usage field from the usage object.** Leave the field on the usage object as an extra and read it in that provider's `cost_calculator.py`. A provider without one gets a new `cost_calculator.py` and an arm for it in `cost_per_token` (`litellm/cost_calculator.py`), which picks the calculator by provider. If the provider's transform builds `Usage` from named fields, as the Anthropic, Gemini, and Bedrock Converse ones do, pass the new field there too. Streamed requests rebuild usage in `ChunkProcessor.calculate_usage` (`litellm/litellm_core_utils/streaming_chunk_builder_utils.py`), which does not copy extras yet the way the Responses-to-chat bridge does (minus a short exclusion list), so until it does, bill the field on streams by naming it there, as Anthropic's `inference_geo` is. A provider that reports its own charge for the request, as OpenRouter does in `usage.cost`, is billed at that charge from `_hidden_params` before any calculator runs, unless the deployment sets its own prices. When the field counts a built-in tool such as web search, add the provider to `get_cost_for_web_search_request` in `litellm/llms/__init__.py`, and add a gate to `litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py` only when no existing gate fires. Then show the field's cost in your proof on every endpoint it reaches, streaming included
|
||||
|
||||
**Do not change what a caller sees to make internals easier.** Each endpoint keeps the schema of the API it emulates. A `/v1/responses` caller gets `input_tokens` and a `/v1/chat/completions` caller gets `prompt_tokens`, whatever the provider sent and whatever cost tracking needs. A provider usage field that shows up as an extra key is fine, since it only adds a key, but renaming, removing, or reshaping a key the endpoint defines is not. Reshaping a public response to feed an internal consumer breaks every user of that endpoint
|
||||
|
||||
If your change makes an architecture decision, such as a new cross-cutting mechanism or a deliberate difference from a provider's API shape, add an ADR to [`docs/adrs/`](docs/adrs/AGENTS.md) in the same PR
|
||||
|
||||
## Building and Running Locally
|
||||
|
||||
### LiteLLM Proxy Server
|
||||
|
|
|
|||
50
README.md
50
README.md
|
|
@ -141,12 +141,56 @@ response = client.chat.completions.create(
|
|||
)
|
||||
```
|
||||
|
||||
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Auto Routing</b> - Choose a model for each prompt (Python SDK + AI Gateway)</summary>
|
||||
|
||||
### Python SDK
|
||||
|
||||
[**Docs: Autorouter SDK**](https://docs.litellm.ai/docs/autorouter_sdk)
|
||||
|
||||
Install with `uv add litellm` and set `OPENAI_API_KEY` for the models below.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from litellm import Router
|
||||
|
||||
router = Router(model_list=[
|
||||
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-6-luna"}},
|
||||
{"model_name": "capable", "litellm_params": {"model": "openai/gpt-6.1-sol"}},
|
||||
{"model_name": "auto", "litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "heuristic",
|
||||
"tiers": {"SIMPLE": "fast", "MEDIUM": "fast",
|
||||
"COMPLEX": "capable", "REASONING": "capable"},
|
||||
"return_raw_model_name": True,
|
||||
},
|
||||
"complexity_router_default_model": "capable",
|
||||
}},
|
||||
])
|
||||
response = asyncio.run(router.acompletion(
|
||||
model="auto", messages=[{"role": "user", "content": "What is 2+2?"}],
|
||||
))
|
||||
print(response.model, response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### AI Gateway (Proxy Server)
|
||||
|
||||
[**Create an Auto Router**](https://docs.litellm.ai/docs/auto_router/setup), then call its model name. Replace `auto` and the URL below with yours, and set `LITELLM_API_KEY` to your virtual key.
|
||||
|
||||
```shell
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"auto","messages":[{"role":"user","content":"What is 2+2?"}]}'
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
|
||||
<summary><b>A2A</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
|
||||
|
||||
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
|
||||
|
||||
|
|
@ -442,7 +486,7 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
|
|||
<tr>
|
||||
<th style={{width: '14%'}}></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/python_sdk">LiteLLM Python SDK</a></strong></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0
|
|||
COPY pyproject.toml README.md rust-toolchain.toml ./
|
||||
COPY .cargo ./.cargo
|
||||
COPY litellm-rust ./litellm-rust
|
||||
COPY model_prices_and_context_window.json ./
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
mkdir -p litellm/rust_bridge /wheels && \
|
||||
|
|
|
|||
|
|
@ -1,37 +1,67 @@
|
|||
"""Guard the cost map on pull requests.
|
||||
|
||||
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
|
||||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
|
||||
file checks: the files parse, the backup copy matches the root file, and every capability pinned in
|
||||
ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
|
||||
otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
|
||||
a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
|
||||
all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
|
||||
whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
|
||||
litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
|
||||
only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
|
||||
(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
|
||||
Bot) get the human checks and may not touch the pins file either.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
PINS_PATH: Final = "ci_cd/cost_map_pins.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
|
||||
TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
|
||||
AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
|
||||
BOT_AUTHOR_TYPE: Final = "Bot"
|
||||
MERGE_CONFLICT_EXIT_CODE: Final = 1
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
CostMap = dict[str, object]
|
||||
_MISSING: Final = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Snapshot:
|
||||
cost_map: str
|
||||
backup: str
|
||||
pins: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MergeFailed:
|
||||
stderr: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pin:
|
||||
key: str
|
||||
field: str
|
||||
value: object
|
||||
checked: str
|
||||
source: str
|
||||
|
||||
|
||||
def _parse_object(text: str, path: str) -> CostMap | str:
|
||||
|
|
@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
|
|||
return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
|
||||
|
||||
|
||||
def _pin(key: str, field: str, spec: object) -> Pin | str:
|
||||
malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
|
||||
if not isinstance(spec, dict) or "value" not in spec:
|
||||
return malformed
|
||||
checked: Final = spec.get("checked")
|
||||
source: Final = spec.get("source")
|
||||
if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
|
||||
return malformed
|
||||
return Pin(key, field, spec["value"], checked, source)
|
||||
|
||||
|
||||
def _pins(parsed: CostMap) -> Iterator[Pin | str]:
|
||||
for key, fields in parsed.items():
|
||||
if not isinstance(fields, dict):
|
||||
yield f"{PINS_PATH}: {key} must map field names to pins"
|
||||
continue
|
||||
for field, spec in fields.items():
|
||||
yield _pin(key, field, spec)
|
||||
|
||||
|
||||
def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
|
||||
for pin in _pins(parsed):
|
||||
if isinstance(pin, str):
|
||||
yield pin
|
||||
elif pin.key not in entries:
|
||||
yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
|
||||
elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
|
||||
yield (
|
||||
f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
|
||||
f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
|
||||
)
|
||||
|
||||
|
||||
def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
|
||||
if not pins_text:
|
||||
return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
|
||||
parsed: Final = _parse_object(pins_text, PINS_PATH)
|
||||
if isinstance(parsed, str):
|
||||
return (parsed,)
|
||||
return tuple(_pin_violations(parsed, _entries(head_map)))
|
||||
|
||||
|
||||
def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
|
||||
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
|
||||
if isinstance(base_map, str):
|
||||
|
|
@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
)
|
||||
|
||||
|
||||
def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
|
||||
if PINS_PATH not in changed_files:
|
||||
return ()
|
||||
if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
|
||||
return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
|
||||
if author_type == BOT_AUTHOR_TYPE:
|
||||
return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
|
||||
return ()
|
||||
|
||||
|
||||
def touches_cost_map(changed_files: Sequence[str]) -> bool:
|
||||
return any(path in GUARDED_PATHS for path in changed_files)
|
||||
return any(path in CHECKED_PATHS for path in changed_files)
|
||||
|
||||
|
||||
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
||||
|
|
@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
|||
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
|
||||
|
||||
|
||||
def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
|
||||
if merged is None:
|
||||
return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
|
||||
if not merged.pins and not base.pins:
|
||||
return ()
|
||||
merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
|
||||
if isinstance(merged_map, str):
|
||||
return (f"after merging into the base branch, {merged_map}",)
|
||||
return pin_failures(merged.pins, merged_map)
|
||||
|
||||
|
||||
def guard_failures(
|
||||
base: Snapshot,
|
||||
head: Snapshot,
|
||||
merged: Snapshot | None,
|
||||
changed_files: Sequence[str],
|
||||
bot: bool,
|
||||
*,
|
||||
|
|
@ -147,6 +241,7 @@ def guard_failures(
|
|||
*_file_failures(
|
||||
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
|
||||
),
|
||||
*_merged_pin_failures(base, merged),
|
||||
*(_bot_failures(base, head_map, changed_files) if bot else ()),
|
||||
)
|
||||
|
||||
|
|
@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
|
|||
|
||||
|
||||
def snapshot(revision: str) -> Snapshot:
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
|
||||
|
||||
|
||||
def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
|
||||
result: Final = subprocess.run(
|
||||
("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
tree: Final = result.stdout.split("\n", 1)[0]
|
||||
if result.returncode == 0:
|
||||
return tree
|
||||
if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
|
||||
return None
|
||||
return MergeFailed(result.stderr.strip())
|
||||
|
||||
|
||||
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
|
||||
|
|
@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base", required=True, help="merge base of the pull request")
|
||||
parser.add_argument("--head", required=True, help="head commit of the pull request")
|
||||
env_target: Final = os.environ.get(TARGET_ENV)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
default=env_target,
|
||||
required=env_target is None,
|
||||
help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
|
||||
)
|
||||
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
|
||||
args: Final = parser.parse_args(argv)
|
||||
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
|
||||
|
|
@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
|
|||
if changed is None:
|
||||
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
|
||||
return 1
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
|
||||
if not bot and not touches_cost_map(changed):
|
||||
print(f"cost map guard passed ({contract_for(bot, changed)})")
|
||||
return 0
|
||||
merged: Final = merged_tree(args.base, args.target, args.head)
|
||||
if isinstance(merged, MergeFailed):
|
||||
print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
|
||||
return 1
|
||||
failures: Final = (
|
||||
*guard_failures(
|
||||
snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
|
||||
),
|
||||
*pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
|
||||
)
|
||||
contract: Final = contract_for(bot, changed)
|
||||
if failures:
|
||||
print(f"cost map guard failed ({contract}):")
|
||||
|
|
|
|||
16
ci_cd/cost_map_pins.json
Normal file
16
ci_cd/cost_map_pins.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"fireworks_ai/accounts/fireworks/models/minimax-m3": {
|
||||
"supports_vision": {
|
||||
"value": true,
|
||||
"checked": "2026-09-26",
|
||||
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
|
||||
}
|
||||
},
|
||||
"fireworks_ai/minimax-m3": {
|
||||
"supports_vision": {
|
||||
"value": true,
|
||||
"checked": "2026-09-26",
|
||||
"source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
|
||||
}
|
||||
}
|
||||
}
|
||||
11
codecov.yaml
11
codecov.yaml
|
|
@ -15,9 +15,10 @@ ignore:
|
|||
flag_management:
|
||||
default_rules:
|
||||
carryforward: true
|
||||
# Dead flags no CI job uploads anymore: their carried-forward sessions were
|
||||
# measured against old revisions, and the stale line maps mark comment lines
|
||||
# of since-edited files as missed, sinking patch coverage on unrelated PRs.
|
||||
# Dead flags no CI job uploads anymore, and flags whose job is path-gated on
|
||||
# PRs: their carried-forward sessions were measured against old revisions, and
|
||||
# the stale line maps mark comment lines of since-edited files as missed,
|
||||
# sinking patch coverage on unrelated PRs.
|
||||
individual_flags:
|
||||
- name: proxy-mgmt-behavior
|
||||
carryforward: false
|
||||
|
|
@ -99,6 +100,10 @@ flag_management:
|
|||
carryforward: false
|
||||
- name: proxy-db-endpoints-and-responses
|
||||
carryforward: false
|
||||
- name: roi-postgres
|
||||
carryforward: false
|
||||
- name: redis-compat
|
||||
carryforward: false
|
||||
|
||||
component_management:
|
||||
individual_components:
|
||||
|
|
|
|||
5
docs/adrs/AGENTS.md
Normal file
5
docs/adrs/AGENTS.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# Architecture decision records for the Python SDK and proxy
|
||||
|
||||
This folder records decisions about the Python SDK and proxy the way [`litellm-rust/docs/adrs/`](../../litellm-rust/docs/adrs/AGENTS.md) records them for the Rust core, and that file's process applies here unchanged: one short `adr_NNN_short_title.md` per decision that is hard to reverse or that a new contributor would reasonably question, with Context, Decision, Alternatives Considered, and Consequences, superseded by a new ADR instead of edited once accepted. Each folder numbers its own ADRs, so a reference to an ADR in the other folder names its path
|
||||
|
||||
Read the ADRs here before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging. A PR that makes a decision of that kind adds its ADR in the same PR
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# ADR 001: Provider usage fields ride as extras on the normalized usage object
|
||||
|
||||
Status: Accepted
|
||||
|
||||
## Context
|
||||
|
||||
Providers report usage fields that no other provider has, such as xAI's `usage.server_side_tool_usage_details.web_search_calls` or Anthropic's `usage.inference_geo`, and cost tracking has to read them or that spend never reaches `LiteLLM_SpendLogs`. One provider response can be served on `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, streaming or not, so a field that only one of those paths carries is billed wrong on the others
|
||||
|
||||
[#30817](https://github.com/BerriAI/litellm/pull/30817) fixed unbilled xAI web searches. One draft did the copy in the xAI Responses transform to keep provider code out of shared files, and that swapped `/v1/responses` usage to the chat shape, so callers of that endpoint lost the Responses schema. The merged version keeps the schema, copies unknown usage fields in the shared Responses-to-chat bridge for every provider, and gates on the reported field without naming the provider
|
||||
|
||||
In October 2026, `Usage` and `ResponseAPIUsage` kept unknown usage keys as pydantic extras, and that bridge copied them. Paths that build usage from named fields kept only the fields they name: the Anthropic, Gemini, and Bedrock Converse transforms, the stream builder, and the bridges from chat usage to `/v1/responses` and `/v1/messages`
|
||||
|
||||
## Decision
|
||||
|
||||
Provider-only usage fields stay on the usage object as pydantic extras, and cost tracking reads them there, in that provider's cost calculator or in a shared gate that checks the field instead of the provider. An extra reaches callers as an added usage key. No transform renames, removes, or reshapes a usage field the endpoint's schema defines to feed cost tracking, though it may fill one LiteLLM's usage object already defines, such as `server_tool_use`
|
||||
|
||||
A shared path that rebuilds usage, such as the stream builder or a bridge between endpoint shapes, copies the extras instead of naming provider fields. A provider transform that builds usage from named fields names the new field. Neither is bypassed with a side channel. Older code that differs, such as the Azure branches in the shared tracker and the Gemini pricing tier read from `_hidden_params`, predates this ADR
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
**Reshape usage in the provider transform**, as a draft of #30817 did. It changes the schema that callers of that endpoint rely on, and every provider and endpoint needs its own copy
|
||||
|
||||
**A typed field on `ResponseAPIUsage` and `Usage` for each provider's quirk**. The types grow without bound and every bridge needs an update for each one. A field several providers report, like `server_tool_use`, does get one
|
||||
|
||||
**A side channel such as `_hidden_params`**. Cost calculators receive `Usage`, so a side channel makes a second source of truth for the same number
|
||||
|
||||
**Provider branches or provider field names in shared code**. They put one provider's logic in a file every provider shares, which is why #30817 replaced its xAI-only gate with one that checks the reported field
|
||||
|
||||
## Consequences
|
||||
|
||||
Benefits:
|
||||
|
||||
- Billing a new provider usage field touches that provider's calculator (plus a `cost_per_token` arm when the calculator is new), its transform when that names fields, and for a built-in tool at most one gate and one `get_cost_for_web_search_request` arm
|
||||
|
||||
Costs:
|
||||
|
||||
- Extras are untyped, so a reader validates them before use, as `_usage_reports_server_side_web_search_calls` does
|
||||
- The stream builder and the bridges from chat usage did not copy extras when this was written, so until they do, a field billed on those paths is named in them, and tests and proof runs cover each endpoint and `"stream": true`
|
||||
- This covers usage fields LiteLLM prices, not request-side params or a provider's own charge for the request, which OpenRouter and opencode Go report and `response_cost_calculator` reads from `_hidden_params` first
|
||||
|
|
@ -15,7 +15,10 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails._content_utils import iter_message_text
|
||||
from litellm.proxy.guardrails._content_utils import (
|
||||
is_non_conversational_request,
|
||||
iter_message_text,
|
||||
)
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_THRESHOLD_VALUE: Final = TypeAdapter(object)
|
||||
|
|
@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
|
||||
async def async_moderation_hook(
|
||||
self,
|
||||
data: dict,
|
||||
data: dict[str, object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
call_type: CallTypesLiteral,
|
||||
):
|
||||
|
|
@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
- Calls Google's Text Moderation API
|
||||
- Rejects request if it fails safety check
|
||||
"""
|
||||
if is_non_conversational_request(call_type, data):
|
||||
verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
|
||||
return
|
||||
# Covers multimodal list content + Responses-API input.
|
||||
text = "".join(iter_message_text(data))
|
||||
if text:
|
||||
|
|
|
|||
|
|
@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
|
|||
|
||||
try:
|
||||
disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
|
||||
verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
|
||||
verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
|
||||
if disabled_callbacks is not None:
|
||||
verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
|
||||
#########################################################
|
||||
# premium user check
|
||||
#########################################################
|
||||
|
|
|
|||
|
|
@ -398,7 +398,7 @@ class CheckBatchCost:
|
|||
deployment. Once that deployment leaves the router, default fallbacks can
|
||||
silently send the retrieve to a provider that never saw the batch, so its
|
||||
404 must not retire a potentially recoverable row."""
|
||||
return self.llm_router.get_deployment(model_id=model_id) is not None
|
||||
return self.llm_router.registry().deployment(model_id) is not None
|
||||
|
||||
@staticmethod
|
||||
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
|
||||
|
|
@ -564,7 +564,7 @@ class CheckBatchCost:
|
|||
def _get_deployment_id_for_bare_model(
|
||||
self, bare_model_name: str, llm_provider: str
|
||||
) -> Optional[str]:
|
||||
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
|
||||
model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
|
||||
deployment_id = (
|
||||
self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
|
||||
)
|
||||
|
|
@ -580,7 +580,7 @@ class CheckBatchCost:
|
|||
) -> Optional[str]:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
for deployment in self.llm_router.get_model_list(model_name=None) or []:
|
||||
for deployment in self.llm_router.registry().deployments() or []:
|
||||
litellm_params = deployment.get("litellm_params") or {}
|
||||
actual_model = litellm_params.get("model")
|
||||
if not isinstance(actual_model, str):
|
||||
|
|
@ -623,8 +623,9 @@ class CheckBatchCost:
|
|||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
|
||||
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
registry = self.llm_router.registry()
|
||||
for deployment_id in registry.names(model_group, kind="ids"):
|
||||
deployment_info = registry.deployment(deployment_id)
|
||||
if deployment_info is None:
|
||||
continue
|
||||
try:
|
||||
|
|
@ -743,7 +744,7 @@ class CheckBatchCost:
|
|||
except (IndexError, AttributeError):
|
||||
pass
|
||||
|
||||
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=cast(str, raw_output_file_id),
|
||||
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
|
||||
|
|
@ -766,7 +767,7 @@ class CheckBatchCost:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
deployment_info = self.llm_router.get_deployment(model_id=model_id)
|
||||
deployment_info = self.llm_router.registry().deployment(model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.info(
|
||||
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
|
||||
|
|
@ -996,7 +997,7 @@ class CheckBatchCost:
|
|||
)
|
||||
|
||||
try:
|
||||
deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
|
||||
deployment_info: Final = self.llm_router.registry().deployment(model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
|
||||
|
|
@ -1009,7 +1010,7 @@ class CheckBatchCost:
|
|||
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
|
||||
)
|
||||
cost_provider: Final = cast(BatchFileProvider, llm_provider)
|
||||
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
|
||||
file_access: Final = {
|
||||
**credentials,
|
||||
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
|
||||
ManagedObjectCreatorAttribution,
|
||||
|
|
@ -273,7 +273,7 @@ class CheckResponsesCost:
|
|||
|
||||
def _resolve_deployment(self, response_id: str) -> Deployment | None:
|
||||
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
|
||||
return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
|
||||
return self.llm_router.registry().deployment(model_id) if model_id is not None else None
|
||||
|
||||
async def _get_response(
|
||||
self,
|
||||
|
|
@ -290,12 +290,14 @@ class CheckResponsesCost:
|
|||
sees provider env vars, so it fails for every deployment whose credentials
|
||||
live in the config; the row then never leaves ``queued``.
|
||||
"""
|
||||
if not via_router:
|
||||
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
|
||||
router_response = await self.llm_router.aget_responses(
|
||||
response_id=response_id, litellm_metadata=litellm_metadata
|
||||
retrieved: Final[object] = (
|
||||
await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
|
||||
if via_router
|
||||
else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
|
||||
)
|
||||
return cast(ResponsesAPIResponse, router_response)
|
||||
if not isinstance(retrieved, ResponsesAPIResponse):
|
||||
raise TypeError(f"Retrieving {response_id} did not return a response object")
|
||||
return retrieved
|
||||
|
||||
async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
|
||||
return await self._get_response(
|
||||
|
|
@ -413,7 +415,7 @@ class CheckResponsesCost:
|
|||
if target.deployment is None or not isinstance(requested, str):
|
||||
return MappingProxyType({})
|
||||
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
|
||||
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
|
||||
self.llm_router.registry().deployments(requested, team_id=team_id) or ()
|
||||
)
|
||||
serving_id: Final = target.deployment.model_info.id
|
||||
return MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
|
|||
if llm_router is None or model_id is None:
|
||||
return None
|
||||
try:
|
||||
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
|
||||
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
except Exception as error:
|
||||
verbose_logger.warning(
|
||||
"Failed to retrieve credentials for provider file "
|
||||
|
|
@ -2650,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
|
||||
}
|
||||
for model_id, model_file_id in specific_model_file_id_mapping.items():
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
delete_data = {
|
||||
**filtered_data,
|
||||
**(
|
||||
|
|
@ -2694,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# against the deployment's configured bucket, which they only
|
||||
# trust from this immutable server-side snapshot, never from
|
||||
# request params.
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
|
||||
credentials = llm_router.registry().credentials(model_id, form="with_provider")
|
||||
if credentials is not None:
|
||||
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -290,7 +290,7 @@ def _check_team_project_limits(
|
|||
|
||||
|
||||
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
|
||||
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
|
||||
return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
|
||||
|
||||
|
||||
def _project_models_expanding_at_request_time(
|
||||
|
|
|
|||
63
examples/bigmodel.md
Normal file
63
examples/bigmodel.md
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# BigModel (Zhipu AI)
|
||||
|
||||
Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: glm
|
||||
litellm_params:
|
||||
model: bigmodel/glm-5.3
|
||||
api_key: os.environ/BIGMODEL_API_KEY
|
||||
api_base: https://open.bigmodel.cn
|
||||
```
|
||||
|
||||
One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
|
||||
|
||||
| Proxy endpoint | BigModel endpoint |
|
||||
| --- | --- |
|
||||
| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
|
||||
| `/v1/responses` | `/api/v1/responses` |
|
||||
| `/v1/messages` | `/api/anthropic/v1/messages` |
|
||||
|
||||
`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
|
||||
|
||||
A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
|
||||
|
||||
The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
chat = litellm.completion(
|
||||
model="bigmodel/glm-5.3",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "enabled"},
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
response = litellm.responses(
|
||||
model="bigmodel/glm-5.3",
|
||||
input="Hello",
|
||||
reasoning={"effort": "low"},
|
||||
store=True,
|
||||
)
|
||||
|
||||
async def messages():
|
||||
return await litellm.anthropic.messages.acreate(
|
||||
model="bigmodel/glm-5.3",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "adaptive"},
|
||||
output_config={"effort": "low"},
|
||||
)
|
||||
```
|
||||
|
||||
Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
|
||||
|
||||
GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
|
||||
|
||||
Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
|
||||
|
||||
The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
|
||||
|
||||
API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27
|
||||
|
|
@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0
|
|||
COPY pyproject.toml README.md rust-toolchain.toml ./
|
||||
COPY .cargo ./.cargo
|
||||
COPY litellm-rust ./litellm-rust
|
||||
COPY model_prices_and_context_window.json ./
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
mkdir -p litellm/rust_bridge /wheels && \
|
||||
|
|
@ -144,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
|||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
|
|||
wins under token auth too, so exporting it here is enough for every worker to
|
||||
pick the pooled URL up unchanged.
|
||||
|
||||
LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
|
||||
before it can answer its first 5 s health ping, so a slow host would kill each one.
|
||||
Workers then load the gateway app through ``worker_app`` without garbage collection
|
||||
pauses, which hold the GIL long enough to miss a ping.
|
||||
|
||||
Run with:
|
||||
python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
|
||||
"""
|
||||
|
|
@ -15,19 +20,19 @@ Run with:
|
|||
import os
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
export_pooled_database_url,
|
||||
start_in_container_pgbouncer,
|
||||
)
|
||||
from litellm_cli.main import without_gc_pauses
|
||||
|
||||
GATEWAY_APP: Final = "gateway.main:app"
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
|
||||
|
||||
GATEWAY_APP: Final = "gateway.launch:worker_app"
|
||||
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
|
||||
|
||||
|
||||
|
|
@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
|
|||
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
|
||||
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
|
||||
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
|
||||
return (GATEWAY_APP, *argv)
|
||||
return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
|
||||
return (GATEWAY_APP, "--factory", *argv)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
|
||||
|
||||
|
||||
def pool_database_url(
|
||||
settings: DatabaseURLSettings,
|
||||
pgbouncer: PgBouncerSettings,
|
||||
settings: "DatabaseURLSettings",
|
||||
pgbouncer: "PgBouncerSettings",
|
||||
environ: Mapping[str, str],
|
||||
) -> str | PgBouncerError | None:
|
||||
) -> "str | PgBouncerError | None":
|
||||
"""Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
|
||||
|
||||
The upstream URL is whatever ``apply_to_env`` assembled from the discrete
|
||||
``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
|
||||
auth the pooler mints and renews the upstream token itself.
|
||||
"""
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
|
||||
|
||||
if not pgbouncer.enabled:
|
||||
return None
|
||||
upstream_url: Final = environ.get("DATABASE_URL")
|
||||
|
|
@ -58,11 +65,20 @@ def pool_database_url(
|
|||
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
|
||||
|
||||
|
||||
def worker_app() -> "FastAPI":
|
||||
with without_gc_pauses():
|
||||
from gateway.main import app
|
||||
return app
|
||||
|
||||
|
||||
def _serve(argv: Sequence[str]) -> None:
|
||||
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
|
||||
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
settings.apply_to_env()
|
||||
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
|
||||
|
|
|
|||
|
|
@ -259,3 +259,9 @@ itself.
|
|||
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
|
||||
|
||||
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
|
||||
|
||||
## Shutdown accounting
|
||||
|
||||
`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
|
||||
|
||||
Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`
|
||||
|
|
|
|||
|
|
@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it.
|
|||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
|
@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME
|
|||
DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
|
||||
REFUSAL: Final = "Existing rows would make a pending schema change fail:"
|
||||
|
||||
Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
|
||||
Kind = Literal["too_long"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -54,9 +52,6 @@ class Check:
|
|||
statement: str
|
||||
source: str
|
||||
index: str | None = None
|
||||
referenced_table: str | None = None
|
||||
referenced_columns: tuple[str, ...] = ()
|
||||
expression: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -100,33 +95,6 @@ class ColumnType:
|
|||
|
||||
ColumnTypeLookup = Callable[[str, str], ColumnType | None]
|
||||
|
||||
_IDENT: Final = r'"?(\w+)"?'
|
||||
_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
|
||||
_CREATE_INDEX: Final = re.compile(
|
||||
rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
|
||||
rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ALTER_TABLE: Final = re.compile(
|
||||
rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
|
||||
_ADD_COLUMN: Final = re.compile(
|
||||
rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
|
||||
_ADD_FOREIGN_KEY: Final = re.compile(
|
||||
rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
|
||||
_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
|
||||
_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
|
||||
_COMMENT: Final = re.compile(r"--[^\n]*")
|
||||
_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
|
||||
_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def migration_data_check_timeout_seconds() -> float:
|
||||
"""Seconds one count query may run before the preflight gives up on verifying it."""
|
||||
|
|
@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float:
|
|||
return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def statements(migration_sql: str) -> tuple[str, ...]:
|
||||
"""The statements of one migration file, comments stripped, empty ones dropped."""
|
||||
parts: Final = _COMMENT.sub("", migration_sql).split(";")
|
||||
return tuple(part.strip() for part in parts if part.strip())
|
||||
|
||||
|
||||
def _split_top_level(text: str) -> tuple[str, ...]:
|
||||
"""Split on the commas outside parentheses."""
|
||||
depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
|
||||
cuts: Final = tuple(
|
||||
position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
|
||||
)
|
||||
starts: Final = (0, *(cut + 1 for cut in cuts))
|
||||
stops: Final = (*cuts, len(text))
|
||||
pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
|
||||
return tuple(piece for piece in pieces if piece)
|
||||
|
||||
|
||||
def _plain_columns(column_list: str) -> tuple[str, ...] | None:
|
||||
"""The column names of an index or constraint column list, None when one entry is an
|
||||
expression the derivation cannot turn into a count."""
|
||||
matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
|
||||
if any(match is None for match in matches):
|
||||
return None
|
||||
return tuple(match.group(1) for match in matches if match is not None)
|
||||
|
||||
|
||||
def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
|
||||
unique, name, table, method, column_list = match.groups()
|
||||
columns: Final = _plain_columns(column_list)
|
||||
if columns is None:
|
||||
return
|
||||
label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
|
||||
if method is None or method.lower() == "btree":
|
||||
yield Check("too_long", table, columns, label, source, index=name)
|
||||
if unique:
|
||||
yield Check("duplicates", table, columns, label, source, index=name)
|
||||
|
||||
|
||||
def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
|
||||
if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
|
||||
yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
|
||||
return
|
||||
if (add_column := _ADD_COLUMN.match(action)) is not None:
|
||||
column, body = add_column.groups()
|
||||
if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
|
||||
yield Check(
|
||||
"any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
|
||||
)
|
||||
return
|
||||
if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
|
||||
constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
|
||||
own: Final = _plain_columns(columns)
|
||||
theirs: Final = _plain_columns(referenced_columns)
|
||||
if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
|
||||
return
|
||||
yield Check(
|
||||
"orphans",
|
||||
table,
|
||||
own,
|
||||
_constraint_label(table, constraint, "FOREIGN KEY"),
|
||||
source,
|
||||
referenced_table=referenced_table,
|
||||
referenced_columns=theirs,
|
||||
)
|
||||
return
|
||||
if (unique := _ADD_UNIQUE.match(action)) is not None:
|
||||
constraint, columns = unique.groups()
|
||||
unique_columns: Final = _plain_columns(columns)
|
||||
if unique_columns is not None:
|
||||
yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
|
||||
return
|
||||
if (check := _ADD_CHECK.match(action)) is not None:
|
||||
constraint, expression, not_valid = check.groups()
|
||||
if not_valid is None:
|
||||
yield Check(
|
||||
"check_violations",
|
||||
table,
|
||||
(),
|
||||
_constraint_label(table, constraint, "CHECK"),
|
||||
source,
|
||||
expression=expression,
|
||||
)
|
||||
|
||||
|
||||
def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
|
||||
named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
|
||||
return f'ALTER TABLE "{table}" ADD {named}{kind}'
|
||||
|
||||
|
||||
def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
|
||||
"""The count queries one SQL statement needs before it can be applied, empty for a
|
||||
statement that cannot fail on existing rows or that the derivation cannot read."""
|
||||
if (index := _CREATE_INDEX.match(statement_text)) is not None:
|
||||
return tuple(_index_checks(index, statement_text, source))
|
||||
if (alter := _ALTER_TABLE.match(statement_text)) is not None:
|
||||
table, actions = alter.groups()
|
||||
return tuple(
|
||||
check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
|
||||
) # comprehension-ok: flattening the per-action checks of one statement
|
||||
return ()
|
||||
|
||||
|
||||
def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
|
||||
return tuple(
|
||||
check
|
||||
for statement_text in statements(migration_sql)
|
||||
for check in checks_for_statement(statement_text, f"migration {name}")
|
||||
) # comprehension-ok: flattening the per-statement checks of one migration
|
||||
|
||||
|
||||
def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
|
||||
"""The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
|
||||
return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
|
||||
|
|
@ -424,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...],
|
|||
|
||||
|
||||
def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
|
||||
from psycopg import sql
|
||||
|
||||
table: Final = sql.Identifier(schema, check.table)
|
||||
columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return _entry_size_query(check, schema, types, limit_bytes)
|
||||
case "duplicates":
|
||||
not_null: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
|
||||
)
|
||||
return sql.SQL(
|
||||
"SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
|
||||
).format(t=table, nn=not_null, cols=columns)
|
||||
case "nulls":
|
||||
return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
|
||||
t=table, col=sql.Identifier(check.columns[0])
|
||||
)
|
||||
case "any_row":
|
||||
return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
|
||||
case "orphans":
|
||||
pairs: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
|
||||
for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
|
||||
)
|
||||
present: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
|
||||
)
|
||||
return sql.SQL(
|
||||
"SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
|
||||
).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
|
||||
case "check_violations":
|
||||
return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
|
||||
t=table,
|
||||
expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file
|
||||
)
|
||||
return _entry_size_query(check, schema, types, limit_bytes)
|
||||
|
||||
|
||||
def _applies(
|
||||
|
|
@ -474,15 +296,7 @@ def _applies(
|
|||
return False
|
||||
if check.index is not None and _index_is_valid(connection, schema, check.index):
|
||||
return False
|
||||
if check.kind == "any_row":
|
||||
return types is None
|
||||
if types is None:
|
||||
return False
|
||||
if check.kind == "too_long":
|
||||
return any(column.can_overflow_btree for column in types)
|
||||
if check.kind == "orphans":
|
||||
return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
|
||||
return True
|
||||
return types is not None and any(column.can_overflow_btree for column in types)
|
||||
|
||||
|
||||
def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
|
||||
|
|
@ -515,21 +329,7 @@ def _run_check(
|
|||
rows: Final = _int(values[0], 0) if values else 0
|
||||
if rows == 0:
|
||||
return None
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
|
||||
case "duplicates":
|
||||
return BlockedChange(check, rows, "duplicated value groups")
|
||||
case "nulls":
|
||||
return BlockedChange(check, rows, "NULL")
|
||||
case "any_row":
|
||||
return BlockedChange(
|
||||
check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
|
||||
)
|
||||
case "orphans":
|
||||
return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
|
||||
case "check_violations":
|
||||
return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
|
||||
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
|
||||
|
||||
|
||||
def find_blockers(
|
||||
|
|
@ -571,19 +371,7 @@ def find_blockers(
|
|||
|
||||
def _remedy(check: Check) -> str:
|
||||
joined: Final = ", ".join(check.columns)
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
|
||||
case "duplicates":
|
||||
return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
|
||||
case "nulls":
|
||||
return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
|
||||
case "any_row":
|
||||
return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
|
||||
case "orphans":
|
||||
return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
|
||||
case "check_violations":
|
||||
return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
|
||||
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
|
||||
|
||||
|
||||
def _describe(finding: Finding) -> str:
|
||||
|
|
@ -607,30 +395,4 @@ def _describe(finding: Finding) -> str:
|
|||
def report(findings: tuple[Finding, ...]) -> str:
|
||||
"""The operator-facing refusal: one block per finding, then what to do next."""
|
||||
lines: Final = "\n".join(_describe(finding) for finding in findings)
|
||||
return (
|
||||
f"{REFUSAL}\n{lines}\n"
|
||||
"No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
|
||||
)
|
||||
|
||||
|
||||
def pending_change_checks(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]",
|
||||
schema: str,
|
||||
migrations_dir: Path | None,
|
||||
indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
|
||||
) -> tuple[Check, ...]:
|
||||
"""Every check the pending migrations and the request-log indexes need. `migrations_dir`
|
||||
is None when the schema is pushed rather than migrated, so only the indexes are checked."""
|
||||
migration_checks: Final = (
|
||||
tuple(
|
||||
check
|
||||
for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
|
||||
for check in checks_for_migration(name, migration_sql)
|
||||
) # comprehension-ok: flattening the per-migration checks
|
||||
if migrations_dir is not None
|
||||
else ()
|
||||
)
|
||||
index_checks: Final = tuple(
|
||||
check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
|
||||
) # comprehension-ok: flattening the per-index checks
|
||||
return migration_checks + index_checks
|
||||
return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up."
|
||||
|
|
|
|||
184
litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py
Normal file
184
litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each
|
||||
inside its own transaction that is always rolled back, so Postgres itself says whether this
|
||||
database's rows and schema take them. Each migration holds the locks the real upgrade will,
|
||||
and releases them before the next one runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm_proxy_extras.migration_data_checks import pending_migrations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psycopg
|
||||
|
||||
_CONCURRENTLY: Final = re.compile(
|
||||
r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE
|
||||
)
|
||||
UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"})
|
||||
|
||||
Migration = tuple[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Applied:
|
||||
name: str
|
||||
seconds: float
|
||||
replayed: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rejected:
|
||||
name: str
|
||||
error: str
|
||||
seconds: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Baseline:
|
||||
"""A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying
|
||||
history, so the dry run asks the same question the boot does. `refusal` is None when the boot
|
||||
will baseline, otherwise its reason (a custom migration folder, or the schema diff)."""
|
||||
|
||||
refusal: str | None
|
||||
|
||||
@property
|
||||
def matches(self) -> bool:
|
||||
return self.refusal is None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DryRun:
|
||||
applied: tuple[Applied, ...]
|
||||
rejected: Rejected | None
|
||||
baseline: Baseline | None = None
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.rejected is None and (self.baseline is None or self.baseline.matches)
|
||||
|
||||
|
||||
def transactional(migration_sql: str) -> str:
|
||||
"""The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones."""
|
||||
return _CONCURRENTLY.sub(r"\1", migration_sql)
|
||||
|
||||
|
||||
def _attempt(
|
||||
connection: psycopg.Connection[tuple[object, ...]],
|
||||
schema: str,
|
||||
chain: tuple[Migration, ...],
|
||||
clock: Callable[[], float],
|
||||
) -> tuple[float, psycopg.Error | None]:
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
started: Final = clock()
|
||||
try:
|
||||
with connection.transaction(force_rollback=True):
|
||||
connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema)))
|
||||
for _, migration_sql in chain:
|
||||
connection.execute(transactional(migration_sql))
|
||||
except psycopg.Error as exc:
|
||||
return clock() - started, exc
|
||||
return clock() - started, None
|
||||
|
||||
|
||||
def _run(
|
||||
connection: psycopg.Connection[tuple[object, ...]],
|
||||
schema: str,
|
||||
pending: tuple[Migration, ...],
|
||||
position: int,
|
||||
done: DryRun,
|
||||
clock: Callable[[], float],
|
||||
) -> DryRun:
|
||||
"""One migration in its own rolled-back transaction. A migration that fails because something
|
||||
an earlier pending migration creates does not exist yet is rerun with those replayed in front."""
|
||||
if done.rejected is not None:
|
||||
return done
|
||||
name: Final = pending[position][0]
|
||||
seconds, error = _attempt(connection, schema, (pending[position],), clock)
|
||||
if error is None:
|
||||
return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline)
|
||||
if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0:
|
||||
return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline)
|
||||
earlier: Final = pending[:position]
|
||||
chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock)
|
||||
if chained_error is None:
|
||||
replayed: Final = tuple(earlier_name for earlier_name, _ in earlier)
|
||||
return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline)
|
||||
return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline)
|
||||
|
||||
|
||||
def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
|
||||
from psycopg import sql
|
||||
|
||||
row: Final = connection.execute(
|
||||
"SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),)
|
||||
).fetchone()
|
||||
return row is not None and row[0] is not None
|
||||
|
||||
|
||||
def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
|
||||
row: Final = connection.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1",
|
||||
(schema,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def dry_run(
|
||||
connection: psycopg.Connection[tuple[object, ...]],
|
||||
schema: str,
|
||||
migrations_dir: Path,
|
||||
baseline_check: Callable[[], str | None] | None = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> DryRun:
|
||||
"""Apply every pending migration in order, each in its own transaction that is rolled back
|
||||
before the next one starts. The first migration Postgres rejects ends the run with its error.
|
||||
`baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for
|
||||
the replay on a database that has tables but no migration history, which the boot baselines."""
|
||||
if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema):
|
||||
return DryRun((), None, Baseline(baseline_check()))
|
||||
pending: Final = pending_migrations(connection, schema, migrations_dir)
|
||||
return reduce(
|
||||
lambda done, position: _run(connection, schema, pending, position, done, clock),
|
||||
range(len(pending)),
|
||||
DryRun((), None),
|
||||
)
|
||||
|
||||
|
||||
def _timing(applied: Applied) -> str:
|
||||
replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else ""
|
||||
return f"\n {applied.name}: {applied.seconds:.1f} s{replay}"
|
||||
|
||||
|
||||
def report(result: DryRun) -> str:
|
||||
"""What the operator reads: the per-migration time the real upgrade will hold its locks,
|
||||
or the migration Postgres rejected with Postgres's own words. Nothing was applied either way."""
|
||||
if result.baseline is not None:
|
||||
if result.baseline.matches:
|
||||
return (
|
||||
"No migration history and the schema matches this build, "
|
||||
"the proxy baselines it at boot, nothing to dry-run."
|
||||
)
|
||||
return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}"
|
||||
timings: Final = "".join(_timing(applied) for applied in result.applied)
|
||||
if result.rejected is None:
|
||||
count: Final = len(result.applied)
|
||||
if count == 0:
|
||||
return "No pending migrations, this database is on this version's schema."
|
||||
return (
|
||||
f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next "
|
||||
f"so its locks are released, nothing changed:{timings}"
|
||||
)
|
||||
return (
|
||||
f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, "
|
||||
f"nothing was applied.\n{result.rejected.error}"
|
||||
+ (f"\nApplied before it in the dry run:{timings}" if timings else "")
|
||||
)
|
||||
|
|
@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
|
|||
)
|
||||
|
||||
|
||||
def has_packaged_history(migrations_dir: Path) -> bool:
|
||||
"""Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships,
|
||||
the only history the boot baselines an existing database onto."""
|
||||
packaged_dir: Final = Path(__file__).parent
|
||||
migrations: Final = migration_files(migrations_dir)
|
||||
return (
|
||||
bool(migrations)
|
||||
and migrations == migration_files(packaged_dir)
|
||||
and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes()
|
||||
)
|
||||
|
||||
|
||||
def baseline_current_schema(
|
||||
coordinator: MigrationCoordinator,
|
||||
schema: str,
|
||||
|
|
@ -95,14 +107,9 @@ def baseline_current_schema(
|
|||
) -> None:
|
||||
from psycopg import sql
|
||||
|
||||
packaged_dir: Final = Path(__file__).parent
|
||||
migrations: Final = migration_files(migrations_dir)
|
||||
if (
|
||||
not migrations
|
||||
or migrations != migration_files(packaged_dir)
|
||||
or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
|
||||
):
|
||||
if not has_packaged_history(migrations_dir):
|
||||
raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
|
||||
migrations: Final = migration_files(migrations_dir)
|
||||
|
||||
coordinator.acquire_prisma_lock()
|
||||
existing: Final = coordinator.connection.execute(
|
||||
|
|
|
|||
|
|
@ -11,18 +11,13 @@ from collections.abc import Callable
|
|||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from litellm_proxy_extras import prisma_toolchain
|
||||
from litellm_proxy_extras._logging import logger
|
||||
from litellm_proxy_extras._redaction import _redact_credentials
|
||||
from litellm_proxy_extras.migration_data_checks import (
|
||||
Finding,
|
||||
find_blockers,
|
||||
migration_data_check_timeout_seconds,
|
||||
pending_change_checks,
|
||||
report,
|
||||
)
|
||||
from litellm_proxy_extras.migration_dry_run import dry_run
|
||||
from litellm_proxy_extras.migration_dry_run import report as dry_run_report
|
||||
from litellm_proxy_extras.migration_lock import held_migration_lock
|
||||
from litellm_proxy_extras.prisma_toolchain import (
|
||||
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
|
||||
|
|
@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import (
|
|||
apply_replica_identity_full,
|
||||
)
|
||||
from litellm_proxy_extras.request_log_indexes import (
|
||||
REQUEST_LOG_INDEXES,
|
||||
ensure_request_log_indexes,
|
||||
filter_request_log_index_diff,
|
||||
)
|
||||
|
|
@ -52,7 +46,7 @@ class LensCheckConnect(Protocol):
|
|||
) -> "psycopg.Connection[tuple[object, ...]]": ...
|
||||
|
||||
|
||||
def str_to_bool(value: Optional[str]) -> bool:
|
||||
def str_to_bool(value: str | None) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
return value.lower() in ("true", "1", "t", "y", "yes")
|
||||
|
|
@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str:
|
|||
).strip()
|
||||
|
||||
|
||||
def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
|
||||
def _without_spend_logs_pk_clauses(statement: str) -> str | None:
|
||||
prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
|
||||
if not prefix_match:
|
||||
return statement
|
||||
|
|
@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int:
|
|||
return max(_migration_timestamp(n) for n in names)
|
||||
|
||||
|
||||
def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
|
||||
def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]:
|
||||
if isinstance(command, tuple):
|
||||
return tuple(_redact_credentials(str(argument)) for argument in command)
|
||||
if isinstance(command, list):
|
||||
|
|
@ -404,7 +398,7 @@ class ProxyExtrasDBManager:
|
|||
return tuple(row) if row is not None else ()
|
||||
|
||||
@staticmethod
|
||||
def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
|
||||
def _failed_migration_logs(migration_name: str, started_at: str) -> str | None:
|
||||
row: Final = ProxyExtrasDBManager._read_migration_ledger(
|
||||
"SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
|
||||
"AND finished_at IS NULL AND rolled_back_at IS NULL",
|
||||
|
|
@ -721,10 +715,10 @@ class ProxyExtrasDBManager:
|
|||
return row is not None
|
||||
|
||||
@staticmethod
|
||||
def _prisma_schema_param(url: str) -> Optional[str]:
|
||||
def _prisma_schema_param(url: str) -> str | None:
|
||||
"""The `schema` query param Prisma uses to pick its target schema,
|
||||
or None when the URL does not set one."""
|
||||
from urllib.parse import urlparse, parse_qsl
|
||||
from urllib.parse import parse_qsl, urlparse
|
||||
|
||||
return next(
|
||||
(v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
|
||||
|
|
@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager:
|
|||
bool: True if setup was successful, False otherwise
|
||||
"""
|
||||
ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
|
||||
ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
|
||||
migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
|
||||
if not migrated:
|
||||
return False
|
||||
|
|
@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager:
|
|||
return True
|
||||
|
||||
@staticmethod
|
||||
def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
|
||||
"""What in the existing rows would make a pending migration or request-log index
|
||||
fail, read with count queries only: no write, no table lock. Empty when nothing
|
||||
blocks, when no database is configured, or when it cannot be reached, so the
|
||||
migration step that follows reports the connection problem the way it does today."""
|
||||
def _baseline_refusal(prisma_dir: Path) -> str | None:
|
||||
"""Why `baseline_current_schema` would refuse this database at boot, None when it would baseline:
|
||||
the migrations folder must be the packaged one, then `prisma migrate diff` must find the live
|
||||
database and this build's schema.prisma identical."""
|
||||
from litellm_proxy_extras.migration_lock import migration_environment
|
||||
from litellm_proxy_extras.migration_recovery import has_packaged_history
|
||||
|
||||
if not has_packaged_history(prisma_dir):
|
||||
return (
|
||||
f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the "
|
||||
"migrations this build ships, and the proxy only baselines onto the packaged history."
|
||||
)
|
||||
try:
|
||||
prisma_toolchain.run_prisma(
|
||||
(
|
||||
_get_prisma_command(),
|
||||
"migrate",
|
||||
"diff",
|
||||
"--from-schema-datasource",
|
||||
str(prisma_dir / "schema.prisma"),
|
||||
"--to-schema-datamodel",
|
||||
str(prisma_dir / "schema.prisma"),
|
||||
"--exit-code",
|
||||
),
|
||||
timeout=prisma_toolchain.prisma_command_timeout(),
|
||||
env=migration_environment(_get_prisma_env()),
|
||||
)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
diff: Final = str(exc.stdout or exc.stderr or exc).strip()
|
||||
return f"the schema differs from this build. prisma migrate diff:\n{diff}"
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return f"prisma migrate diff did not finish within {exc.timeout} s"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
|
||||
"""`--check_migrations_against_existing_data`: the pending migrations applied for real on
|
||||
DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports
|
||||
what the upgrade would hit. (True, the per-migration timings) when they all apply,
|
||||
(False, Postgres's own error) when one does not."""
|
||||
import psycopg
|
||||
|
||||
if not use_migrate:
|
||||
return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.")
|
||||
database_url: Final = os.environ.get("DATABASE_URL")
|
||||
if not database_url:
|
||||
return ()
|
||||
return (False, "DATABASE_URL is not set.")
|
||||
direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
|
||||
os.environ.get("DIRECT_URL") or database_url
|
||||
)
|
||||
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
|
||||
migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
|
||||
indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
|
||||
prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir())
|
||||
try:
|
||||
with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
|
||||
checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
|
||||
return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
|
||||
result: Final = dry_run(
|
||||
connection,
|
||||
schema,
|
||||
prisma_dir / "migrations",
|
||||
baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir),
|
||||
)
|
||||
except psycopg.OperationalError as exc:
|
||||
logger.warning(
|
||||
"Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
|
||||
)
|
||||
return ()
|
||||
|
||||
@staticmethod
|
||||
def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
|
||||
"""The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
|
||||
findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
|
||||
return report(findings) if findings else None
|
||||
|
||||
@staticmethod
|
||||
def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
|
||||
"""Refuse to run migrations that the existing rows would make fail, before any is
|
||||
applied. The RuntimeError carries the report and reaches the operator through the
|
||||
proxy's "Database migration cannot proceed" exit."""
|
||||
refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
|
||||
if refusal is not None:
|
||||
raise RuntimeError(refusal)
|
||||
return (False, f"Could not reach the database: {_redact_credentials(str(exc))}")
|
||||
return (result.ok, dry_run_report(result))
|
||||
|
||||
@staticmethod
|
||||
def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
|
||||
|
|
@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager:
|
|||
time.sleep(random.randrange(5, 15))
|
||||
finally:
|
||||
os.chdir(original_dir)
|
||||
pass
|
||||
return False
|
||||
|
|
|
|||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -4345,6 +4345,7 @@ dependencies = [
|
|||
"litellm-inference-transcription",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-model-catalog",
|
||||
"litellm-router-types",
|
||||
"litellm-secrets",
|
||||
"litellm-secrets-aws",
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[
|
|||
|mapping| {
|
||||
mapping.error_str.contains("invalid_request_error")
|
||||
&& !mapping.error_str.contains("Incorrect API key provided")
|
||||
&& !matches!(mapping.status, Some(401 | 404))
|
||||
},
|
||||
PublicError::BadRequest,
|
||||
),
|
||||
|
|
@ -132,6 +133,23 @@ mod tests {
|
|||
assert_eq!(classified(Some(400), text), Some(expected));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::unauthorized(Some(401), None)]
|
||||
#[case::not_found(Some(404), None)]
|
||||
#[case::forbidden(Some(403), Some(PublicError::BadRequest))]
|
||||
#[case::rate_limited(Some(429), Some(PublicError::BadRequest))]
|
||||
#[case::server_error(Some(500), Some(PublicError::BadRequest))]
|
||||
#[case::no_status(None, Some(PublicError::BadRequest))]
|
||||
fn invalid_request_errors_leave_401_and_404_to_the_status_mapping(
|
||||
#[case] status: Option<u16>,
|
||||
#[case] expected: Option<PublicError>,
|
||||
) {
|
||||
assert_eq!(
|
||||
classified(status, "invalid_request_error bad field"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::rate_limit_before_context_window(
|
||||
"rate limit and This model's maximum context length is 10",
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use rstest::rstest;
|
|||
use super::*;
|
||||
|
||||
const MODEL: &str = "azure_ai/doc-intelligence/prebuilt-read";
|
||||
const LAYOUT_MODEL: &str = "azure_ai/doc-intelligence/prebuilt-layout";
|
||||
|
||||
fn read_request(base: &str, options: Value) -> LiteLLMOcrRequest {
|
||||
ocr_request(MODEL, base, options)
|
||||
|
|
@ -70,6 +71,7 @@ async fn pages_features_and_extra_options_map_to_the_analyze_call() {
|
|||
#[case(json!({"pages": "1&&features=bad"}), Error::Pages(PagesFault::Range))]
|
||||
#[case(json!({"features": "languages&pages=1"}), Error::Features)]
|
||||
#[case(json!({"req_format": "azure"}), Error::RequestFormat)]
|
||||
#[case(json!({"outputContentFormat": "html"}), Error::OutputContentFormat)]
|
||||
#[tokio::test]
|
||||
async fn invalid_pages_features_and_format_are_rejected_before_sending(
|
||||
#[case] options: Value,
|
||||
|
|
@ -130,6 +132,137 @@ async fn an_inline_document_is_sent_as_base64_and_only_page_text_is_kept(#[case]
|
|||
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::lower_case("markdown")]
|
||||
#[case::capitalized("Markdown")]
|
||||
#[tokio::test]
|
||||
async fn markdown_output_is_requested_on_the_analyze_call_and_split_into_pages(
|
||||
#[case] output_content_format: &str,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({
|
||||
"status": "succeeded",
|
||||
"analyzeResult": {
|
||||
"content": "# Tïtle\n\n<!-- PageBreak -->\n\n| a |",
|
||||
"contentFormat": "markdown",
|
||||
"stringIndexType": "unicodeCodePoint",
|
||||
"pages": [
|
||||
{"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]},
|
||||
{"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]}
|
||||
]
|
||||
}
|
||||
}))])
|
||||
.await;
|
||||
|
||||
let response = perform(ocr_request(
|
||||
LAYOUT_MODEL,
|
||||
&upstream.uri(),
|
||||
json!({"outputContentFormat": output_content_format, "pages": [0, 1]}),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(
|
||||
sent.query("outputContentFormat").as_deref(),
|
||||
Some("markdown")
|
||||
);
|
||||
assert_eq!(
|
||||
sent.query("stringIndexType").as_deref(),
|
||||
Some("unicodeCodePoint")
|
||||
);
|
||||
assert_eq!(sent.query("pages").as_deref(), Some("1,2"));
|
||||
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
|
||||
assert_eq!(
|
||||
response.content.as_deref(),
|
||||
Some("# Tïtle\n\n<!-- PageBreak -->\n\n| a |")
|
||||
);
|
||||
let pages = response
|
||||
.pages
|
||||
.iter()
|
||||
.map(|page| page.markdown.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(pages, ["# Tïtle\n\n<!-- PageBreak -->\n\n", "| a |"]);
|
||||
assert_eq!(
|
||||
response.extra_fields.get("stringIndexType"),
|
||||
Some(&json!("unicodeCodePoint"))
|
||||
);
|
||||
assert_eq!(
|
||||
response.usage_info.as_ref().unwrap().pages_processed,
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))]
|
||||
#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))]
|
||||
#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))]
|
||||
#[case::omitted(LAYOUT_MODEL, json!({}), None)]
|
||||
#[tokio::test]
|
||||
async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets(
|
||||
#[case] model: &str,
|
||||
#[case] options: Value,
|
||||
#[case] output_content_format: Option<&str>,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({
|
||||
"status": "succeeded",
|
||||
"analyzeResult": {
|
||||
"contentFormat": "text",
|
||||
"stringIndexType": "textElements",
|
||||
"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]
|
||||
}
|
||||
}))])
|
||||
.await;
|
||||
|
||||
let response = perform(ocr_request(model, &upstream.uri(), options))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(
|
||||
sent.query("outputContentFormat").as_deref(),
|
||||
output_content_format
|
||||
);
|
||||
assert_eq!(sent.query("stringIndexType"), None);
|
||||
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
|
||||
assert_eq!(response.pages[0].markdown, "hello");
|
||||
assert_eq!(response.extra_fields.get("stringIndexType"), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))]
|
||||
#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))]
|
||||
#[case::empty(json!({"outputContentFormat": ""}), None)]
|
||||
#[case::empty_camel_case_falls_back_to_snake_case(
|
||||
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
|
||||
Some("markdown")
|
||||
)]
|
||||
#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)]
|
||||
#[tokio::test]
|
||||
async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling(
|
||||
#[case] options: Value,
|
||||
#[case] output_content_format: Option<&str>,
|
||||
) {
|
||||
let upstream = upstream([json_response(json!({
|
||||
"status": "succeeded",
|
||||
"analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]}
|
||||
}))])
|
||||
.await;
|
||||
|
||||
let response = perform(ocr_request(MODEL, &upstream.uri(), options))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sent = only_request(&upstream).await;
|
||||
assert_eq!(
|
||||
sent.query("outputContentFormat").as_deref(),
|
||||
output_content_format
|
||||
);
|
||||
assert_eq!(sent.query("output_content_format"), None);
|
||||
assert_eq!(sent.query("drop_params"), None);
|
||||
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
|
||||
assert_eq!(response.pages[0].markdown, "hello");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_format_normalizes_pages_and_keeps_the_provider_response() {
|
||||
let operation = json!({
|
||||
|
|
|
|||
|
|
@ -1,14 +1,31 @@
|
|||
use serde_json::Value;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionResponseData {
|
||||
pub text: String,
|
||||
pub usage: Option<AudioTranscriptionUsage>,
|
||||
}
|
||||
|
||||
impl AudioTranscriptionResponseData {
|
||||
pub fn into_json(self) -> Value {
|
||||
serde_json::json!({
|
||||
"text": self.text,
|
||||
})
|
||||
match self.usage {
|
||||
None => serde_json::json!({"text": self.text}),
|
||||
Some(usage) => serde_json::json!({
|
||||
"text": self.text,
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": usage.input_tokens,
|
||||
"output_tokens": usage.output_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
use std::{collections::BTreeSet, time::Duration};
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_auth::{InputSource, Sourced};
|
||||
|
|
@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
|
|||
use reqwest::Url;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use serde_with::serde_as;
|
||||
use serde_with::{DefaultOnError, serde_as};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::base_llm::ocr::{
|
||||
|
|
@ -17,9 +20,10 @@ use crate::base_llm::ocr::{
|
|||
handler::{CallHooks, OcrClient, read_json_response},
|
||||
settings::OcrSettings,
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS,
|
||||
OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest,
|
||||
ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
|
||||
BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES,
|
||||
OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext,
|
||||
PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
|
||||
drops_unsupported_params, response_format,
|
||||
},
|
||||
};
|
||||
use litellm_llms_types::formats::ocr::{
|
||||
|
|
@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
|
|||
|
||||
const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";
|
||||
const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT";
|
||||
const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat";
|
||||
const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format";
|
||||
const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint";
|
||||
const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType";
|
||||
const MARKDOWN_MODEL_ID: &str = "prebuilt-layout";
|
||||
|
||||
#[derive(
|
||||
Clone,
|
||||
Copy,
|
||||
Debug,
|
||||
PartialEq,
|
||||
strum::EnumString,
|
||||
strum::IntoStaticStr,
|
||||
strum::Display,
|
||||
strum::VariantArray,
|
||||
serde_with::SerializeDisplay,
|
||||
)]
|
||||
#[strum(ascii_case_insensitive)]
|
||||
pub enum OutputContentFormat {
|
||||
#[strum(serialize = "text")]
|
||||
Text,
|
||||
#[strum(serialize = "markdown")]
|
||||
Markdown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct DocumentIntelligenceParams {
|
||||
|
|
@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams {
|
|||
pub pages: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub features: Option<String>,
|
||||
#[serde(
|
||||
rename = "outputContentFormat",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub output_content_format: Option<OutputContentFormat>,
|
||||
#[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")]
|
||||
pub string_index_type: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
|
|
@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation {
|
|||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
struct AzureDocumentIntelligenceAnalyzeResult {
|
||||
pub content: Option<String>,
|
||||
#[serde(rename = "contentFormat")]
|
||||
pub content_format: Option<String>,
|
||||
#[serde(rename = "stringIndexType")]
|
||||
pub string_index_type: Option<String>,
|
||||
#[serde(default)]
|
||||
pub pages: Vec<AzureDocumentIntelligencePage>,
|
||||
pub tables: Option<Vec<Map<String, Value>>>,
|
||||
|
|
@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage {
|
|||
pub unit: Option<String>,
|
||||
#[serde(default)]
|
||||
pub lines: Vec<AzureDocumentIntelligenceLine>,
|
||||
#[serde(default)]
|
||||
#[serde_as(deserialize_as = "DefaultOnError")]
|
||||
pub spans: Option<Vec<AzureDocumentIntelligenceSpan>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
|
|
@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine {
|
|||
pub content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
struct AzureDocumentIntelligenceSpan {
|
||||
pub offset: usize,
|
||||
pub length: usize,
|
||||
}
|
||||
|
||||
impl AzureDocumentIntelligenceSpan {
|
||||
fn end(&self) -> usize {
|
||||
self.offset.saturating_add(self.length)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AzureDocumentIntelligenceOcrConfig;
|
||||
|
||||
|
|
@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
type Environment = Vec<(String, String)>;
|
||||
|
||||
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
|
||||
&["pages", "features", "req_format"]
|
||||
&[
|
||||
"pages",
|
||||
"features",
|
||||
OUTPUT_CONTENT_FORMAT_PARAM,
|
||||
OUTPUT_CONTENT_FORMAT_ALIAS,
|
||||
"req_format",
|
||||
DROP_PARAMS_PARAM,
|
||||
]
|
||||
}
|
||||
|
||||
fn get_api_key_env_var(&self) -> Option<&'static str> {
|
||||
|
|
@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
fn map_ocr_params(
|
||||
&self,
|
||||
non_default_params: &CallArguments,
|
||||
_model: &str,
|
||||
model: &str,
|
||||
) -> Result<DocumentIntelligenceParams, Error> {
|
||||
let output_content_format = requested_output_content_format(non_default_params)?;
|
||||
let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown)
|
||||
&& response_format(non_default_params)? != OcrResponseFormat::Native
|
||||
&& model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID);
|
||||
Ok(DocumentIntelligenceParams {
|
||||
pages: normalize_pages_param(non_default_params.get("pages"))?,
|
||||
features: normalize_features_param(non_default_params.get("features"))?,
|
||||
output_content_format,
|
||||
string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result<Option<String>,
|
|||
Ok(Some(normalized.join(",")))
|
||||
}
|
||||
|
||||
fn requested_output_content_format(
|
||||
params: &CallArguments,
|
||||
) -> Result<Option<OutputContentFormat>, Error> {
|
||||
let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS]
|
||||
.into_iter()
|
||||
.filter_map(|name| params.get(name))
|
||||
.find(|format| !is_unset(format));
|
||||
match requested.map(parse_output_content_format).transpose() {
|
||||
Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None),
|
||||
parsed => parsed,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unset(format: &Value) -> bool {
|
||||
match format {
|
||||
Value::Null => true,
|
||||
Value::String(format) => format.trim().is_empty(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_output_content_format(format: &Value) -> Result<OutputContentFormat, Error> {
|
||||
format
|
||||
.as_str()
|
||||
.ok_or(Error::OutputContentFormat)?
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| Error::OutputContentFormat)
|
||||
}
|
||||
|
||||
fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, Error> {
|
||||
let source = document.source();
|
||||
if source.is_empty() {
|
||||
|
|
@ -367,10 +464,27 @@ fn transform_completed_response(
|
|||
));
|
||||
}
|
||||
let result = response.analyze_result.unwrap_or_default();
|
||||
let markdown_content = result
|
||||
.content
|
||||
.as_deref()
|
||||
.filter(|_| {
|
||||
result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown))
|
||||
&& result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX)
|
||||
&& result.pages.iter().all(|page| page.spans.is_some())
|
||||
})
|
||||
.map(|content| {
|
||||
CodePointText::new(
|
||||
content,
|
||||
result
|
||||
.pages
|
||||
.iter()
|
||||
.flat_map(|page| page.spans.iter().flatten()),
|
||||
)
|
||||
});
|
||||
let pages = result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|page| transform_azure_page(page, dpi))
|
||||
.map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
|
||||
Ok(LiteLLMOcrResponse {
|
||||
|
|
@ -381,11 +495,20 @@ fn transform_completed_response(
|
|||
pages_processed: Some(pages_processed),
|
||||
..Default::default()
|
||||
}),
|
||||
extra_fields: result
|
||||
.string_index_type
|
||||
.filter(|unit| unit == CODE_POINT_STRING_INDEX)
|
||||
.map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))]))
|
||||
.unwrap_or_default(),
|
||||
..LiteLLMOcrResponse::new(model, pages)
|
||||
})
|
||||
}
|
||||
|
||||
fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
|
||||
fn transform_azure_page(
|
||||
page: AzureDocumentIntelligencePage,
|
||||
markdown_content: Option<&CodePointText<'_>>,
|
||||
dpi: i64,
|
||||
) -> Result<OcrPage, Error> {
|
||||
let index = page
|
||||
.page_number
|
||||
.unwrap_or(1)
|
||||
|
|
@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
|
|||
page.unit.as_deref().unwrap_or("inch"),
|
||||
dpi,
|
||||
)?;
|
||||
let markdown = page
|
||||
.lines
|
||||
.iter()
|
||||
.map(|line| line.content.as_deref().unwrap_or_default())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let markdown = match markdown_content {
|
||||
Some(content) => page
|
||||
.spans
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|span| content.slice(span))
|
||||
.collect::<String>(),
|
||||
None => page
|
||||
.lines
|
||||
.iter()
|
||||
.map(|line| line.content.as_deref().unwrap_or_default())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
};
|
||||
Ok(OcrPage {
|
||||
index,
|
||||
markdown,
|
||||
|
|
@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
|
|||
})
|
||||
}
|
||||
|
||||
struct CodePointText<'a> {
|
||||
text: &'a str,
|
||||
byte_offsets: BTreeMap<usize, usize>,
|
||||
}
|
||||
|
||||
impl<'a> CodePointText<'a> {
|
||||
fn new<'s>(
|
||||
text: &'a str,
|
||||
spans: impl Iterator<Item = &'s AzureDocumentIntelligenceSpan>,
|
||||
) -> Self {
|
||||
let span_bounds = spans
|
||||
.flat_map(|span| [span.offset, span.end()])
|
||||
.collect::<BTreeSet<_>>();
|
||||
Self {
|
||||
text,
|
||||
byte_offsets: text
|
||||
.char_indices()
|
||||
.map(|(byte_offset, _)| byte_offset)
|
||||
.enumerate()
|
||||
.filter(|(code_point, _)| span_bounds.contains(code_point))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str {
|
||||
let byte_offset = |code_point: usize| {
|
||||
self.byte_offsets
|
||||
.get(&code_point)
|
||||
.copied()
|
||||
.unwrap_or(self.text.len())
|
||||
};
|
||||
self.text
|
||||
.get(byte_offset(span.offset)..byte_offset(span.end()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_dimensions(
|
||||
width: f64,
|
||||
height: f64,
|
||||
|
|
@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig {
|
|||
.features
|
||||
.iter()
|
||||
.map(|features| ("features", features.as_str())),
|
||||
)
|
||||
.chain(
|
||||
params
|
||||
.output_content_format
|
||||
.map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())),
|
||||
)
|
||||
.chain(
|
||||
params
|
||||
.string_index_type
|
||||
.map(|index_type| ("stringIndexType", index_type)),
|
||||
),
|
||||
)
|
||||
.into_string()
|
||||
|
|
@ -697,8 +875,12 @@ mod tests {
|
|||
}
|
||||
|
||||
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
|
||||
map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout")
|
||||
}
|
||||
|
||||
fn map_for_model(value: Value, model: &str) -> Result<DocumentIntelligenceParams, Error> {
|
||||
let arguments = serde_json::from_value(value).unwrap();
|
||||
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
|
||||
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model)
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -828,6 +1010,218 @@ mod tests {
|
|||
assert_eq!(map(json!({"features": []})).unwrap().features, None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::markdown(
|
||||
json!({"outputContentFormat": "markdown"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::markdown_litellm_format(
|
||||
json!({"outputContentFormat": "markdown", "req_format": "litellm"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::markdown_native_format(
|
||||
json!({"outputContentFormat": "markdown", "req_format": "native"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
None
|
||||
)]
|
||||
#[case::capitalized_markdown(
|
||||
json!({"outputContentFormat": "Markdown"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)]
|
||||
#[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)]
|
||||
#[case::padded_markdown(
|
||||
json!({"outputContentFormat": " markdown\n"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::snake_case_markdown(
|
||||
json!({"output_content_format": "markdown"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::camel_case_wins_over_snake_case(
|
||||
json!({"outputContentFormat": "text", "output_content_format": "markdown"}),
|
||||
Some(OutputContentFormat::Text),
|
||||
None
|
||||
)]
|
||||
#[case::empty_camel_case_falls_back_to_snake_case(
|
||||
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::null_camel_case_falls_back_to_snake_case(
|
||||
json!({"outputContentFormat": null, "output_content_format": "markdown"}),
|
||||
Some(OutputContentFormat::Markdown),
|
||||
Some("unicodeCodePoint")
|
||||
)]
|
||||
#[case::empty(json!({"outputContentFormat": ""}), None, None)]
|
||||
#[case::blank(json!({"outputContentFormat": " "}), None, None)]
|
||||
#[case::null(json!({"outputContentFormat": null}), None, None)]
|
||||
#[case::omitted(json!({}), None, None)]
|
||||
fn output_content_format_maps_to_query_fields(
|
||||
#[case] options: Value,
|
||||
#[case] format: Option<OutputContentFormat>,
|
||||
#[case] string_index_type: Option<&str>,
|
||||
) {
|
||||
let mapped = map(options).unwrap();
|
||||
|
||||
assert_eq!(mapped.output_content_format, format);
|
||||
assert_eq!(mapped.string_index_type, string_index_type);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))]
|
||||
#[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))]
|
||||
#[case::read("azure_ai/doc-intelligence/prebuilt-read", None)]
|
||||
#[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)]
|
||||
#[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)]
|
||||
fn only_the_layout_model_asks_for_code_point_offsets(
|
||||
#[case] model: &str,
|
||||
#[case] string_index_type: Option<&str>,
|
||||
) {
|
||||
let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
mapped.output_content_format,
|
||||
Some(OutputContentFormat::Markdown)
|
||||
);
|
||||
assert_eq!(mapped.string_index_type, string_index_type);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown(json!({"outputContentFormat": "html"}))]
|
||||
#[case::number(json!({"outputContentFormat": 1}))]
|
||||
#[case::boolean(json!({"outputContentFormat": true}))]
|
||||
#[case::list(json!({"outputContentFormat": ["markdown"]}))]
|
||||
#[case::snake_case_unknown(json!({"output_content_format": "html"}))]
|
||||
#[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))]
|
||||
fn invalid_output_content_format_is_a_request_error(#[case] options: Value) {
|
||||
let error = map(options).unwrap_err();
|
||||
|
||||
assert!(matches!(error, Error::OutputContentFormat), "{error:?}");
|
||||
assert_eq!(error.http_status_code(), Some(400));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))]
|
||||
#[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))]
|
||||
#[case::snake_case_unknown(
|
||||
json!({"output_content_format": "html", "drop_params": true, "pages": [0]})
|
||||
)]
|
||||
fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) {
|
||||
let mapped = map(options).unwrap();
|
||||
|
||||
assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"}));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))]
|
||||
#[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))]
|
||||
fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) {
|
||||
let mapped = map(options).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
mapped.output_content_format,
|
||||
Some(OutputContentFormat::Markdown)
|
||||
);
|
||||
}
|
||||
|
||||
fn markdown_pages(analyze_result: Value) -> Vec<String> {
|
||||
let raw =
|
||||
serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result}))
|
||||
.unwrap();
|
||||
AzureDocumentIntelligenceOcrConfig
|
||||
.transform_ocr_response("model", &raw, OcrResponseFormat::Litellm)
|
||||
.unwrap()
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(|page| page.markdown)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn markdown_pages_are_code_point_slices_of_the_content() {
|
||||
let content = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n## Två 🙂\n\n|a|\n\n> tail";
|
||||
let second_page_start = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n".chars().count();
|
||||
let pages = markdown_pages(json!({
|
||||
"content": content,
|
||||
"contentFormat": "markdown",
|
||||
"stringIndexType": "unicodeCodePoint",
|
||||
"pages": [
|
||||
{"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}],
|
||||
"lines": [{"content": "Ünïcode क्ष"}]},
|
||||
{"pageNumber": 2, "spans": [
|
||||
{"offset": second_page_start, "length": 8},
|
||||
{"offset": second_page_start + 10, "length": 3}
|
||||
], "lines": [{"content": "Två"}]}
|
||||
]
|
||||
}));
|
||||
|
||||
assert_eq!(
|
||||
pages,
|
||||
["# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n", "## Två 🙂|a|"]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn spans_past_the_content_end_are_clamped() {
|
||||
let pages = markdown_pages(json!({
|
||||
"content": "ab",
|
||||
"contentFormat": "markdown",
|
||||
"stringIndexType": "unicodeCodePoint",
|
||||
"pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}]
|
||||
}));
|
||||
|
||||
assert_eq!(pages, ["b"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::text_content(json!("text"), json!("unicodeCodePoint"))]
|
||||
#[case::grapheme_offsets(json!("markdown"), json!("textElements"))]
|
||||
#[case::no_index_type(json!("markdown"), Value::Null)]
|
||||
#[case::no_content_format(Value::Null, json!("unicodeCodePoint"))]
|
||||
fn other_responses_keep_line_joined_pages(
|
||||
#[case] content_format: Value,
|
||||
#[case] string_index_type: Value,
|
||||
) {
|
||||
let pages = markdown_pages(json!({
|
||||
"content": "# A\n\nB",
|
||||
"contentFormat": content_format,
|
||||
"stringIndexType": string_index_type,
|
||||
"pages": [{"spans": [{"offset": 0, "length": 6}],
|
||||
"lines": [{"content": "A"}, {"content": "B"}]}]
|
||||
}));
|
||||
|
||||
assert_eq!(pages, ["A\nB"]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::null_spans(json!("markdown"), json!(null))]
|
||||
#[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))]
|
||||
#[case::missing_length(json!("markdown"), json!([{"offset": 0}]))]
|
||||
#[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))]
|
||||
#[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))]
|
||||
fn unusable_page_spans_keep_line_joined_pages(
|
||||
#[case] content_format: Value,
|
||||
#[case] broken_spans: Value,
|
||||
) {
|
||||
let pages = markdown_pages(json!({
|
||||
"content": "# A\n\nB",
|
||||
"contentFormat": content_format,
|
||||
"stringIndexType": "unicodeCodePoint",
|
||||
"pages": [
|
||||
{"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]},
|
||||
{"spans": broken_spans, "lines": [{"content": "B"}]}
|
||||
]
|
||||
}));
|
||||
|
||||
assert_eq!(pages, ["A", "B"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_endpoint_cannot_receive_environment_key() {
|
||||
let connection = OcrConnection {
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ pub enum Error {
|
|||
Pages(PagesFault),
|
||||
#[error("invalid OCR features")]
|
||||
Features,
|
||||
#[error(
|
||||
"invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`."
|
||||
)]
|
||||
OutputContentFormat,
|
||||
#[error("OCR model cannot be a dot segment")]
|
||||
DotModel,
|
||||
#[error("OCR response exceeds the size limit of {limit} bytes")]
|
||||
|
|
@ -200,6 +204,7 @@ impl Error {
|
|||
| Self::InlineDocumentTooLarge
|
||||
| Self::Pages(_)
|
||||
| Self::Features
|
||||
| Self::OutputContentFormat
|
||||
| Self::DotModel
|
||||
| Self::TooLarge { .. }
|
||||
| Self::ResponseField { .. }
|
||||
|
|
@ -242,6 +247,7 @@ impl Error {
|
|||
| Self::InlineDocumentTooLarge
|
||||
| Self::Pages(_)
|
||||
| Self::Features
|
||||
| Self::OutputContentFormat
|
||||
| Self::DotModel
|
||||
| Self::TooLarge { .. }
|
||||
| Self::ResponseField { .. }
|
||||
|
|
@ -291,6 +297,7 @@ impl Error {
|
|||
| Self::InlineDocumentTooLarge
|
||||
| Self::Pages(_)
|
||||
| Self::Features
|
||||
| Self::OutputContentFormat
|
||||
| Self::DotModel
|
||||
| Self::InvalidProvider(_)
|
||||
| Self::InvalidModel { .. }
|
||||
|
|
@ -345,6 +352,7 @@ impl Error {
|
|||
| Self::InlineDocumentTooLarge
|
||||
| Self::Pages(_)
|
||||
| Self::Features
|
||||
| Self::OutputContentFormat
|
||||
| Self::DotModel
|
||||
| Self::InvalidProvider(_)
|
||||
| Self::InvalidModel { .. }
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
|
|||
pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
|
||||
pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
|
||||
pub const OCR_POLL_RETRY_SECS: u64 = 2;
|
||||
pub const DROP_PARAMS_PARAM: &str = "drop_params";
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OcrCredentialInputs {
|
||||
|
|
@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result<OcrResponseFor
|
|||
.map(|format| format.unwrap_or_default())
|
||||
}
|
||||
|
||||
pub fn drops_unsupported_params(optional_params: &CallArguments) -> bool {
|
||||
optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DecodedOcrResponse<T> {
|
||||
pub data: T,
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ use litellm_auth_aws::{
|
|||
};
|
||||
use litellm_core_utils::core_helpers::json_type_name;
|
||||
use litellm_llms_types::{
|
||||
formats::audio_transcription::AudioTranscriptionResponseData,
|
||||
providers::bedrock::ConverseResponse,
|
||||
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
|
||||
providers::bedrock::{ConverseResponse, ConverseUsage},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
|
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
if let Some(prompt) = optional_string(&optional_params, "prompt") {
|
||||
instruction.push_str(&format!(" Additional context: {prompt}"));
|
||||
}
|
||||
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
|
||||
if let Some(temperature) = optional_params.get("temperature") {
|
||||
inference_config.insert("temperature".to_string(), temperature.clone());
|
||||
}
|
||||
let temperature = optional_params
|
||||
.get("temperature")
|
||||
.cloned()
|
||||
.unwrap_or(json!(0));
|
||||
Ok(AudioTranscriptionRequestData {
|
||||
body: json!({
|
||||
"messages": [{
|
||||
|
|
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
{"text": instruction}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": inference_config,
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
|
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
_model: &str,
|
||||
response_json: Value,
|
||||
) -> Result<AudioTranscriptionResponseData, Error> {
|
||||
let reports_usage = response_json.get("usage").is_some();
|
||||
let response: ConverseResponse =
|
||||
serde_json::from_value(response_json).map_err(|error| {
|
||||
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
|
||||
|
|
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
Ok(AudioTranscriptionResponseData {
|
||||
text: response.content_text(),
|
||||
usage: reports_usage.then(|| transcription_usage(&response.usage)),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
}
|
||||
|
||||
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
|
||||
AudioTranscriptionUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
total_tokens: usage
|
||||
.total_tokens
|
||||
.unwrap_or(usage.input_tokens + usage.output_tokens),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_matches_python_shape() {
|
||||
let params = Map::from_iter([
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("temperature".to_string(), json!(0)),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]);
|
||||
#[rstest]
|
||||
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
|
||||
#[case::default_temperature(None, json!(0))]
|
||||
fn request_sends_audio_and_instruction_without_system_block(
|
||||
#[case] temperature: Option<Value>,
|
||||
#[case] expected_temperature: Value,
|
||||
) {
|
||||
let params = Map::from_iter(
|
||||
[
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]
|
||||
.into_iter()
|
||||
.chain(temperature.map(|value| ("temperature".to_string(), value))),
|
||||
);
|
||||
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms);
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_request(
|
||||
|
|
@ -225,22 +245,39 @@ mod tests {
|
|||
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_concatenates_content_blocks() {
|
||||
#[rstest]
|
||||
#[case::reported_total(
|
||||
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
|
||||
)]
|
||||
#[case::derived_total(
|
||||
json!({"inputTokens": 1, "outputTokens": 2}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
|
||||
)]
|
||||
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.text, "hello world");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello world"}));
|
||||
assert_eq!(result.into_json(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn response_without_usage_reports_none() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
|
|||
## Tests
|
||||
|
||||
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
|
||||
|
||||
`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
|
|||
|
||||
[dependencies]
|
||||
litellm-cost = { workspace = true, features = ["serde"] }
|
||||
litellm-error.workspace = true
|
||||
|
||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
|
||||
|
|
@ -24,5 +25,4 @@ time.workspace = true
|
|||
|
||||
[dev-dependencies]
|
||||
jsonschema = { version = "0.55.1", default-features = false }
|
||||
litellm-error.workspace = true
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs
Normal file
26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
use std::{env, fs, path::PathBuf, process::ExitCode};
|
||||
|
||||
use litellm_model_catalog::Catalog;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
|
||||
eprintln!("usage: check-registry <model_prices_and_context_window.json>");
|
||||
return ExitCode::from(2);
|
||||
};
|
||||
let body = match fs::read(&path) {
|
||||
Ok(body) => body,
|
||||
Err(error) => {
|
||||
eprintln!("{}: {error}", path.display());
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let problems = Catalog::problems(&body);
|
||||
for problem in &problems {
|
||||
eprintln!("{}", litellm_error::Report::new(problem));
|
||||
}
|
||||
if problems.is_empty() {
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ use indexmap::IndexMap;
|
|||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::error::{Error, Loaded, Rejects, Repr};
|
||||
use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
|
||||
use crate::fallback::{FallbackGeneralizations, FallbackRule};
|
||||
use crate::model::Model;
|
||||
use crate::store::{Rows, Store};
|
||||
|
|
@ -211,6 +211,36 @@ impl Catalog {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn problems(body: &[u8]) -> Vec<Error> {
|
||||
let Loaded {
|
||||
value: catalog,
|
||||
errors,
|
||||
} = match Self::parse(body) {
|
||||
Ok(loaded) => loaded,
|
||||
Err(fatal) => return vec![fatal],
|
||||
};
|
||||
let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
|
||||
EntryWire::read(entry.fields())
|
||||
.unknown
|
||||
.into_iter()
|
||||
.map(move |(key, _)| {
|
||||
Error::new(
|
||||
pointer_child(&pointer_for_key(model), &key),
|
||||
Repr::UnreadField,
|
||||
)
|
||||
})
|
||||
});
|
||||
let no_fallback_rules = catalog
|
||||
.fallback_rules()
|
||||
.is_none_or(<[_]>::is_empty)
|
||||
.then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
|
||||
errors
|
||||
.into_iter()
|
||||
.chain(unread)
|
||||
.chain(no_fallback_rules)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
|
||||
let (_, _, errors) = read_row(fields);
|
||||
errors
|
||||
|
|
@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
|
|||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
|
||||
|
||||
#[rstest]
|
||||
#[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
|
||||
#[case::not_json("{".to_owned(), vec![("", "json")])]
|
||||
#[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
|
||||
#[case::unread_key(
|
||||
format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
|
||||
vec![("/m/typo_cost", "unread_field")]
|
||||
)]
|
||||
#[case::empty_model_key(
|
||||
format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
|
||||
vec![("//future_field", "unread_field")]
|
||||
)]
|
||||
#[case::unread_key_on_invalid_row(
|
||||
format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
|
||||
vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
|
||||
)]
|
||||
#[case::escaped_unread_key_on_invalid_row(
|
||||
format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
|
||||
vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
|
||||
)]
|
||||
#[case::no_rules(
|
||||
r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
|
||||
vec![("/fallback_generalizations", "no_fallback_rules")]
|
||||
)]
|
||||
#[case::empty_rules(
|
||||
r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
|
||||
vec![("/fallback_generalizations", "no_fallback_rules")]
|
||||
)]
|
||||
fn problems_list_what_a_cost_map_change_must_fix(
|
||||
#[case] body: String,
|
||||
#[case] expected: Vec<(&str, &str)>,
|
||||
) {
|
||||
let problems: Vec<_> = Catalog::problems(body.as_bytes())
|
||||
.iter()
|
||||
.map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
|
||||
.collect();
|
||||
let expected: Vec<_> = expected
|
||||
.into_iter()
|
||||
.map(|(path, kind)| (path.to_owned(), kind))
|
||||
.collect();
|
||||
assert_eq!(problems, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
|
||||
let body = br#"{
|
||||
"good":{"litellm_provider":"test","custom_metadata":true},
|
||||
"bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
|
||||
}"#;
|
||||
let loaded = Catalog::parse(body).unwrap();
|
||||
assert_eq!(loaded.errors.len(), 1);
|
||||
assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
|
||||
let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
|
||||
assert_eq!(rejected.len(), 1);
|
||||
assert!(catalog.lookup("bad").is_none());
|
||||
let good = catalog.lookup("good").unwrap();
|
||||
assert!(good.entry.model().is_some());
|
||||
assert_eq!(
|
||||
good.entry.field("custom_metadata"),
|
||||
Some(&Value::Bool(true))
|
||||
);
|
||||
assert!(
|
||||
Catalog::problems(body)
|
||||
.iter()
|
||||
.any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
|
||||
);
|
||||
assert!(
|
||||
Catalog::problems(body)
|
||||
.iter()
|
||||
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,6 +109,12 @@ pub(crate) enum Repr {
|
|||
#[source]
|
||||
cause: Option<DateCause>,
|
||||
},
|
||||
#[error("not a field litellm reads")]
|
||||
#[strum_discriminants(strum(serialize = "unread_field"))]
|
||||
UnreadField,
|
||||
#[error("has no rules")]
|
||||
#[strum_discriminants(strum(serialize = "no_fallback_rules"))]
|
||||
NoFallbackRules,
|
||||
}
|
||||
|
||||
/// What a problem takes out of the served map
|
||||
|
|
@ -148,7 +154,7 @@ impl Error {
|
|||
| Repr::NoOffPeakHours
|
||||
| Repr::EmptyList
|
||||
| Repr::InvalidDate { .. } => Rejects::Row,
|
||||
Repr::MissingProvider => Rejects::Nothing,
|
||||
Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -334,6 +340,8 @@ mod tests {
|
|||
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
|
||||
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
|
||||
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
|
||||
#[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
|
||||
#[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
|
||||
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
|
||||
assert_eq!(Error::from(error).rejects(), expected);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
|
||||
let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
|
||||
let catalog = loaded.value;
|
||||
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
|
||||
let unrecognized: Vec<_> = catalog.unrecognized().collect();
|
||||
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
|
||||
assert!(
|
||||
catalog
|
||||
.fallback_rules()
|
||||
.is_some_and(|rules| !rules.is_empty())
|
||||
);
|
||||
fn the_bundled_registry_has_no_problems() {
|
||||
let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY)
|
||||
.iter()
|
||||
.map(report)
|
||||
.collect();
|
||||
assert!(problems.is_empty(), "{problems:#?}");
|
||||
const REFERENCE_MODEL_COUNT: usize = 4303;
|
||||
let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
|
||||
assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,18 @@ GIL handling to `litellm-host-python`.
|
|||
|
||||
## Data Handling
|
||||
|
||||
- Reuse large immutable bundled data and parsed catalogs across calls, including in
|
||||
replacement APIs. Share Rust storage and cache repeated Python projections instead
|
||||
of reparsing, serializing or copying the full payload on every access
|
||||
- Cache owned Python objects with `PyOnceLock<Py<T>>`, then bind and clone references
|
||||
while attached. Do not use blocking `OnceLock::get_or_init` while attached
|
||||
([PyO3 initialization](https://pyo3.rs/v0.29.2/faq.html),
|
||||
[reference ownership](https://pyo3.rs/v0.29.2/types.html))
|
||||
- Never reuse Python objects across interpreters or interpreter reinitialization.
|
||||
Once-only caches are for immutable data. Reloadable/custom data needs versioned
|
||||
entries or invalidation, and caller-mutable Python containers must not be shared
|
||||
- Test unchanged contents and allocation reuse, not just equal values. Keep this
|
||||
coverage when replacing a bundled-data accessor with a catalog API
|
||||
- OCR payloads can contain personal data and large base64 images. Do not log
|
||||
payloads or provider responses.
|
||||
- Avoid copying large payloads more than needed. The current JSON round-trip is
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ litellm-secrets-types.workspace = true
|
|||
litellm-llms-types.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
pyo3.workspace = true
|
||||
prost.workspace = true
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ mod execution;
|
|||
mod http;
|
||||
mod logger;
|
||||
mod marshal;
|
||||
mod model_registry;
|
||||
mod preflight;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
|
|
@ -30,6 +31,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::bundled_model_cost_map;
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{acompletion, completion};
|
||||
|
|
@ -110,6 +113,7 @@ mod tests {
|
|||
"NativeClickHouseSpendConfig",
|
||||
"NativeClickHouseSpendStorage",
|
||||
"trace_encode_error",
|
||||
"bundled_model_cost_map",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
|
|
|
|||
30
litellm-rust/crates/python-bridge/src/model_registry.rs
Normal file
30
litellm-rust/crates/python-bridge/src/model_registry.rs
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
use pyo3::prelude::*;
|
||||
use pyo3::sync::PyOnceLock;
|
||||
use pyo3::types::PyBytes;
|
||||
|
||||
#[pyfunction]
|
||||
pub fn bundled_model_cost_map(py: Python<'_>) -> Bound<'_, PyBytes> {
|
||||
static BYTES: PyOnceLock<Py<PyBytes>> = PyOnceLock::new();
|
||||
BYTES
|
||||
.get_or_init(py, || {
|
||||
PyBytes::new(py, litellm_model_catalog::BUNDLED_REGISTRY).unbind()
|
||||
})
|
||||
.bind(py)
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
fn repeated_calls_reuse_the_bundled_registry_bytes() {
|
||||
Python::initialize();
|
||||
let first = Python::attach(|py| bundled_model_cost_map(py).unbind());
|
||||
Python::attach(|py| {
|
||||
let second = bundled_model_cost_map(py);
|
||||
assert!(first.bind(py).is(&second));
|
||||
assert_eq!(second.as_bytes(), litellm_model_catalog::BUNDLED_REGISTRY);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ use pyo3::{
|
|||
|
||||
use super::{
|
||||
errors::route_failure,
|
||||
project::{OcrHostHandles, SETTINGS_ERROR_MARKER, project_request},
|
||||
project::{OcrHostHandles, ROUTE_HOST_MODULE, SETTINGS_ERROR_MARKER, project_request},
|
||||
};
|
||||
use crate::{marshal::public_response, routes::RequestView};
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ impl OcrPythonHost {
|
|||
_ => "",
|
||||
};
|
||||
let mapped = py
|
||||
.import("litellm.rust_bridge.host.routes.ocr")
|
||||
.import(ROUTE_HOST_MODULE)
|
||||
.and_then(|module| module.getattr("map_failure"))
|
||||
.and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider)))
|
||||
.and_then(|mapped| mapped.extract::<Py<PyBaseException>>().map_err(PyErr::from));
|
||||
|
|
@ -115,7 +115,7 @@ impl PythonBinding for OcrPythonHost {
|
|||
py: Python<'_>,
|
||||
response: LiteLLMOcrResponse,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
public_response(py, "litellm.rust_bridge.host.routes.ocr", &response)
|
||||
public_response(py, ROUTE_HOST_MODULE, &response)
|
||||
}
|
||||
|
||||
fn encode_stream_head(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use litellm_inference_ocr::{
|
|||
types::{LiteLLMOcrRequest, OcrDocumentInput},
|
||||
wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input},
|
||||
};
|
||||
use litellm_llms::base_llm::ocr::error::Error;
|
||||
use litellm_llms::base_llm::ocr::{error::Error, transformation::DROP_PARAMS_PARAM};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -19,6 +19,8 @@ use crate::{
|
|||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.ocr";
|
||||
|
||||
/// What the host keeps after projection: the caller's token callable that answers the
|
||||
/// token operation, and the provider name the failure mapping reports.
|
||||
pub(super) struct OcrHostHandles {
|
||||
|
|
@ -139,6 +141,15 @@ pub(super) fn project_request(
|
|||
Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
|
||||
None => optional_params,
|
||||
};
|
||||
let optional_params = if names.contains(&DROP_PARAMS_PARAM) {
|
||||
fold_drop_params(
|
||||
bound.py(),
|
||||
optional_params,
|
||||
present(kwargs, bound, DROP_PARAMS_PARAM)?,
|
||||
)?
|
||||
} else {
|
||||
optional_params
|
||||
};
|
||||
let input_sources = request_input_sources(
|
||||
kwargs,
|
||||
names
|
||||
|
|
@ -216,6 +227,23 @@ fn fold_module_globals(
|
|||
.collect())
|
||||
}
|
||||
|
||||
fn fold_drop_params(
|
||||
py: Python<'_>,
|
||||
optional_params: Map<String, Value>,
|
||||
requested: Option<Bound<'_, PyAny>>,
|
||||
) -> PyResult<Map<String, Value>> {
|
||||
let drops: bool = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("drops_unsupported_params")?
|
||||
.call1((requested,))?
|
||||
.extract()?;
|
||||
Ok(optional_params
|
||||
.into_iter()
|
||||
.filter(|(name, _)| name != DROP_PARAMS_PARAM)
|
||||
.chain(drops.then(|| (DROP_PARAMS_PARAM.to_string(), Value::Bool(true))))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// A misconfigured litellm setting is the operator's error, not the request's or the
|
||||
/// provider's, so the host raises it as is instead of mapping it onto a provider failure.
|
||||
pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error";
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm._logging import (
|
|||
turn_on_json,
|
||||
log_level,
|
||||
)
|
||||
from litellm._logging import suppress_startup_output as suppress_startup_output
|
||||
import re
|
||||
from litellm.constants import (
|
||||
DEFAULT_BATCH_SIZE,
|
||||
|
|
@ -519,7 +520,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
|
|||
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
|
||||
disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars
|
||||
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
|
||||
http2: bool = False
|
||||
http2: bool | None = None
|
||||
network_mock: bool = False # When True, use mock transport — no real network calls
|
||||
|
||||
####### STOP SEQUENCE LIMIT #######
|
||||
|
|
@ -591,7 +592,7 @@ vertex_ai_safety_settings: Optional[dict] = None
|
|||
####### COMPLETION MODELS ###################
|
||||
from typing import Set
|
||||
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_chat_completion_models: Set[str] = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
|
|
@ -619,6 +620,8 @@ vertex_openai_models: Set = set()
|
|||
vertex_minimax_models: Set = set()
|
||||
vertex_moonshot_models: Set = set()
|
||||
vertex_zai_models: Set = set()
|
||||
vertex_qwen_models: Final[Set[str]] = set()
|
||||
vertex_ai_models: Final[Set[str]] = set()
|
||||
ai21_models: Set = set()
|
||||
ai21_chat_models: Set = set()
|
||||
nlp_cloud_models: Set = set()
|
||||
|
|
@ -635,6 +638,7 @@ watsonx_models: Set = set()
|
|||
gemini_models: Set = set()
|
||||
xai_models: Set = set()
|
||||
zai_models: Set = set()
|
||||
bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime
|
||||
deepseek_models: Set = set()
|
||||
tencent_models: Set = set()
|
||||
runwayml_models: Set = set()
|
||||
|
|
@ -681,6 +685,8 @@ moonshot_models: Set = set()
|
|||
publicai_models: Set = set()
|
||||
darkbloom_models: Set = set()
|
||||
openinfer_models: Final[Set[str]] = set()
|
||||
kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
streamlake_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
|
|
@ -832,6 +838,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
elif value.get("litellm_provider") == "vertex_ai-zai_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_zai_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-qwen_models":
|
||||
vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
|
||||
elif value.get("litellm_provider") == "vertex_ai":
|
||||
vertex_ai_models.add(key.removeprefix("vertex_ai/"))
|
||||
elif value.get("litellm_provider") == "ai21":
|
||||
if value.get("mode") == "chat":
|
||||
ai21_chat_models.add(key)
|
||||
|
|
@ -876,6 +886,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
xai_models.add(key)
|
||||
elif value.get("litellm_provider") == "zai":
|
||||
zai_models.add(key)
|
||||
elif value.get("litellm_provider") == "bigmodel":
|
||||
bigmodel_models.add(key)
|
||||
elif value.get("litellm_provider") == "fal_ai":
|
||||
fal_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepseek":
|
||||
|
|
@ -970,6 +982,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
darkbloom_models.add(key)
|
||||
elif value.get("litellm_provider") == "openinfer":
|
||||
openinfer_models.add(key)
|
||||
elif value.get("litellm_provider") == "kyma":
|
||||
kyma_models.add(key)
|
||||
elif value.get("litellm_provider") == "streamlake":
|
||||
streamlake_models.add(key)
|
||||
elif value.get("litellm_provider") == "zerogpu":
|
||||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
|
|
@ -1104,6 +1120,7 @@ model_list = list(
|
|||
| text_completion_inception_models
|
||||
| xai_models
|
||||
| zai_models
|
||||
| bigmodel_models
|
||||
| fal_ai_models
|
||||
| deepseek_models
|
||||
| modelscope_models
|
||||
|
|
@ -1145,6 +1162,8 @@ model_list = list(
|
|||
| publicai_models
|
||||
| darkbloom_models
|
||||
| openinfer_models
|
||||
| kyma_models
|
||||
| streamlake_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
|
|
@ -1200,7 +1219,16 @@ def _build_models_by_provider() -> dict:
|
|||
| vertex_deepseek_models
|
||||
| vertex_minimax_models
|
||||
| vertex_moonshot_models
|
||||
| vertex_zai_models,
|
||||
| vertex_zai_models
|
||||
| vertex_qwen_models
|
||||
| vertex_mistral_models
|
||||
| vertex_llama3_models
|
||||
| vertex_ai_ai21_models
|
||||
| vertex_openai_models
|
||||
| vertex_ai_image_models
|
||||
| vertex_ai_video_models
|
||||
| vertex_embedding_models
|
||||
| vertex_ai_models,
|
||||
"ai21": ai21_models | ai21_chat_models,
|
||||
"bedrock": bedrock_models | bedrock_converse_models,
|
||||
"petals": petals_models,
|
||||
|
|
@ -1217,6 +1245,7 @@ def _build_models_by_provider() -> dict:
|
|||
"text-completion-inception": text_completion_inception_models,
|
||||
"xai": xai_models,
|
||||
"zai": zai_models,
|
||||
"bigmodel": bigmodel_models,
|
||||
"fal_ai": fal_ai_models,
|
||||
"deepseek": deepseek_models,
|
||||
"tencent": tencent_models,
|
||||
|
|
@ -1265,6 +1294,8 @@ def _build_models_by_provider() -> dict:
|
|||
"publicai": publicai_models,
|
||||
"darkbloom": darkbloom_models,
|
||||
"openinfer": openinfer_models,
|
||||
"kyma": kyma_models,
|
||||
"streamlake": streamlake_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
|
|||
|
||||
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
|
||||
|
||||
_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def internal_sub_call() -> Generator[None]:
|
||||
|
|
@ -123,6 +125,30 @@ def current_service_caller() -> str | None:
|
|||
return _service_caller.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def mirrored_call() -> Generator[None]:
|
||||
"""A silent_model mirror of a request: every litellm call inside, its retries and fallbacks included."""
|
||||
token: Final = _mirrored_call.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_restore(_mirrored_call, token)
|
||||
|
||||
|
||||
def in_mirrored_call() -> bool:
|
||||
return _mirrored_call.get()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def outside_mirrored_call() -> Generator[None]:
|
||||
"""Logging a mirrored call: the calls its callbacks start are not mirrors themselves."""
|
||||
token: Final = _mirrored_call.set(False)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_restore(_mirrored_call, token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def pinned_billing_time(moment: datetime) -> Generator[None]:
|
||||
"""Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@ import sys
|
|||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from types import FrameType
|
||||
from typing import Final, TextIO
|
||||
from urllib.parse import unquote
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
|
|
@ -598,6 +600,7 @@ _LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
|
|||
_LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
|
||||
_PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
|
||||
_COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
|
||||
_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
|
||||
|
||||
|
||||
def _stream_is_tty(stream: TextIO | None) -> bool:
|
||||
|
|
@ -612,15 +615,28 @@ def _stream_is_tty(stream: TextIO | None) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
|
||||
"""The plain-text log format, colorized only when both streams are an interactive terminal.
|
||||
def _color_enabled(stream: TextIO | None) -> bool:
|
||||
"""True when ANSI colors written to the stream reach an interactive terminal.
|
||||
|
||||
Honors the NO_COLOR convention from no-color.org: color is disabled when
|
||||
NO_COLOR is present with a non-empty value.
|
||||
"""
|
||||
if os.environ.get("NO_COLOR"):
|
||||
return _PLAIN_LOG_FORMAT
|
||||
return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
|
||||
return not os.environ.get("NO_COLOR") and _stream_is_tty(stream)
|
||||
|
||||
|
||||
def _terminal_colors(stdout: TextIO | None, stderr: TextIO | None) -> bool:
|
||||
"""True when both streams are an interactive color terminal, the one rule for log lines and startup prints."""
|
||||
return _color_enabled(stdout) and _color_enabled(stderr)
|
||||
|
||||
|
||||
def terminal_text(message: str) -> str:
|
||||
"""The message with its ANSI colors kept only on a color terminal, by the rule the plain log format uses."""
|
||||
return message if _terminal_colors(sys.stdout, sys.stderr) else _ANSI_ESCAPE.sub("", message)
|
||||
|
||||
|
||||
def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
|
||||
"""The plain-text log format, colorized only when both streams are an interactive terminal."""
|
||||
return _COLOR_LOG_FORMAT if _terminal_colors(stdout, stderr) else _PLAIN_LOG_FORMAT
|
||||
|
||||
|
||||
class LevelRoutingStreamHandler(logging.StreamHandler):
|
||||
|
|
@ -642,8 +658,8 @@ class LevelRoutingStreamHandler(logging.StreamHandler):
|
|||
super().emit(record)
|
||||
|
||||
|
||||
def _parse_json_logs_env(value: str | None) -> bool:
|
||||
"""Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
|
||||
def _parse_env_flag(value: str | None) -> bool:
|
||||
"""Strict opt-in parse for a boolean env var such as JSON_LOGS: only "true" (any case) enables it.
|
||||
|
||||
Matches the reader in litellm-proxy-extras/_logging.py. The previous
|
||||
bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
|
||||
|
|
@ -656,7 +672,18 @@ def resolve_log_level(log_level: str) -> int:
|
|||
return getattr(logging, log_level.upper())
|
||||
|
||||
|
||||
json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
|
||||
json_logs: Final = _parse_env_flag(os.getenv("JSON_LOGS"))
|
||||
suppress_startup_output: Final = _parse_env_flag(os.getenv("LITELLM_SUPPRESS_STARTUP_OUTPUT"))
|
||||
|
||||
|
||||
class StartupOutputSettings(TypedDict, total=False):
|
||||
json_logs: ReadOnly[object]
|
||||
suppress_startup_output: ReadOnly[object]
|
||||
|
||||
|
||||
STARTUP_OUTPUT_SETTINGS: Final = TypeAdapter(StartupOutputSettings)
|
||||
|
||||
|
||||
# Create a handler for the logger (you may need to adapt this based on your needs)
|
||||
log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
|
||||
numeric_level: Final[int] = resolve_log_level(log_level)
|
||||
|
|
@ -743,7 +770,7 @@ class JsonFormatter(Formatter):
|
|||
return dt.isoformat()
|
||||
|
||||
def format(self, record):
|
||||
message_str: Final = record.getMessage()
|
||||
message_str: Final = _ANSI_ESCAPE.sub("", record.getMessage())
|
||||
json_record: Final[dict[str, object]] = {
|
||||
"message": message_str,
|
||||
"level": record.levelname,
|
||||
|
|
@ -799,12 +826,14 @@ class CorrelationPlainFormatter(logging.Formatter):
|
|||
"""Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter.
|
||||
|
||||
Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs
|
||||
behaves the same whether or not json_logs is enabled.
|
||||
behaves the same whether or not json_logs is enabled. Unless it was built with the color format,
|
||||
it also drops the ANSI color codes a message carries, which a piped log would store as text.
|
||||
"""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
rendered: Final = super().format(record)
|
||||
formatted: Final = rendered if _is_redacted(record) else redact_string(rendered)
|
||||
uncolored: Final = rendered if self._fmt == _COLOR_LOG_FORMAT else _ANSI_ESCAPE.sub("", rendered)
|
||||
formatted: Final = uncolored if _is_redacted(record) else redact_string(uncolored)
|
||||
trace_id: Final = getattr(record, "trace_id", None)
|
||||
session_id: Final = getattr(record, "session_id", None)
|
||||
if not trace_id and not session_id:
|
||||
|
|
@ -898,29 +927,46 @@ verbose_proxy_logger.addFilter(_diagnostic_filter)
|
|||
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
|
||||
verbose_logger.addFilter(_diagnostic_filter)
|
||||
|
||||
_ANSI_ESCAPE: Final = re.compile(r"\x1b\[[0-9;]*m")
|
||||
|
||||
|
||||
def print_startup_message(message: str) -> None:
|
||||
"""Write a boot-time message meant for the operator's terminal.
|
||||
|
||||
Plain mode prints it unchanged, colors and spacing included. With JSON logs on, every line of
|
||||
stdout is read by a log pipeline, so the message goes out as one INFO record of the proxy logger
|
||||
instead, on stdout like the print whatever handlers the logging config gave that logger, with the
|
||||
ANSI color codes and surrounding blank lines removed, located at the caller and emitted at any log
|
||||
level, whole and unredacted as the print showed it; a blank spacer produces no record.
|
||||
LITELLM_SUPPRESS_STARTUP_OUTPUT or litellm_settings.suppress_startup_output turns every one of them
|
||||
off; a warning goes through print_startup_warning instead. Plain mode prints it with its spacing,
|
||||
keeping the ANSI colors only on a color terminal by the same check as the plain log format, so a
|
||||
container log never stores escape codes.
|
||||
With JSON logs on, every line of stdout is read by a log pipeline, so the message goes out as one
|
||||
INFO record of the proxy logger instead, on stdout like the print whatever handlers the logging
|
||||
config gave that logger, with the ANSI color codes and surrounding blank lines removed, located at
|
||||
the caller and emitted at any log level, whole and unredacted as the print showed it; a blank
|
||||
spacer produces no record.
|
||||
"""
|
||||
if litellm.suppress_startup_output is True:
|
||||
return
|
||||
frame: Final = inspect.currentframe()
|
||||
_write_startup_message(message, frame.f_back if frame is not None else None, logging.INFO)
|
||||
|
||||
|
||||
def print_startup_warning(message: str) -> None:
|
||||
"""Write a boot-time warning as print_startup_message writes a message, even with suppress_startup_output
|
||||
on, since it names something the operator has to fix. With JSON logs on it is a WARNING record, on
|
||||
stdout where the print put it, so a collector that filters on the level field keeps it."""
|
||||
frame: Final = inspect.currentframe()
|
||||
_write_startup_message(message, frame.f_back if frame is not None else None, logging.WARNING)
|
||||
|
||||
|
||||
def _write_startup_message(message: str, caller: FrameType | None, level: int) -> None:
|
||||
if not litellm.json_logs:
|
||||
print(message) # noqa: T201 # plain mode: this is the operator-terminal print it replaces
|
||||
import click
|
||||
|
||||
click.echo(terminal_text(message))
|
||||
return
|
||||
text: Final = _ANSI_ESCAPE.sub("", message).strip("\n")
|
||||
if not text:
|
||||
return
|
||||
frame: Final = inspect.currentframe()
|
||||
caller: Final = frame.f_back if frame is not None else None
|
||||
record: Final = verbose_proxy_logger.makeRecord(
|
||||
verbose_proxy_logger.name,
|
||||
logging.INFO,
|
||||
level,
|
||||
caller.f_code.co_filename if caller is not None else __file__,
|
||||
caller.f_lineno if caller is not None else 0,
|
||||
text,
|
||||
|
|
@ -934,8 +980,8 @@ def print_startup_message(message: str) -> None:
|
|||
|
||||
|
||||
def _startup_json_handler() -> logging.Handler:
|
||||
"""The proxy's JSON handler on stdout, filters included, for one startup message."""
|
||||
handler: Final = LevelRoutingStreamHandler()
|
||||
"""The proxy's JSON handler on stdout, filters included, for one startup message or warning."""
|
||||
handler: Final = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
handler.addFilter(_secret_filter)
|
||||
handler.addFilter(_correlation_filter)
|
||||
|
|
|
|||
325
litellm/anthropic_beta_gated_fields.py
Normal file
325
litellm/anthropic_beta_gated_fields.py
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
"""Body fields that an ``anthropic-beta`` value gates.
|
||||
|
||||
``anthropic_beta_headers_config.json`` says which beta each provider leg
|
||||
carries. When a leg drops a beta, the request fields that beta unlocks
|
||||
would make the provider 400. Under ``drop_params`` the same request degrades on
|
||||
that leg by dropping those fields too; without it the request goes out as sent
|
||||
and the provider answers its own 400, like any unsupported param. Every helper
|
||||
here is copy-on-write: the caller's containers are returned untouched when
|
||||
nothing is stripped, and a shallow copy is returned otherwise, so a fallback
|
||||
leg still sends the full request.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence, Set
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER,
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER,
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER,
|
||||
)
|
||||
|
||||
ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
|
||||
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
|
||||
|
||||
THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
|
||||
THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
|
||||
SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
|
||||
SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
|
||||
SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
|
||||
SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
|
||||
|
||||
_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
|
||||
|
||||
BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
|
||||
{
|
||||
ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER: (THINKING_BLOCK_BINDING,),
|
||||
ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER: (THINKING_DISPLAY_UPDATES,),
|
||||
ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: (SYSTEM_CLEAR_AT,),
|
||||
ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER: (SYSTEM_OUTPUT_CONFIG,),
|
||||
ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
|
||||
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
|
||||
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _betas_unlocking(field: str) -> frozenset[str]:
|
||||
return frozenset(beta for beta, fields in BETA_GATED_FIELDS.items() if field in fields)
|
||||
|
||||
|
||||
_FIELD_BETAS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
|
||||
{
|
||||
field: _betas_unlocking(field) for fields in BETA_GATED_FIELDS.values() for field in fields
|
||||
} # comprehension-ok: flattening the beta-to-fields map into a field-to-betas index
|
||||
)
|
||||
|
||||
|
||||
def field_is_gated(field: str, active_betas: Set[str]) -> bool:
|
||||
"""True when no active beta unlocks ``field``, so the field must be stripped."""
|
||||
return _FIELD_BETAS[field].isdisjoint(active_betas)
|
||||
|
||||
|
||||
_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_SEQUENCE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
|
||||
_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
|
||||
|
||||
|
||||
def _as_mapping(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _MAPPING_ADAPTER.validate_python(value, strict=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _as_sequence(value: object) -> tuple[object, ...] | None:
|
||||
try:
|
||||
return _SEQUENCE_ADAPTER.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_tool_change_block(block: object) -> bool:
|
||||
mapping: Final = _as_mapping(block)
|
||||
block_type: Final = mapping.get("type") if mapping is not None else None
|
||||
return isinstance(block_type, str) and block_type in _TOOL_CHANGE_BLOCK_TYPES
|
||||
|
||||
|
||||
def _header_beta_values(headers: Mapping[str, str]) -> frozenset[str]:
|
||||
joined: Final = ",".join(value for key, value in headers.items() if key.lower() == "anthropic-beta")
|
||||
return frozenset(piece.strip() for piece in joined.split(",") if piece.strip())
|
||||
|
||||
|
||||
def _validated_headers(headers: object) -> Mapping[str, str]:
|
||||
try:
|
||||
return _HEADERS_ADAPTER.validate_python(headers)
|
||||
except ValidationError:
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
def active_beta_values(headers: object, body_betas: object = ()) -> frozenset[str]:
|
||||
"""The ``anthropic-beta`` values a request will send: the comma-joined header (key matched
|
||||
case-insensitively) plus the body's ``anthropic_beta`` list on legs that carry betas in the body."""
|
||||
body_values: Final = _as_sequence(body_betas) or ()
|
||||
body_betas_as_strings: Final = frozenset(beta for beta in body_values if isinstance(beta, str))
|
||||
return _header_beta_values(_validated_headers(headers)) | body_betas_as_strings
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StrippedThinking:
|
||||
thinking: object
|
||||
stripped: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StrippedMessages:
|
||||
messages: Sequence[object]
|
||||
stripped: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StrippedRequest:
|
||||
request: Mapping[str, object]
|
||||
stripped: tuple[str, ...]
|
||||
|
||||
|
||||
def strip_beta_gated_thinking(thinking: object, active_betas: Set[str], *, drop_params: object) -> StrippedThinking:
|
||||
"""Drop ``thinking`` sub-fields whose beta is absent. Returns the same object when nothing is gated."""
|
||||
mapping: Final = _as_mapping(thinking)
|
||||
if mapping is None or not drops_unsupported_params(drop_params):
|
||||
return StrippedThinking(thinking=thinking, stripped=())
|
||||
binding_gated: Final = "block_binding" in mapping and field_is_gated(THINKING_BLOCK_BINDING, active_betas)
|
||||
display_gated: Final = mapping.get("display") == "updates" and field_is_gated(
|
||||
THINKING_DISPLAY_UPDATES, active_betas
|
||||
)
|
||||
if not binding_gated and not display_gated:
|
||||
return StrippedThinking(thinking=thinking, stripped=())
|
||||
gated_keys: Final = frozenset(
|
||||
{*(("block_binding",) if binding_gated else ()), *(("display",) if display_gated else ())}
|
||||
)
|
||||
stripped: Final = (
|
||||
*((THINKING_BLOCK_BINDING,) if binding_gated else ()),
|
||||
*((THINKING_DISPLAY_UPDATES,) if display_gated else ()),
|
||||
)
|
||||
return StrippedThinking(
|
||||
thinking={key: value for key, value in mapping.items() if key not in gated_keys},
|
||||
stripped=stripped,
|
||||
)
|
||||
|
||||
|
||||
def _strip_beta_gated_system_message(message: Mapping[str, object], active_betas: Set[str]) -> StrippedMessages:
|
||||
"""One mid-conversation ``role: system`` message. ``messages`` holds the rewritten message, or nothing
|
||||
when the strip left it with neither content nor ``output_config``, since such a system turn is itself a 400."""
|
||||
clear_at_gated: Final = "clear_at" in message and field_is_gated(SYSTEM_CLEAR_AT, active_betas)
|
||||
output_config_gated: Final = "output_config" in message and field_is_gated(SYSTEM_OUTPUT_CONFIG, active_betas)
|
||||
content: Final = message.get("content")
|
||||
blocks: Final = _as_sequence(content) if not isinstance(content, str) else None
|
||||
tool_changes_gated: Final = (
|
||||
blocks is not None
|
||||
and any(_is_tool_change_block(block) for block in blocks)
|
||||
and field_is_gated(SYSTEM_TOOL_CHANGES, active_betas)
|
||||
)
|
||||
if not clear_at_gated and not output_config_gated and not tool_changes_gated:
|
||||
return StrippedMessages(messages=(message,), stripped=())
|
||||
kept_content: Final[object] = (
|
||||
[block for block in blocks if not _is_tool_change_block(block)]
|
||||
if tool_changes_gated and blocks is not None
|
||||
else content
|
||||
)
|
||||
gated_keys: Final = frozenset(
|
||||
{*(("clear_at",) if clear_at_gated else ()), *(("output_config",) if output_config_gated else ())}
|
||||
)
|
||||
rewritten: Final = {
|
||||
key: (kept_content if key == "content" else value) for key, value in message.items() if key not in gated_keys
|
||||
}
|
||||
stripped: Final = (
|
||||
*((SYSTEM_CLEAR_AT,) if clear_at_gated else ()),
|
||||
*((SYSTEM_OUTPUT_CONFIG,) if output_config_gated else ()),
|
||||
*((SYSTEM_TOOL_CHANGES,) if tool_changes_gated else ()),
|
||||
)
|
||||
if kept_content in (None, "", []) and "output_config" not in rewritten:
|
||||
return StrippedMessages(messages=(), stripped=(*stripped, SYSTEM_MESSAGE_EMPTIED))
|
||||
return StrippedMessages(messages=(rewritten,), stripped=stripped)
|
||||
|
||||
|
||||
def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> StrippedMessages:
|
||||
mapping: Final = _as_mapping(message)
|
||||
if mapping is None or mapping.get("role") != "system":
|
||||
return StrippedMessages(messages=(message,), stripped=())
|
||||
return _strip_beta_gated_system_message(mapping, active_betas)
|
||||
|
||||
|
||||
def _ends_on_assistant(messages: Sequence[object]) -> bool:
|
||||
last: Final = _as_mapping(messages[-1]) if messages else None
|
||||
return last is not None and last.get("role") == "assistant"
|
||||
|
||||
|
||||
def strip_beta_gated_system_messages(
|
||||
messages: Sequence[object], active_betas: Set[str], *, drop_params: object
|
||||
) -> StrippedMessages:
|
||||
"""Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same
|
||||
sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation
|
||||
ending on an assistant turn, which the model would read as a prefill to continue."""
|
||||
if not drops_unsupported_params(drop_params):
|
||||
return StrippedMessages(messages=messages, stripped=())
|
||||
per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages)
|
||||
stripped: Final = tuple(
|
||||
name for result in per_message for name in result.stripped
|
||||
) # comprehension-ok: flattening per-message strip reports
|
||||
if not stripped:
|
||||
return StrippedMessages(messages=messages, stripped=())
|
||||
kept: Final = [
|
||||
message for result in per_message for message in result.messages
|
||||
] # comprehension-ok: flattening per-message results
|
||||
if _ends_on_assistant(kept) and not _ends_on_assistant(messages):
|
||||
return StrippedMessages(messages=messages, stripped=())
|
||||
return StrippedMessages(messages=kept, stripped=stripped)
|
||||
|
||||
|
||||
def strip_beta_gated_request_fields(
|
||||
request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
|
||||
) -> StrippedRequest:
|
||||
"""Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
|
||||
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
|
||||
thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
|
||||
messages: Final = request.get("messages")
|
||||
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
|
||||
system_messages: Final = (
|
||||
strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
|
||||
if message_sequence is not None
|
||||
else StrippedMessages(messages=(), stripped=())
|
||||
)
|
||||
stripped: Final = (*thinking.stripped, *system_messages.stripped)
|
||||
if not stripped:
|
||||
return StrippedRequest(request=request, stripped=())
|
||||
rewritten: Final = {
|
||||
**request,
|
||||
**({"thinking": thinking.thinking} if thinking.stripped else {}),
|
||||
**({"messages": system_messages.messages} if system_messages.stripped else {}),
|
||||
}
|
||||
return StrippedRequest(request=rewritten, stripped=stripped)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StrippedNativeRequest:
|
||||
thinking: object
|
||||
thinking_stripped: bool
|
||||
messages: Sequence[object]
|
||||
messages_stripped: bool
|
||||
stripped: tuple[str, ...]
|
||||
|
||||
|
||||
def unstripped_native_request(optional_params: object, messages: object) -> StrippedNativeRequest:
|
||||
"""The identity result for legs that strip later, once their final ``anthropic_beta`` list exists."""
|
||||
params: Final = _as_mapping(optional_params) or MappingProxyType({})
|
||||
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
|
||||
return StrippedNativeRequest(
|
||||
thinking=params.get("thinking"),
|
||||
thinking_stripped=False,
|
||||
messages=message_sequence if message_sequence is not None else (),
|
||||
messages_stripped=False,
|
||||
stripped=(),
|
||||
)
|
||||
|
||||
|
||||
def strip_beta_gated_native_request(
|
||||
headers: object, optional_params: object, messages: object, *, body_betas: object = (), drop_params: object
|
||||
) -> StrippedNativeRequest:
|
||||
"""The ``/v1/messages`` seam, fed the final (already filtered) headers, the final body ``anthropic_beta``
|
||||
list on legs that carry betas in the body, plus the body parts it may rewrite."""
|
||||
if not drops_unsupported_params(drop_params):
|
||||
return unstripped_native_request(optional_params, messages)
|
||||
active_betas: Final = active_beta_values(headers, body_betas)
|
||||
params: Final = _as_mapping(optional_params) or MappingProxyType({})
|
||||
thinking: Final = strip_beta_gated_thinking(params.get("thinking"), active_betas, drop_params=drop_params)
|
||||
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
|
||||
system_messages: Final = (
|
||||
strip_beta_gated_system_messages(message_sequence, active_betas, drop_params=drop_params)
|
||||
if message_sequence is not None
|
||||
else StrippedMessages(messages=(), stripped=())
|
||||
)
|
||||
return StrippedNativeRequest(
|
||||
thinking=thinking.thinking,
|
||||
thinking_stripped=bool(thinking.stripped),
|
||||
messages=system_messages.messages,
|
||||
messages_stripped=bool(system_messages.stripped),
|
||||
stripped=(*thinking.stripped, *system_messages.stripped),
|
||||
)
|
||||
|
||||
|
||||
def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None:
|
||||
if not stripped:
|
||||
return
|
||||
described: Final = ", ".join(
|
||||
f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})"
|
||||
if field in _FIELD_BETAS
|
||||
else field
|
||||
for field in sorted(set(stripped))
|
||||
)
|
||||
verbose_logger.warning(
|
||||
"Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on "
|
||||
"this request, either since this provider leg does not send that anthropic-beta (see "
|
||||
"anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user "
|
||||
"messages.",
|
||||
described,
|
||||
model,
|
||||
provider,
|
||||
)
|
||||
|
||||
|
||||
def strip_beta_gated_request_body(
|
||||
request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str
|
||||
) -> Mapping[str, object]:
|
||||
gated: Final = strip_beta_gated_request_fields(
|
||||
request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params
|
||||
)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=provider)
|
||||
return gated.request
|
||||
|
|
@ -248,6 +248,7 @@
|
|||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
|
||||
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
|
||||
"web-fetch-2025-09-10": null,
|
||||
"web-search-2025-03-05": "web-search-2025-03-05"
|
||||
|
|
|
|||
|
|
@ -241,6 +241,13 @@ def get_provider_name(provider: str) -> str:
|
|||
return aliases.get(provider, provider)
|
||||
|
||||
|
||||
def has_beta_headers_config(provider: str) -> bool:
|
||||
config: Final = _bundled_beta_headers_config()
|
||||
aliases: Final = config.get("provider_aliases", {})
|
||||
canonical_provider: Final = (aliases.get(provider) if isinstance(aliases, Mapping) else None) or provider
|
||||
return canonical_provider in config
|
||||
|
||||
|
||||
def filter_and_transform_beta_headers(
|
||||
beta_headers: list[str],
|
||||
provider: str,
|
||||
|
|
@ -397,7 +404,7 @@ def update_request_with_filtered_beta(
|
|||
headers: dict,
|
||||
request_data: dict,
|
||||
provider: str,
|
||||
) -> tuple[dict, dict]:
|
||||
) -> tuple[dict[str, str], dict[str, object]]:
|
||||
"""
|
||||
Update both headers and request body beta fields based on provider support.
|
||||
Modifies both dicts in place and returns them.
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ def resolve_embedding_max_input_tokens(
|
|||
return configured_max_input_tokens
|
||||
if router is None:
|
||||
return None
|
||||
deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
|
||||
deployment_max_input_tokens, _ = router.registry().group(embedding_model, field="token_limits")
|
||||
return deployment_max_input_tokens
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -455,17 +455,33 @@ class Cache:
|
|||
3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component)
|
||||
"""
|
||||
litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
|
||||
metadata_sources: Final[tuple[dict, ...]] = (
|
||||
kwargs.get("metadata") or {},
|
||||
kwargs.get("litellm_metadata") or {},
|
||||
litellm_params.get("metadata") or {},
|
||||
litellm_params.get("litellm_metadata") or {},
|
||||
metadata_sources: Final = cast( # cast-ok: each source is the untyped request's metadata dict or empty
|
||||
"tuple[dict[str, object], dict[str, object], dict[str, object], dict[str, object]]",
|
||||
(
|
||||
kwargs.get("metadata") or {},
|
||||
kwargs.get("litellm_metadata") or {},
|
||||
litellm_params.get("metadata") or {},
|
||||
litellm_params.get("litellm_metadata") or {},
|
||||
),
|
||||
)
|
||||
model_group: Final[str | None] = next(
|
||||
(source["model_group"] for source in metadata_sources if source.get("model_group")), None
|
||||
metadata, litellm_metadata, params_metadata, params_litellm_metadata = metadata_sources
|
||||
call_model_info: Final[object] = kwargs.get("model_info")
|
||||
routed_source: Final = next(
|
||||
(
|
||||
source
|
||||
for source in (litellm_metadata, metadata, params_litellm_metadata, params_metadata)
|
||||
if call_model_info and source.get("model_group") and source.get("model_info") == call_model_info
|
||||
),
|
||||
None,
|
||||
)
|
||||
model_group: Final = (
|
||||
str(routed_source["model_group"])
|
||||
if routed_source is not None
|
||||
else next((str(source["model_group"]) for source in metadata_sources if source.get("model_group")), None)
|
||||
)
|
||||
caching_group_sources: Final = (routed_source,) if routed_source is not None else metadata_sources
|
||||
caching_group: Final = next(
|
||||
(group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None
|
||||
(group for source in caching_group_sources if (group := self._get_caching_group(source, model_group))), None
|
||||
)
|
||||
model_id_suffix: Final = (
|
||||
f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else ""
|
||||
|
|
|
|||
|
|
@ -830,6 +830,13 @@ class LLMCachingHandler:
|
|||
if is_internal_call.get():
|
||||
return
|
||||
|
||||
if logging_obj.defer_async_logging and not logging_obj.stream:
|
||||
from litellm.utils import dispatch_success_logging
|
||||
|
||||
logging_obj.model_call_details["cache_hit"] = cache_hit # rebind-ok: capture cache status before deferring
|
||||
dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
|
||||
return
|
||||
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
|
|||
close them, and each bucket is ordered by deadline, so a reap walks the entries
|
||||
that are due rather than the whole queue.
|
||||
|
||||
A bucket whose loop has been collected is dropped, and its clients are left to the
|
||||
collector, as they are when the queue is full.
|
||||
|
||||
The queue holds its clients weakly, so waiting out a grace window never keeps
|
||||
alive anything the collector would have reclaimed first.
|
||||
"""
|
||||
|
|
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -44,6 +48,8 @@ from collections.abc import Awaitable, Callable, Iterator
|
|||
from dataclasses import dataclass, replace
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.constants import (
|
||||
EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
|
||||
EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
|
||||
|
|
@ -53,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
|
|||
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
|
||||
|
||||
_BucketKey = str | int
|
||||
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
|
||||
_LOOP_TOKEN_COUNTER: Final = itertools.count()
|
||||
_LOOP_TOKEN_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -63,12 +72,12 @@ class _PendingClose:
|
|||
collector would otherwise have reclaimed first.
|
||||
|
||||
``needs_loop`` is set for a client whose close is a coroutine; those can only
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_id``.
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_token``.
|
||||
A client that closes synchronously carries neither constraint.
|
||||
"""
|
||||
|
||||
client_ref: "weakref.ref[object]"
|
||||
loop_id: int | None
|
||||
loop_token: int | None
|
||||
needs_loop: bool
|
||||
close_after: float
|
||||
|
||||
|
|
@ -77,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
|
|||
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
|
||||
if not pending.needs_loop:
|
||||
return _CLOSABLE_ANYWHERE
|
||||
if pending.loop_id is None:
|
||||
if pending.loop_token is None:
|
||||
return _CLOSABLE_ON_ANY_LOOP
|
||||
return pending.loop_id
|
||||
return pending.loop_token
|
||||
|
||||
|
||||
def _running_loop_id() -> int | None:
|
||||
def running_loop_token() -> int | None:
|
||||
try:
|
||||
return id(asyncio.get_running_loop())
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
try:
|
||||
token: Final[int | None] = _LOOP_TOKENS.get(loop)
|
||||
if token is not None:
|
||||
return token
|
||||
with _LOOP_TOKEN_LOCK:
|
||||
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
|
||||
except TypeError:
|
||||
return id(loop)
|
||||
|
||||
|
||||
def _close_function(client: object) -> Callable[[], object] | None:
|
||||
|
|
@ -94,13 +111,24 @@ def _close_function(client: object) -> Callable[[], object] | None:
|
|||
return close_fn
|
||||
|
||||
|
||||
def _transport_of(client: object) -> object:
|
||||
"""The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
|
||||
def _mounted_transports(holder: object) -> tuple[object, ...]:
|
||||
mounts: Final[object] = getattr(holder, "_mounts", None)
|
||||
if not isinstance(mounts, dict):
|
||||
return ()
|
||||
return tuple(
|
||||
mounted
|
||||
for mounted in mounts.values() # pyright: ignore[reportUnknownVariableType] # untyped httpx mount map
|
||||
if isinstance(mounted, (httpx.AsyncBaseTransport, httpx.BaseTransport))
|
||||
)
|
||||
|
||||
|
||||
def _transports_of(client: object) -> tuple[object, ...]:
|
||||
"""The httpx transports behind an SDK wrapper, a litellm handler, or a bare client, proxy mounts included."""
|
||||
for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
|
||||
transport: object = getattr(holder, "_transport", None)
|
||||
if transport is not None:
|
||||
return transport
|
||||
return None
|
||||
return (transport, *_mounted_transports(holder))
|
||||
return ()
|
||||
|
||||
|
||||
def _connection_is_idle(connection: object) -> bool:
|
||||
|
|
@ -123,6 +151,17 @@ def _pool_has_busy_connection(transport: object) -> bool | None:
|
|||
)
|
||||
|
||||
|
||||
def _transport_in_flight(transport: object) -> bool:
|
||||
spread_in_flight: Final[object] = getattr(transport, "has_request_in_flight", None)
|
||||
if callable(spread_in_flight):
|
||||
return bool(spread_in_flight())
|
||||
pooled_busy: Final = _pool_has_busy_connection(transport)
|
||||
if pooled_busy is not None:
|
||||
return pooled_busy
|
||||
session: Final[object] = getattr(transport, "client", None)
|
||||
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
|
||||
|
||||
|
||||
def _has_connection_in_flight(client: object) -> bool:
|
||||
"""Whether the client is servicing a request right now.
|
||||
|
||||
|
|
@ -138,12 +177,7 @@ def _has_connection_in_flight(client: object) -> bool:
|
|||
try:
|
||||
if getattr(getattr(client, "connection_pool", None), "_in_use_connections", None):
|
||||
return True
|
||||
transport: Final = _transport_of(client)
|
||||
pooled_busy: Final = _pool_has_busy_connection(transport)
|
||||
if pooled_busy is not None:
|
||||
return pooled_busy
|
||||
session: Final[object] = getattr(transport, "client", None)
|
||||
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
|
||||
return any(_transport_in_flight(transport) for transport in _transports_of(client))
|
||||
except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle
|
||||
return False
|
||||
|
||||
|
|
@ -167,6 +201,8 @@ class EvictedClientCloser:
|
|||
self._clock = clock
|
||||
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
|
||||
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
|
||||
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
|
||||
self._pending_count = 0
|
||||
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
|
||||
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
|
||||
|
|
@ -197,13 +233,19 @@ class EvictedClientCloser:
|
|||
close_fn: Final = _close_function(client)
|
||||
if close_fn is None:
|
||||
return
|
||||
loop_token: Final = running_loop_token()
|
||||
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
self._discard_dead_loops()
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
if needs_loop and loop_token is not None:
|
||||
self._watch_loop(loop_token)
|
||||
self._enqueue(
|
||||
_PendingClose(
|
||||
client_ref=weakref.ref(client),
|
||||
loop_id=_running_loop_id(),
|
||||
needs_loop=inspect.iscoroutinefunction(close_fn),
|
||||
loop_token=loop_token,
|
||||
needs_loop=needs_loop,
|
||||
close_after=self._clock() + self._grace_seconds,
|
||||
)
|
||||
)
|
||||
|
|
@ -217,7 +259,7 @@ class EvictedClientCloser:
|
|||
if not self._pending_count:
|
||||
return
|
||||
now: Final = self._clock()
|
||||
for pending in self._take_due(_running_loop_id(), now):
|
||||
for pending in self._take_due(running_loop_token(), now):
|
||||
client = pending.client_ref()
|
||||
if client is None:
|
||||
continue
|
||||
|
|
@ -230,6 +272,29 @@ class EvictedClientCloser:
|
|||
def pending_count(self) -> int:
|
||||
return self._pending_count
|
||||
|
||||
def _watch_loop(self, loop_token: int) -> None:
|
||||
if loop_token in self._watched_loop_tokens:
|
||||
return
|
||||
try:
|
||||
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
|
||||
except TypeError:
|
||||
return
|
||||
self._watched_loop_tokens.add(loop_token)
|
||||
|
||||
def _discard_dead_loops(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
|
||||
def _discard_dead_loops_locked(self) -> None:
|
||||
while self._dead_loop_tokens:
|
||||
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
|
||||
|
||||
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
|
||||
self._watched_loop_tokens.discard(loop_token)
|
||||
bucket: Final = self._buckets.pop(loop_token, None)
|
||||
if bucket is not None:
|
||||
self._pending_count -= len(bucket)
|
||||
|
||||
def _enqueue(self, pending: _PendingClose) -> None:
|
||||
"""Append to the entry's bucket, dropping any dead entries it queues behind.
|
||||
|
||||
|
|
@ -238,6 +303,7 @@ class EvictedClientCloser:
|
|||
the front rather than having to be searched for.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
|
||||
while bucket and bucket[0].client_ref() is None:
|
||||
bucket.popleft()
|
||||
|
|
@ -245,9 +311,12 @@ class EvictedClientCloser:
|
|||
bucket.append(pending)
|
||||
self._pending_count += 1
|
||||
|
||||
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
|
||||
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (
|
||||
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
|
||||
)
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
|
||||
|
||||
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
|
|
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
|
|||
self.evicted_client_closer.schedule(evicted)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
def update_cache_key_with_event_loop(self, key):
|
||||
def update_cache_key_with_event_loop(self, key: str) -> str:
|
||||
"""
|
||||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
If none, use the key as is.
|
||||
"""
|
||||
try:
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
stringified_event_loop: Final = str(id(event_loop))
|
||||
return f"{key}-{stringified_event_loop}"
|
||||
except RuntimeError: # handle no current running event loop
|
||||
return key
|
||||
token: Final = running_loop_token()
|
||||
return f"{key}-{token}" if token is not None else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
def _scoped_key(self, key: str, loop_bound: bool) -> str:
|
||||
return self.update_cache_key_with_event_loop(key) if loop_bound else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
|
||||
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
|
||||
if litellm_owned_client:
|
||||
self.evicted_client_closer.mark_owned(value)
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
return super().set_cache(key, value, **kwargs)
|
||||
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
|
||||
|
||||
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
if litellm_owned_client:
|
||||
|
|
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
|
|||
key = self.update_cache_key_with_event_loop(key)
|
||||
return await super().async_set_cache(key, value, **kwargs)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
|
||||
scoped_key: Final = self._scoped_key(key, loop_bound)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
return super().get_cache(key, **kwargs)
|
||||
return super().get_cache(scoped_key, **kwargs)
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
async def async_get_cache(self, key: str, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
|
||||
return await super().async_get_cache(key, **kwargs)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import asyncio
|
|||
import json
|
||||
import os
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
|
@ -292,10 +292,7 @@ class QdrantSemanticCache(BaseCache):
|
|||
prompt: Final = get_semantic_cache_prompt_from_messages(messages)
|
||||
|
||||
# create an embedding for prompt
|
||||
embedding_response: Final = cast(
|
||||
EmbeddingResponse,
|
||||
self._get_embedding(prompt, metadata=kwargs.get("metadata")),
|
||||
)
|
||||
embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
|
||||
|
||||
# get the embedding
|
||||
embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])
|
||||
|
|
@ -331,10 +328,7 @@ class QdrantSemanticCache(BaseCache):
|
|||
prompt: Final = get_semantic_cache_prompt_from_messages(messages)
|
||||
|
||||
# convert to embedding
|
||||
embedding_response: Final = cast(
|
||||
EmbeddingResponse,
|
||||
self._get_embedding(prompt, metadata=kwargs.get("metadata")),
|
||||
)
|
||||
embedding_response: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
|
||||
|
||||
# get the embedding
|
||||
embedding: Final = _EMBEDDING_VECTOR.validate_python(embedding_response["data"][0]["embedding"])
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ import asyncio
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
|
|
@ -38,6 +40,11 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
|
||||
_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
|
||||
str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
class RedisSemanticCache(BaseCache):
|
||||
"""
|
||||
Redis-backed semantic cache for LLM responses.
|
||||
|
|
@ -380,7 +387,7 @@ class RedisSemanticCache(BaseCache):
|
|||
)
|
||||
return embedding_response["data"][0]["embedding"]
|
||||
|
||||
def _get_cache_logic(self, cached_response: Any) -> object:
|
||||
def _get_cache_logic(self, cached_response: str | bytes | None) -> object:
|
||||
"""
|
||||
Process the cached response to prepare it for use.
|
||||
|
||||
|
|
@ -391,24 +398,23 @@ class RedisSemanticCache(BaseCache):
|
|||
The processed cache response, or None if input was None
|
||||
"""
|
||||
if cached_response is None:
|
||||
return cached_response
|
||||
return None
|
||||
|
||||
# Convert bytes to string if needed
|
||||
if isinstance(cached_response, bytes):
|
||||
cached_response = cached_response.decode("utf-8")
|
||||
response_text: Final = (
|
||||
cached_response.decode("utf-8") if isinstance(cached_response, bytes) else cached_response
|
||||
)
|
||||
|
||||
# Convert string representation to Python object
|
||||
try:
|
||||
cached_response = json.loads(cached_response)
|
||||
return json.loads(response_text)
|
||||
except json.JSONDecodeError:
|
||||
try:
|
||||
cached_response = ast.literal_eval(cached_response)
|
||||
return ast.literal_eval(response_text)
|
||||
except (ValueError, SyntaxError) as e:
|
||||
print_verbose(f"Error parsing cached response: {e}")
|
||||
return None
|
||||
|
||||
return cached_response
|
||||
|
||||
def set_cache(self, key: str, value: object, **kwargs) -> None:
|
||||
"""
|
||||
Store a value in the semantic cache.
|
||||
|
|
@ -494,7 +500,7 @@ class RedisSemanticCache(BaseCache):
|
|||
similarity: Final = 1 - vector_distance
|
||||
|
||||
cached_prompt: Final = cache_hit["prompt"]
|
||||
cached_response: Final = cache_hit["response"]
|
||||
cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
|
||||
|
||||
# update kwargs["metadata"] with similarity, don't rewrite the original metadata
|
||||
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
|
||||
|
|
@ -645,7 +651,7 @@ class RedisSemanticCache(BaseCache):
|
|||
similarity: Final = 1 - vector_distance
|
||||
|
||||
cached_prompt: Final = cache_hit["prompt"]
|
||||
cached_response: Final = cache_hit["response"]
|
||||
cached_response: Final = _CACHE_RESPONSE_ADAPTER.validate_python(cache_hit["response"])
|
||||
|
||||
# update kwargs["metadata"] with similarity, don't rewrite the original metadata
|
||||
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
|
||||
|
|
|
|||
|
|
@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
replays_openai_history=custom_llm_provider == "openai",
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
replays_openai_history=custom_llm_provider == "openai",
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,432 @@
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, pairwise
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
REPLAY_FIELD: Final = "openai_responses_replay"
|
||||
PHASE_FIELD: Final = "phase"
|
||||
|
||||
Phase: TypeAlias = Literal["commentary", "final_answer"]
|
||||
|
||||
|
||||
class _Frozen(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class _OutputText(_Frozen):
|
||||
type: Literal["output_text"]
|
||||
text: str
|
||||
|
||||
|
||||
class _OutputMessage(_Frozen):
|
||||
type: Literal["message"]
|
||||
role: Literal["assistant"]
|
||||
content: tuple[_OutputText, ...]
|
||||
phase: Phase | None = None
|
||||
|
||||
|
||||
class _OutputReasoning(_Frozen):
|
||||
type: Literal["reasoning"]
|
||||
id: str
|
||||
|
||||
|
||||
class _OutputToolCall(_Frozen):
|
||||
type: Literal["function_call", "custom_tool_call"]
|
||||
call_id: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
class _OutputToolSearch(_Frozen):
|
||||
type: Literal["tool_search_call", "tool_search_output"]
|
||||
|
||||
|
||||
_OutputItem: TypeAlias = Annotated[
|
||||
_OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type")
|
||||
]
|
||||
_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem)
|
||||
|
||||
|
||||
class _AnyMessage(_Frozen):
|
||||
type: Literal["message"]
|
||||
|
||||
|
||||
class _MessageEntry(_Frozen):
|
||||
type: Literal["message"]
|
||||
parts: tuple[Annotated[int, Field(ge=0)], ...]
|
||||
phase: Phase | None = None
|
||||
|
||||
|
||||
class _IndexedEntry(_Frozen):
|
||||
type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"]
|
||||
index: Annotated[int, Field(ge=0)]
|
||||
|
||||
|
||||
_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")]
|
||||
|
||||
|
||||
class _Replay(_Frozen):
|
||||
version: Literal[1]
|
||||
items: tuple[_Entry, ...]
|
||||
checksum: str
|
||||
|
||||
|
||||
class _Namespaced(_Frozen):
|
||||
namespace: str | None = None
|
||||
|
||||
|
||||
class _ToolFunction(_Frozen):
|
||||
name: str = ""
|
||||
arguments: str = ""
|
||||
provider_specific_fields: _Namespaced | None = None
|
||||
|
||||
|
||||
class _ToolCall(_Frozen):
|
||||
id: str
|
||||
function: _ToolFunction
|
||||
provider_specific_fields: _Namespaced | None = None
|
||||
|
||||
|
||||
class _SummaryText(_Frozen):
|
||||
type: str = "summary_text"
|
||||
text: str = ""
|
||||
|
||||
|
||||
class _Reasoning(_Frozen):
|
||||
id: str | None = None
|
||||
encrypted_content: str | None = None
|
||||
summary: tuple[_SummaryText, ...] | None = None
|
||||
|
||||
|
||||
class _ProviderFields(_Frozen):
|
||||
phase: Phase | None = None
|
||||
tool_search_items: tuple[Mapping[str, object], ...] | None = None
|
||||
openai_responses_replay: object = None
|
||||
|
||||
|
||||
class _ChatMessage(_Frozen):
|
||||
role: str | None = None
|
||||
content: str | None = None
|
||||
tool_calls: tuple[_ToolCall, ...] | None = None
|
||||
reasoning_items: tuple[_Reasoning, ...] | None = None
|
||||
phase: Phase | None = None
|
||||
provider_specific_fields: _ProviderFields | None = None
|
||||
|
||||
|
||||
class _PhaseFields(_Frozen):
|
||||
phase: Phase | None = None
|
||||
|
||||
|
||||
class _RequestedPhaseFields(_Frozen):
|
||||
phase: object = None
|
||||
|
||||
|
||||
class _RequestedPhase(_Frozen):
|
||||
phase: object = None
|
||||
provider_specific_fields: _RequestedPhaseFields | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnknownPhase:
|
||||
value: object
|
||||
|
||||
|
||||
_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase)
|
||||
|
||||
|
||||
class _RetainedFields(_PhaseFields):
|
||||
openai_responses_replay: object = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayedTurn:
|
||||
items: tuple[Mapping[str, object], ...]
|
||||
custom_call_ids: frozenset[str]
|
||||
|
||||
|
||||
def _as_json(value: object) -> object:
|
||||
return value.model_dump() if isinstance(value, BaseModel) else value
|
||||
|
||||
|
||||
def _parsed_output_item(item: object) -> _OutputItem | None:
|
||||
try:
|
||||
return _OUTPUT_ITEM.validate_python(_as_json(item))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_message(item: object) -> bool:
|
||||
try:
|
||||
_AnyMessage.model_validate(_as_json(item))
|
||||
except ValidationError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _chat_message(message: object) -> _ChatMessage | None:
|
||||
try:
|
||||
return _ChatMessage.model_validate(_as_json(message))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def requested_phase(message: object) -> Phase | UnknownPhase | None:
|
||||
try:
|
||||
requested: Final = _RequestedPhase.model_validate(_as_json(message))
|
||||
except ValidationError:
|
||||
return None
|
||||
fields: Final = requested.provider_specific_fields
|
||||
value: Final = requested.phase or (fields.phase if fields else None)
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return _PHASE.validate_python(value)
|
||||
except ValidationError:
|
||||
return UnknownPhase(value)
|
||||
|
||||
|
||||
def reasoning_input_item(
|
||||
item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "reasoning",
|
||||
**({"id": item_id} if item_id else {}),
|
||||
"summary": list(summary or ()),
|
||||
**({"encrypted_content": encrypted_content} if encrypted_content else {}),
|
||||
}
|
||||
|
||||
|
||||
def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
retained: Final = _RetainedFields.model_validate(provider_specific_fields)
|
||||
except ValidationError:
|
||||
return None
|
||||
return retained.model_dump(exclude_none=True) or None
|
||||
|
||||
|
||||
def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str:
|
||||
parsed: Final = map(_parsed_output_item, output_items)
|
||||
messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage))
|
||||
return "".join(_texts(messages[:1] if first_message_only else messages))
|
||||
|
||||
|
||||
def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = message.provider_specific_fields
|
||||
items: Final = (fields.tool_search_items if fields else None) or ()
|
||||
return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output"))
|
||||
|
||||
|
||||
def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str:
|
||||
fields: Final = message.provider_specific_fields
|
||||
bound: Final = {
|
||||
"content": message.content or "",
|
||||
"tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()],
|
||||
"reasoning": [
|
||||
[item.id, item.encrypted_content, [part.text for part in item.summary or ()]]
|
||||
for item in message.reasoning_items or ()
|
||||
],
|
||||
"tool_search_items": list((fields.tool_search_items if fields else None) or ()),
|
||||
"phase": message.phase or (fields.phase if fields else None),
|
||||
"items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries],
|
||||
}
|
||||
encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest()
|
||||
|
||||
|
||||
def _kind(item: _OutputItem) -> str:
|
||||
match item:
|
||||
case _OutputToolCall():
|
||||
return "tool_call"
|
||||
case _OutputToolSearch():
|
||||
return "tool_search"
|
||||
case _OutputMessage() | _OutputReasoning():
|
||||
return item.type
|
||||
|
||||
|
||||
def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry:
|
||||
item: Final = items[position]
|
||||
if isinstance(item, _OutputMessage):
|
||||
return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase)
|
||||
ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item))
|
||||
return _IndexedEntry(type=item.type, index=ordinal)
|
||||
|
||||
|
||||
def _texts(items: Sequence[_OutputItem]) -> Iterator[str]:
|
||||
for item in items:
|
||||
if isinstance(item, _OutputMessage):
|
||||
yield from (part.text for part in item.content)
|
||||
|
||||
|
||||
def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
|
||||
reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning))
|
||||
calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall))
|
||||
tool_calls: Final = message.tool_calls or ()
|
||||
searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch))
|
||||
return (
|
||||
"".join(_texts(items)) == (message.content or "")
|
||||
and reasoning_ids == tuple(item.id for item in message.reasoning_items or ())
|
||||
and len(calls) == len(tool_calls)
|
||||
and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls))
|
||||
and searches == len(_tool_search_entries(message))
|
||||
)
|
||||
|
||||
|
||||
def _content_phase(
|
||||
output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool
|
||||
) -> Phase | None:
|
||||
messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw))
|
||||
if not messages:
|
||||
return None
|
||||
content_message: Final = messages[0] if first_message_only else messages[-1]
|
||||
return content_message.phase if isinstance(content_message, _OutputMessage) else None
|
||||
|
||||
|
||||
def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage:
|
||||
if phase is None:
|
||||
return message
|
||||
fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase})
|
||||
return message.model_copy(update={"provider_specific_fields": fields})
|
||||
|
||||
|
||||
def replay_fields(
|
||||
output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool
|
||||
) -> Mapping[str, object]:
|
||||
parsed: Final = tuple(map(_parsed_output_item, output_items))
|
||||
phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only)
|
||||
phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {})
|
||||
items: Final = tuple(item for item in parsed if item is not None)
|
||||
parsed_message: Final = _chat_message(message)
|
||||
chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None
|
||||
if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat):
|
||||
return phase_fields
|
||||
entries: Final = tuple(_entry(items, position) for position in range(len(items)))
|
||||
replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries))
|
||||
return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)})
|
||||
|
||||
|
||||
def _stored_replay(message: _ChatMessage) -> _Replay | None:
|
||||
fields: Final = message.provider_specific_fields
|
||||
if fields is None or fields.openai_responses_replay is None:
|
||||
return None
|
||||
try:
|
||||
return _Replay.model_validate(fields.openai_responses_replay)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]:
|
||||
lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries))
|
||||
bounds: Final = tuple(accumulate(lengths, initial=0))
|
||||
parts: Final = tuple(content[start:end] for start, end in pairwise(bounds))
|
||||
groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0))
|
||||
return tuple(parts[start:end] for start, end in pairwise(groups))
|
||||
|
||||
|
||||
def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": text} for text in texts],
|
||||
**({PHASE_FIELD: phase} if phase else {}),
|
||||
}
|
||||
|
||||
|
||||
def _function_call_item(call: _ToolCall) -> Mapping[str, object]:
|
||||
namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields
|
||||
namespace: Final = namespaced.namespace if namespaced else None
|
||||
return {
|
||||
"type": "function_call",
|
||||
"call_id": call.id,
|
||||
"name": call.function.name,
|
||||
"arguments": call.function.arguments,
|
||||
**({"namespace": namespace} if namespace else {}),
|
||||
}
|
||||
|
||||
|
||||
def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "custom_tool_call",
|
||||
"call_id": call.id,
|
||||
"name": call.function.name,
|
||||
"input": call.function.arguments,
|
||||
}
|
||||
|
||||
|
||||
def _reasoning_item(item: _Reasoning) -> Mapping[str, object]:
|
||||
summary: Final = tuple(part.model_dump() for part in item.summary or ())
|
||||
return reasoning_input_item(item.id, summary, item.encrypted_content)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Resolved:
|
||||
message: _ChatMessage
|
||||
texts: Mapping[int, tuple[str, ...]]
|
||||
tool_searches: tuple[Mapping[str, object], ...]
|
||||
|
||||
|
||||
def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]:
|
||||
if isinstance(entry, _MessageEntry):
|
||||
return _message_item(resolved.texts[position], entry.phase)
|
||||
tool_calls: Final = resolved.message.tool_calls or ()
|
||||
match entry.type:
|
||||
case "reasoning":
|
||||
return _reasoning_item((resolved.message.reasoning_items or ())[entry.index])
|
||||
case "function_call":
|
||||
return _function_call_item(tool_calls[entry.index])
|
||||
case "custom_tool_call":
|
||||
return _custom_tool_call_item(tool_calls[entry.index])
|
||||
case "tool_search_call" | "tool_search_output":
|
||||
return dict(resolved.tool_searches[entry.index])
|
||||
|
||||
|
||||
def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
|
||||
tool_calls: Final = len(message.tool_calls or ())
|
||||
tool_searches: Final = len(_tool_search_entries(message))
|
||||
sizes: Final = MappingProxyType(
|
||||
{
|
||||
"reasoning": len(message.reasoning_items or ()),
|
||||
"function_call": tool_calls,
|
||||
"custom_tool_call": tool_calls,
|
||||
"tool_search_call": tool_searches,
|
||||
"tool_search_output": tool_searches,
|
||||
}
|
||||
)
|
||||
parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
|
||||
indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
|
||||
return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
|
||||
|
||||
|
||||
def _resolve(
|
||||
message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
|
||||
) -> ReplayedTurn | None:
|
||||
indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry))
|
||||
if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed):
|
||||
return None
|
||||
messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry))
|
||||
calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call"))
|
||||
texts: Final = _message_texts(messages, message.content or "")
|
||||
positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry))
|
||||
resolved: Final = _Resolved(
|
||||
message=message,
|
||||
texts=MappingProxyType(dict(zip(positions, texts, strict=True))),
|
||||
tool_searches=_tool_search_entries(message),
|
||||
)
|
||||
tool_calls: Final = message.tool_calls or ()
|
||||
return ReplayedTurn(
|
||||
items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)),
|
||||
custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"),
|
||||
)
|
||||
|
||||
|
||||
def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None:
|
||||
chat: Final = _chat_message(message)
|
||||
stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
|
||||
if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
|
||||
return None
|
||||
if not _fits(chat, stored.items):
|
||||
return None
|
||||
return _resolve(chat, stored.items, replay_tool_search)
|
||||
|
|
@ -24,9 +24,19 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.completion_extras.litellm_responses_transformation.history import (
|
||||
PHASE_FIELD,
|
||||
UnknownPhase,
|
||||
output_message_text,
|
||||
reasoning_input_item,
|
||||
replay_fields,
|
||||
replay_items,
|
||||
requested_phase,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
)
|
||||
|
|
@ -52,7 +62,6 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
PromptCacheBreakpoint,
|
||||
Reasoning,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
|
|
@ -490,7 +499,7 @@ def _fold_choices_into_one(
|
|||
choices: Sequence["Choices"],
|
||||
tool_calls: Sequence[Mapping[str, object]],
|
||||
trailing_reasoning_content: str | None,
|
||||
trailing_reasoning_item: _BuiltReasoningItem | None,
|
||||
trailing_reasoning_items: Sequence[_BuiltReasoningItem],
|
||||
tool_search_items: Sequence[Mapping[str, object]],
|
||||
structured_output: bool = False,
|
||||
) -> "Choices":
|
||||
|
|
@ -509,7 +518,7 @@ def _fold_choices_into_one(
|
|||
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
|
||||
reasoning_items: Final = (
|
||||
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
|
||||
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
|
||||
*trailing_reasoning_items,
|
||||
)
|
||||
return Choices(
|
||||
message=Message(
|
||||
|
|
@ -526,6 +535,61 @@ def _fold_choices_into_one(
|
|||
)
|
||||
|
||||
|
||||
def _with_replay_fields(
|
||||
choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool
|
||||
) -> "Choices":
|
||||
fields: Final = {
|
||||
**(choice.message.provider_specific_fields or {}),
|
||||
**replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output),
|
||||
}
|
||||
message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None})
|
||||
return choice.model_copy(update={"message": message})
|
||||
|
||||
|
||||
_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
|
||||
|
||||
def completed_stream_history(
|
||||
output_payload: object,
|
||||
tool_search_fields: Mapping[str, object] | None,
|
||||
*,
|
||||
completed: bool,
|
||||
structured_output: bool,
|
||||
) -> dict[str, object] | None: # mutable-ok: Delta.provider_specific_fields is typed as a dict
|
||||
items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload)
|
||||
calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call"))
|
||||
streamed_message: Final = {
|
||||
"role": "assistant",
|
||||
"content": output_message_text(items, first_message_only=structured_output),
|
||||
"tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)],
|
||||
"reasoning_items": list(_reasoning_items_from_output_items(items)),
|
||||
"provider_specific_fields": tool_search_fields,
|
||||
}
|
||||
fields: Final = {
|
||||
**(tool_search_fields or {}),
|
||||
**replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output),
|
||||
}
|
||||
return fields or None
|
||||
|
||||
|
||||
def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]:
|
||||
match requested_phase(msg) if replay_openai_history else None:
|
||||
case None:
|
||||
return {}
|
||||
case UnknownPhase(value=value) if drop_params:
|
||||
verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value)
|
||||
return {}
|
||||
case UnknownPhase(value=value):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"phase must be 'commentary' or 'final_answer', got {value!r}. "
|
||||
"To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
|
||||
)
|
||||
case phase:
|
||||
return {PHASE_FIELD: phase}
|
||||
|
||||
|
||||
def _responses_event_type(chunk: Mapping[str, object]) -> object:
|
||||
event_type: Final = chunk.get("type")
|
||||
return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type
|
||||
|
|
@ -567,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
|
|||
return "length"
|
||||
|
||||
|
||||
_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
|
||||
|
||||
|
||||
def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
|
||||
if marker is None or not drop_params:
|
||||
return marker
|
||||
try:
|
||||
return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
|
||||
except ValidationError:
|
||||
verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
|
||||
return None
|
||||
|
||||
|
||||
def _input_file_from_file_value(file_value: object) -> dict[str, object]:
|
||||
if not isinstance(file_value, dict):
|
||||
return {"type": "input_file"}
|
||||
|
|
@ -667,15 +718,7 @@ def _reasoning_item_to_response_input(
|
|||
An item without an id is sent without one: the Responses API accepts that and
|
||||
verifies the encrypted content on its own, while it rejects any id it did not mint.
|
||||
"""
|
||||
item_id: Final = r_item.get("id")
|
||||
encrypted_content: Final = r_item.get("encrypted_content")
|
||||
return {
|
||||
"type": "reasoning",
|
||||
**({"id": item_id} if item_id else {}),
|
||||
# summary is always required by the Responses API, even when empty
|
||||
"summary": r_item.get("summary") or [],
|
||||
**({"encrypted_content": encrypted_content} if encrypted_content else {}),
|
||||
}
|
||||
return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content")))
|
||||
|
||||
|
||||
def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
|
||||
|
|
@ -804,12 +847,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
keep_prompt_cache_breakpoints: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
replay_openai_history: bool = False,
|
||||
) -> tuple[list[object], str | None]:
|
||||
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
|
||||
messages,
|
||||
drop_params=drop_params,
|
||||
replay_tool_search=replay_tool_search,
|
||||
include_namespace=include_namespace,
|
||||
replay_openai_history=replay_openai_history,
|
||||
)
|
||||
return (
|
||||
converted_input_items
|
||||
|
|
@ -825,10 +870,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
drop_params: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
replay_openai_history: bool = False,
|
||||
) -> tuple[list[object], str | None]:
|
||||
input_items: Final[list[object]] = []
|
||||
instructions: str | None = None
|
||||
custom_tool_call_ids: Final = frozenset(
|
||||
replayed_turns: Final = tuple(
|
||||
replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None
|
||||
for msg in messages
|
||||
)
|
||||
replayed_custom_call_ids: Final = frozenset(
|
||||
chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None)
|
||||
)
|
||||
custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset(
|
||||
tool_call["id"]
|
||||
for msg in messages
|
||||
if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list)
|
||||
|
|
@ -849,7 +902,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
tool_calls = msg.get("tool_calls")
|
||||
tool_call_id = msg.get("tool_call_id")
|
||||
|
||||
if role == "system":
|
||||
if (replayed_turn := replayed_turns[index]) is not None:
|
||||
input_items.extend(replayed_turn.items)
|
||||
elif role == "system":
|
||||
# Extract system message as instructions
|
||||
if isinstance(content, str) and index < leading_system_count:
|
||||
if instructions:
|
||||
|
|
@ -915,6 +970,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
"content": self._convert_content_to_responses_format(
|
||||
content, "assistant", drop_params=drop_params
|
||||
),
|
||||
**_assistant_phase_field(msg, replay_openai_history, drop_params),
|
||||
}
|
||||
)
|
||||
for tool_call in tool_calls:
|
||||
|
|
@ -955,6 +1011,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
"content": self._convert_content_to_responses_format(
|
||||
content, cast(str, role), drop_params=drop_params
|
||||
),
|
||||
**_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
|
||||
}
|
||||
)
|
||||
elif role == "assistant":
|
||||
|
|
@ -1081,6 +1138,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
|
||||
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
|
||||
include_namespace=True,
|
||||
replay_openai_history=provider == "openai",
|
||||
)
|
||||
# OpenAI's Responses API rejects an empty input. For a system-only
|
||||
# request, carry the system message as a system-role input item instead
|
||||
|
|
@ -1165,6 +1223,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
output_items: Sequence[object],
|
||||
handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
|
||||
structured_output: bool = False,
|
||||
replay_openai_history: bool = False,
|
||||
completed: bool = False,
|
||||
) -> list["Choices"]:
|
||||
"""
|
||||
Convert Responses API output items to chat completion choices.
|
||||
|
|
@ -1174,6 +1234,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
handle_raw_dict_callback: Optional callback for handling raw dict items
|
||||
structured_output: The chat request carried a json response_format, so only the
|
||||
first message item becomes content
|
||||
replay_openai_history: Record the output's item sequence for the next turn
|
||||
completed: The response finished, so its item sequence is whole
|
||||
|
||||
Returns:
|
||||
List of Choices objects
|
||||
|
|
@ -1196,7 +1258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
choices: Final[list[Choices]] = []
|
||||
index = 0
|
||||
reasoning_content: str | None = None
|
||||
pending_reasoning_item: _BuiltReasoningItem | None = None
|
||||
pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
|
||||
|
||||
# Collect all tool calls to put them in a single choice
|
||||
# (Chat Completions API expects all tool calls in one message)
|
||||
|
|
@ -1205,12 +1267,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
for item in output_items:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
pending_reasoning_item = _build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
pending_reasoning_items = (
|
||||
*(pending_reasoning_items if replay_openai_history else ()),
|
||||
_build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
),
|
||||
)
|
||||
reasoning_content = " ".join(
|
||||
s["text"]
|
||||
for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
|
||||
if s.get("text")
|
||||
)
|
||||
reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for content in item.content:
|
||||
|
|
@ -1227,7 +1296,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
annotations=annotations,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
([pending_reasoning_item] if pending_reasoning_item is not None else None),
|
||||
(list(pending_reasoning_items) or None),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -1240,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
)
|
||||
|
||||
reasoning_content = None # flush
|
||||
pending_reasoning_item = None # flush
|
||||
pending_reasoning_items = () # flush
|
||||
index += 1
|
||||
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
|
|
@ -1300,16 +1369,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
output_items, "stop", tool_search_items=tool_search_items
|
||||
)
|
||||
]
|
||||
return [
|
||||
_fold_choices_into_one(
|
||||
choices=choices,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
trailing_reasoning_content=reasoning_content,
|
||||
trailing_reasoning_item=pending_reasoning_item,
|
||||
tool_search_items=tool_search_items,
|
||||
structured_output=structured_output,
|
||||
)
|
||||
]
|
||||
folded: Final = _fold_choices_into_one(
|
||||
choices=choices,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
trailing_reasoning_content=reasoning_content,
|
||||
trailing_reasoning_items=pending_reasoning_items,
|
||||
tool_search_items=tool_search_items,
|
||||
structured_output=structured_output,
|
||||
)
|
||||
if not replay_openai_history:
|
||||
return [folded]
|
||||
return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)]
|
||||
|
||||
@staticmethod
|
||||
def _build_empty_choice(
|
||||
|
|
@ -1443,6 +1513,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
replay_openai_history=_served_provider(
|
||||
cast("Mapping[str, object]", litellm_params) # cast-ok: litellm_params is an untyped dict
|
||||
)
|
||||
== "openai",
|
||||
completed=raw_response.status == "completed",
|
||||
)
|
||||
|
||||
response_is_incomplete: Final = raw_response.status == "incomplete" or (
|
||||
|
|
@ -1506,9 +1581,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
structured_output: bool = False,
|
||||
replays_openai_history: bool = False,
|
||||
) -> BaseModelResponseIterator:
|
||||
return OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response, sync_stream, json_mode, structured_output=structured_output
|
||||
streaming_response,
|
||||
sync_stream,
|
||||
json_mode,
|
||||
structured_output=structured_output,
|
||||
replays_openai_history=replays_openai_history,
|
||||
)
|
||||
|
||||
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
|
||||
|
|
@ -1586,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if original_type == "text":
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
self._convert_content_str_to_input_text(item.get("text", ""), role),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: text -> %s", converted)
|
||||
|
|
@ -1599,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
cast(ChatCompletionImageObject, item), role
|
||||
),
|
||||
),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: image_url -> %s", converted)
|
||||
elif original_type == "input_audio":
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
{"type": "input_audio", "input_audio": item.get("input_audio")},
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: input_audio -> %s", converted)
|
||||
|
|
@ -1622,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
_input_file_from_file_value(
|
||||
cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked
|
||||
),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: file -> %s", converted)
|
||||
|
|
@ -1646,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
# Default to input_text for unknown types
|
||||
converted = with_prompt_cache_breakpoint(
|
||||
self._convert_content_str_to_input_text(str(item.get("text", item)), role),
|
||||
_prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
|
||||
)
|
||||
result.append(converted)
|
||||
verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted)
|
||||
|
|
@ -1890,8 +1970,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
structured_output: bool = False,
|
||||
replays_openai_history: bool = False,
|
||||
):
|
||||
super().__init__(streaming_response, sync_stream, json_mode)
|
||||
self._replays_openai_history: Final = replays_openai_history
|
||||
self._chat_completion_id: str | None = None
|
||||
self._served_service_tier: str | None = None
|
||||
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
|
||||
|
|
@ -1937,6 +2019,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
def translate_responses_chunk_to_openai_stream(
|
||||
parsed_chunk: dict | BaseModel,
|
||||
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
|
||||
replays_openai_history: bool = False,
|
||||
structured_output: bool = False,
|
||||
) -> "ModelResponseStream":
|
||||
"""
|
||||
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
|
||||
|
|
@ -1944,6 +2028,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
Args:
|
||||
parsed_chunk: Dict containing the Responses API event chunk
|
||||
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
|
||||
replays_openai_history: Record the completed output's item sequence for the next turn
|
||||
structured_output: Only the first message item became content
|
||||
|
||||
Returns:
|
||||
ModelResponseStream: OpenAI-formatted streaming chunk
|
||||
|
|
@ -2154,6 +2240,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
|
||||
_reasoning_items_from_output_items(output_items)
|
||||
)
|
||||
output_payload: Final = cast(object, output_items) # cast-ok: untyped
|
||||
tool_search_fields: Final = _tool_search_provider_fields(
|
||||
_tool_search_items_from_output_items(output_payload)
|
||||
)
|
||||
|
||||
usage = None
|
||||
if response_data.get("usage"):
|
||||
|
|
@ -2167,10 +2257,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(
|
||||
content="",
|
||||
content=None if replays_openai_history else "",
|
||||
reasoning_items=terminal_reasoning_items_typed,
|
||||
provider_specific_fields=_tool_search_provider_fields(
|
||||
_tool_search_items_from_output_items(cast(object, output_items)) # cast-ok: untyped
|
||||
provider_specific_fields=(
|
||||
completed_stream_history(
|
||||
output_payload,
|
||||
tool_search_fields,
|
||||
completed=event_type == "response.completed",
|
||||
structured_output=structured_output,
|
||||
)
|
||||
if replays_openai_history
|
||||
else tool_search_fields
|
||||
),
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
|
|
@ -2216,7 +2313,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
self._empty_content_chunk()
|
||||
if self._is_text_delta_after_structured_answer(chunk)
|
||||
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
|
||||
chunk, tool_call_index_map=self._tool_call_index_map
|
||||
chunk,
|
||||
tool_call_index_map=self._tool_call_index_map,
|
||||
replays_openai_history=self._replays_openai_history,
|
||||
structured_output=self._structured_output,
|
||||
)
|
||||
)
|
||||
if self._structured_output and _is_message_item_done(chunk):
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_
|
|||
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
|
||||
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
|
||||
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 60, 1, 86400
|
||||
)
|
||||
|
||||
# Maximum wall-clock seconds a streaming response is allowed to run.
|
||||
|
|
@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096
|
||||
|
|
@ -1064,6 +1065,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"https://api.reka.ai/v1",
|
||||
"https://gigachat.devices.sberbank.ru/api/v1",
|
||||
"https://api.openinfer.ai/v1",
|
||||
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -1145,6 +1147,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"prism",
|
||||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
]
|
||||
|
||||
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.constants import (
|
|||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
|
||||
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
|
|
@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
parse_prompt_tokens_details,
|
||||
select_cost_metric_for_model,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import response_ms_since_logging_start
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
cost_per_token as anthropic_cost_per_token,
|
||||
)
|
||||
|
|
@ -688,7 +689,7 @@ def cost_per_token(
|
|||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
return generic_cost_per_token(
|
||||
model=model_without_prefix,
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1396,9 +1397,9 @@ def _apply_cost_margin(
|
|||
|
||||
def _store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
prompt_tokens_cost_usd_dollar: float,
|
||||
completion_tokens_cost_usd_dollar: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float,
|
||||
prompt_tokens_cost_usd_dollar: float | None,
|
||||
completion_tokens_cost_usd_dollar: float | None,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float | None,
|
||||
total_cost_usd_dollar: float,
|
||||
additional_costs: dict | None = None,
|
||||
original_cost: float | None = None,
|
||||
|
|
@ -1500,9 +1501,45 @@ def _response_time_ms_for_cost(
|
|||
return float(stamped)
|
||||
if total_time:
|
||||
return total_time
|
||||
if litellm_logging_obj is not None:
|
||||
return litellm_logging_obj.get_response_ms()
|
||||
return 0.0
|
||||
if litellm_logging_obj is None:
|
||||
return 0.0
|
||||
reported_ms: Final = response_ms_since_logging_start(litellm_logging_obj)
|
||||
if reported_ms is not None and litellm_logging_obj.stream is not True:
|
||||
return reported_ms
|
||||
return litellm_logging_obj.get_response_ms()
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
model: str | None,
|
||||
region_name: str | None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
) -> float:
|
||||
discounted_cost, discount_percent, discount_amount = _apply_cost_discount(
|
||||
base_cost=reported_cost, custom_llm_provider=custom_llm_provider, model=model, region_name=region_name
|
||||
)
|
||||
adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
|
||||
base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=None,
|
||||
completion_tokens_cost_usd_dollar=None,
|
||||
cost_for_built_in_tools_cost_usd_dollar=None,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
discount_percent=discount_percent,
|
||||
discount_amount=discount_amount,
|
||||
margin_percent=margin_percent,
|
||||
margin_fixed_amount=margin_fixed_amount,
|
||||
margin_total_amount=margin_total_amount,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
return adjusted_cost
|
||||
|
||||
|
||||
def completion_cost(
|
||||
|
|
@ -1616,6 +1653,9 @@ def completion_cost(
|
|||
)
|
||||
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
caller_set_prices: Final = (
|
||||
explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None
|
||||
)
|
||||
custom_priced_request_model: Final = model if custom_pricing is True else None
|
||||
configured_cost_per_query: Final = (
|
||||
_configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None
|
||||
|
|
@ -1666,6 +1706,7 @@ def completion_cost(
|
|||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
priced_response: Final[object] = completion_response
|
||||
for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
model = candidate_model
|
||||
try:
|
||||
|
|
@ -1984,6 +2025,19 @@ def completion_cost(
|
|||
total_time=total_time,
|
||||
)
|
||||
model_for_discount = custom_priced_request_model or model
|
||||
if (
|
||||
not caller_set_prices
|
||||
and (reported_cost := billed_usage_cost(priced_response, custom_llm_provider)) is not None
|
||||
):
|
||||
return _adjusted_provider_billed_cost(
|
||||
reported_cost=reported_cost,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=model_for_discount,
|
||||
region_name=region_name if isinstance(region_name, str) else None,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
# Calculate cost based on prompt_tokens, completion_tokens
|
||||
if (
|
||||
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ class AuthenticationError(openai.AuthenticationError):
|
|||
litellm_debug_info: str | None = None,
|
||||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.status_code = 401
|
||||
self.message = f"litellm.AuthenticationError: {message}"
|
||||
|
|
@ -152,7 +153,7 @@ class AuthenticationError(openai.AuthenticationError):
|
|||
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
|
||||
)
|
||||
super().__init__(
|
||||
self.message, response=self.response, body=None
|
||||
self.message, response=self.response, body=body
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -189,6 +190,7 @@ class NotFoundError(openai.NotFoundError):
|
|||
litellm_debug_info: str | None = None,
|
||||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.status_code = 404
|
||||
self.message = f"litellm.NotFoundError: {message}"
|
||||
|
|
@ -202,7 +204,7 @@ class NotFoundError(openai.NotFoundError):
|
|||
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
|
||||
)
|
||||
super().__init__(
|
||||
self.message, response=self.response, body=None
|
||||
self.message, response=self.response, body=body
|
||||
) # Call the base class constructor with the parameters it needs
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -321,6 +323,17 @@ class UnsupportedEndpointError(BadRequestError):
|
|||
pass
|
||||
|
||||
|
||||
class EndpointNotSupportedByModelError(BadRequestError):
|
||||
def __init__(self, message: str, model: str, llm_provider: str = "") -> None:
|
||||
super().__init__( # pyright: ignore[reportUnknownMemberType] # inherited public exception constructor is untyped
|
||||
message=message, model=model, llm_provider=llm_provider
|
||||
)
|
||||
self.message = message
|
||||
self.type = "invalid_request_error"
|
||||
self.param = "model"
|
||||
self.code = "400"
|
||||
|
||||
|
||||
class UnprocessableEntityError(openai.UnprocessableEntityError):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None:
|
|||
return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields)
|
||||
|
||||
|
||||
def _genai_content_parts(value: object) -> Sequence[object]:
|
||||
single_part: Final = _validated(_CONFIG_FIELDS, value)
|
||||
return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or ()
|
||||
|
||||
|
||||
def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]:
|
||||
items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or ()
|
||||
return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None)
|
||||
|
|
@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]:
|
|||
fields: Final = _validated(_CONFIG_FIELDS, content)
|
||||
if fields is None:
|
||||
return ()
|
||||
parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or ()
|
||||
parts: Final = _genai_content_parts(fields.get("parts", ()))
|
||||
converted: Final = _genai_message_parts(parts)
|
||||
role: Final = fields.get("role", "user")
|
||||
if role == "user":
|
||||
|
|
|
|||
|
|
@ -68,6 +68,11 @@ from litellm.utils import (
|
|||
)
|
||||
|
||||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
_PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE: Final = (
|
||||
LlmProviders.DASHSCOPE,
|
||||
LlmProviders.QWENCLOUD,
|
||||
LlmProviders.QWEN_AI_PLATFORM,
|
||||
)
|
||||
|
||||
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
|
||||
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
|
||||
|
|
@ -271,6 +276,7 @@ def image_generation(
|
|||
"""
|
||||
try:
|
||||
args: Final = locals()
|
||||
requested_api_base: Final = api_base
|
||||
aimg_generation: Final = kwargs.get("aimg_generation", False)
|
||||
litellm_call_id: Final = kwargs.get("litellm_call_id", None)
|
||||
logger_fn: Final = kwargs.get("logger_fn", None)
|
||||
|
|
@ -468,7 +474,11 @@ def image_generation(
|
|||
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
|
||||
|
||||
# Resolve api_base from litellm.api_base if not explicitly provided
|
||||
_api_base: Final = api_base or litellm.api_base
|
||||
_api_base: Final = (
|
||||
requested_api_base
|
||||
if custom_llm_provider in _PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE
|
||||
else api_base or litellm.api_base
|
||||
)
|
||||
litellm_params_dict["api_base"] = _api_base
|
||||
|
||||
return llm_http_handler.image_generation_handler(
|
||||
|
|
@ -641,6 +651,8 @@ def image_generation(
|
|||
timeout=timeout,
|
||||
client=custom_client,
|
||||
)
|
||||
elif custom_llm_provider == LlmProviders.KYMA.value:
|
||||
raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
return model_response
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -349,7 +349,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
except Exception:
|
||||
return 0
|
||||
|
||||
async def send_daily_reports(self, router) -> bool:
|
||||
async def send_daily_reports(self, router: Router) -> bool:
|
||||
"""
|
||||
Send a daily report on:
|
||||
- Top 5 deployments with most failed requests
|
||||
|
|
@ -365,7 +365,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
False -> if not sent
|
||||
"""
|
||||
|
||||
ids: Final = router.get_model_ids()
|
||||
ids: Final = router.registry().names(kind="ids")
|
||||
|
||||
# get keys
|
||||
failed_request_keys: Final = [f"{id}:{SlackAlertingCacheKeys.failed_requests_key.value}" for id in ids]
|
||||
|
|
@ -430,7 +430,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
message += "\tNone\n"
|
||||
for i in range(len(top_5_failed)):
|
||||
key = failed_request_keys[top_5_failed[i]].split(":")[0]
|
||||
_deployment = router.get_model_info(key)
|
||||
_deployment = router.registry().deployment_dict(key)
|
||||
if isinstance(_deployment, dict):
|
||||
deployment_name = _deployment["litellm_params"].get("model", "")
|
||||
else:
|
||||
|
|
@ -452,7 +452,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
message += "\tNone\n"
|
||||
for i in range(len(top_5_slowest)):
|
||||
key = latency_keys[top_5_slowest[i]].split(":")[0]
|
||||
_deployment = router.get_model_info(key)
|
||||
_deployment = router.registry().deployment_dict(key)
|
||||
if _deployment is not None:
|
||||
deployment_name = _deployment["litellm_params"].get("model", "")
|
||||
else:
|
||||
|
|
@ -802,7 +802,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
if self.llm_router is None:
|
||||
return
|
||||
|
||||
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = self.llm_router.registry().deployment(deployment_id)
|
||||
|
||||
if deployment is None:
|
||||
return
|
||||
|
|
@ -953,7 +953,7 @@ class SlackAlerting(CustomBatchLogger):
|
|||
return
|
||||
|
||||
### EXTRACT MODEL DETAILS ###
|
||||
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
|
||||
deployment: Final = self.llm_router.registry().deployment(deployment_id)
|
||||
if deployment is None:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
|
|||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
client: Final = get_httpx_client()
|
||||
try:
|
||||
response: Final = client.post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
response: Final = get_httpx_client().post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
finally:
|
||||
client.close()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
|
|
|||
|
|
@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
|
|||
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
|
||||
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
|
||||
_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
|
||||
_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
|
||||
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
|
||||
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
|
||||
)
|
||||
|
|
@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None:
|
|||
|
||||
|
||||
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
|
||||
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
|
||||
|
||||
|
||||
def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
|
||||
hosted_flag: Final = (
|
||||
None
|
||||
if custom_llm_provider is None
|
||||
|
|
@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str
|
|||
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
|
||||
if model_map_flag is not None:
|
||||
return model_map_flag
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
|
||||
if version_match is None:
|
||||
fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
|
||||
if fine_tuned_base is not None:
|
||||
return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None
|
||||
return _named_model_breakpoint_support(model)
|
||||
|
||||
|
||||
def _named_model_breakpoint_support(model: str) -> bool | None:
|
||||
bare_model: Final = model.rsplit("/", 1)[-1].lower()
|
||||
if _O_SERIES_PATTERN.match(bare_model) is not None:
|
||||
return False
|
||||
version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
|
||||
if version_match is None:
|
||||
return None
|
||||
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
|
||||
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
|
||||
|
||||
|
|
|
|||
|
|
@ -513,7 +513,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
data: dict,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
call_type: CallTypesLiteral,
|
||||
) -> Any:
|
||||
) -> object:
|
||||
pass
|
||||
|
||||
async def async_post_call_streaming_hook(
|
||||
|
|
|
|||
|
|
@ -205,11 +205,12 @@ class DatadogCostManagementLogger(CustomBatchLogger):
|
|||
# see _RESERVED_TAG_KEYS for the rationale.
|
||||
if self.cost_tag_keys:
|
||||
allow: Final = set(self.cost_tag_keys)
|
||||
client_allow: Final = allow - {"is_silent_experiment"}
|
||||
for rt in log.get("request_tags") or []:
|
||||
if not isinstance(rt, str) or ":" not in rt:
|
||||
continue
|
||||
k, _, v = rt.partition(":")
|
||||
if k in allow and v:
|
||||
if k in client_allow and v:
|
||||
self._set_custom_tag(tags, k, v)
|
||||
for k, v in metadata.items():
|
||||
if k in allow and v is not None and not isinstance(v, (dict, list)):
|
||||
|
|
@ -218,7 +219,7 @@ class DatadogCostManagementLogger(CustomBatchLogger):
|
|||
nested = metadata.get(nested_key)
|
||||
if isinstance(nested, dict):
|
||||
for k, v in nested.items():
|
||||
if k in allow and v is not None and not isinstance(v, (dict, list)):
|
||||
if k in client_allow and v is not None and not isinstance(v, (dict, list)):
|
||||
self._set_custom_tag(tags, k, str(v))
|
||||
|
||||
return tags
|
||||
|
|
|
|||
|
|
@ -833,6 +833,12 @@ class LangFuseLogger:
|
|||
("vertex_location", vertex_location, bool(vertex_location)),
|
||||
("aws_region_name", aws_region_name, bool(aws_region_name)),
|
||||
("cache_hit", kwargs.get("cache_hit") or False, "cache_hit" in kwargs),
|
||||
(
|
||||
"is_silent_experiment",
|
||||
"is_silent_experiment" in allowlisted_metadata
|
||||
and allowlisted_metadata["is_silent_experiment"] is True,
|
||||
"is_silent_experiment" in allowlisted_metadata,
|
||||
),
|
||||
)
|
||||
enrichments: Final[Mapping[str, object]] = {
|
||||
key: value for key, value, include in candidate_enrichments if include
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
|
||||
every call but one whose logging payload shows it never reached a provider."""
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
|
||||
if not _a_destination_carries_the_trace():
|
||||
return False
|
||||
call: Final = LLMCallEvent.from_dict(kwargs)
|
||||
return call.payload is None or call.reached_a_provider
|
||||
|
||||
|
||||
def _a_destination_carries_the_trace() -> bool:
|
||||
"""Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
|
||||
trace."""
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
|
||||
return ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
|
||||
|
||||
class LangfuseOtelLogger(OpenTelemetry):
|
||||
def __init__(self, config=None, *args, **kwargs):
|
||||
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
|
||||
|
|
@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"""
|
||||
return None
|
||||
|
||||
def _emit_once(
|
||||
self,
|
||||
kwargs: dict[str, object], # mutable-ok: the v1 dedupe marker is written into the request's kwargs
|
||||
*scope: object,
|
||||
) -> bool:
|
||||
"""Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
|
||||
return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope) # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict[str, object], # mutable-ok: the v1 hook contract
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: str | None = None,
|
||||
) -> None:
|
||||
"""Record the proxy failure of a request no Langfuse destination carries."""
|
||||
if not _a_destination_carries_the_trace():
|
||||
await super().async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
request_data, original_exception, user_api_key_dict, traceback_str
|
||||
)
|
||||
|
||||
async def async_service_success_hook(self, *args, **kwargs):
|
||||
"""
|
||||
Langfuse should not receive service success logs.
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class BaseLLMObsOTELAttributes(ABC):
|
|||
pass
|
||||
|
||||
|
||||
def cast_as_primitive_value_type(value) -> str | bool | int | float:
|
||||
def cast_as_primitive_value_type(value: object) -> str | bool | int | float:
|
||||
"""
|
||||
Converts a value to an OTEL-supported primitive for Arize/Phoenix observability.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
|
|||
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
|
||||
from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
|
|
@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
|
|||
mcp_tool_call_span_name,
|
||||
service_span_name,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
|
||||
|
|
@ -168,6 +169,9 @@ class SpanEmitter:
|
|||
self._mappers: list[AttributeMapper] = (
|
||||
list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
|
||||
)
|
||||
self._langfuse: Final = (
|
||||
None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
|
||||
)
|
||||
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
|
||||
# only — the value is unused — so it behaves like a capped set.
|
||||
self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
|
||||
|
|
@ -295,17 +299,27 @@ class SpanEmitter:
|
|||
)
|
||||
else None
|
||||
)
|
||||
mapped: Final = MappingProxyType(
|
||||
{key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
for_langfuse: Final = (
|
||||
dict(
|
||||
sorted(
|
||||
self._langfuse.map(data).items(),
|
||||
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
)
|
||||
)
|
||||
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
else _NO_ATTRIBUTES
|
||||
)
|
||||
mapped: Final = MappingProxyType({**configured, **for_langfuse})
|
||||
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
|
||||
reserved: Final = len(stamped_later.keys() - mapped.keys())
|
||||
overwrites: Final = carried_keys(span) & mapped.keys()
|
||||
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
|
||||
# The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
|
||||
# so a span at its attribute limit loses nothing else to them.
|
||||
fitted: Final = fit_indexed_messages(
|
||||
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
|
||||
attribute_budget(span, reserved),
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
|
||||
)
|
||||
for key, value in mapped.items():
|
||||
if key in overwrites or key in fitted:
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
tracer_provider: TracerProvider | None = None,
|
||||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
console_default: bool = False,
|
||||
**kwargs: Unpack[_CustomLoggerOptions],
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._tracer_provider: TracerProvider = (
|
||||
tracer_provider
|
||||
if tracer_provider is not None
|
||||
else build_tracer_provider(self.config, tenant_overrides=True)
|
||||
else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
|
||||
)
|
||||
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
|
||||
self._metrics_recorder = self._init_metrics(meter_provider)
|
||||
|
|
@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
mappers=resolve_mappers(self.config.mapper_names),
|
||||
event_recorder=self._init_events(logger_provider),
|
||||
)
|
||||
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
|
||||
self._tenant_tracers = TenantTracerCache(
|
||||
self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
|
||||
)
|
||||
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
|
||||
self._init_otel_logger_on_litellm_proxy()
|
||||
|
||||
|
|
@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
|
||||
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
|
||||
# upstream stamp, since the payload proves the call completed.
|
||||
if carrier is None and (
|
||||
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
|
||||
):
|
||||
if carrier is None and not call.reached_a_provider:
|
||||
return None
|
||||
try:
|
||||
return self._finish_carrier(carrier, call, start_time, end_time)
|
||||
|
|
@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
|
|||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
settings: Mapping[str, object] = MappingProxyType({}),
|
||||
console_default: bool = False,
|
||||
) -> OpenTelemetryV2:
|
||||
return _logger_class(config)(
|
||||
config=config,
|
||||
|
|
@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
|
|||
tracer_provider=tracer_provider,
|
||||
logger_provider=logger_provider,
|
||||
meter_provider=meter_provider,
|
||||
console_default=console_default,
|
||||
**settings,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,18 @@ class LangfuseMapper:
|
|||
),
|
||||
}
|
||||
|
||||
#: Every attribute this mapper can write on a model call.
|
||||
LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
*_LLM_CALL_ATTRS,
|
||||
*_BLOB_ATTRS,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LANGFUSE_TRACE_USER_ID,
|
||||
LANGFUSE_TRACE_SESSION_ID,
|
||||
LANGFUSE_TRACE_TAGS,
|
||||
}
|
||||
)
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
|
|
|
|||
|
|
@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
|
|||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec ``credential_gated_exporters`` appends, which would
|
||||
otherwise eat itself when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
||||
|
||||
class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
|
||||
def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
|
||||
super().__init__(settings_cls)
|
||||
|
|
|
|||
|
|
@ -272,6 +272,11 @@ class LLMCallEvent:
|
|||
trace: TraceControls
|
||||
session_id: str | None
|
||||
|
||||
@property
|
||||
def reached_a_provider(self) -> bool:
|
||||
"""Whether the call was handed to a provider or answered from the response cache."""
|
||||
return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
|
|||
LANGFUSE_OBSERVATION_OUTPUT,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LangfuseMapper,
|
||||
)
|
||||
from litellm.integrations.otel.model.config import (
|
||||
CaptureMessageContent,
|
||||
|
|
@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
|
|||
OpenTelemetryV2Config,
|
||||
capture_message_content_from_env,
|
||||
captures_span_content,
|
||||
is_unconfigured_placeholder,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
|
|
@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
|
|||
return scope is not None and scope.name == LITELLM_TRACER_NAME
|
||||
|
||||
|
||||
def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
|
||||
"""``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
|
||||
def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
|
||||
"""``span`` without the Langfuse attributes only a Langfuse destination asked for.
|
||||
|
||||
Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
|
||||
That is the request metadata the Langfuse mapper writes and, on a model call written by
|
||||
a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
|
||||
every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
|
||||
destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
|
||||
its own code set.
|
||||
"""
|
||||
attributes: Final = span.attributes or _NO_ATTRIBUTES
|
||||
if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
|
||||
dropped: Final = (
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
if langfuse_mapped or not is_llm_call_span(span)
|
||||
else LangfuseMapper.LLM_CALL_ATTRIBUTES
|
||||
)
|
||||
if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
|
||||
return span
|
||||
return _CountedSpanView(
|
||||
span,
|
||||
MappingProxyType(
|
||||
{key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
|
||||
),
|
||||
span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -658,6 +666,7 @@ def _for_destination(
|
|||
span: ReadableSpan,
|
||||
destination: "OtelDestination",
|
||||
default_capture: str,
|
||||
langfuse_mapped: bool,
|
||||
) -> ReadableSpan:
|
||||
"""The view of ``span`` a tenant destination receives.
|
||||
|
||||
|
|
@ -672,13 +681,16 @@ def _for_destination(
|
|||
and the query string of the request URL, which can hold the same key. A
|
||||
destination whose effective setting disables span content loses the prompt and
|
||||
response content too, and a destination for any backend but Langfuse loses the
|
||||
request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
|
||||
stays, so the tenant still gets the whole trace tree.
|
||||
Langfuse attributes only a Langfuse destination asked for (see
|
||||
:func:`without_destination_langfuse_attributes`). The span itself stays, so the
|
||||
tenant still gets the whole trace tree.
|
||||
"""
|
||||
extra: Final = destination.resource_attributes
|
||||
defaults: Final = destination.resource_defaults
|
||||
source: Final = (
|
||||
span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
|
||||
span
|
||||
if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
|
||||
else without_destination_langfuse_attributes(span, langfuse_mapped)
|
||||
)
|
||||
attributes: Final = source.attributes or _NO_ATTRIBUTES
|
||||
database: Final = _is_database_span(attributes)
|
||||
|
|
@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
|
||||
pending_drains: int = _MAX_PENDING_DRAINS,
|
||||
drain_pool: _DrainPool | None = None,
|
||||
langfuse_mapped: bool = True,
|
||||
) -> None:
|
||||
self._operator_sinks: Final = operator_sinks
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
self._excluded_db_systems: Final = excluded_db_systems
|
||||
self._default_capture: Final = default_capture
|
||||
self._default_capture_by_backend: Final = default_capture_by_backend
|
||||
|
|
@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
try:
|
||||
processor.on_end(
|
||||
_scoped(
|
||||
_for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
|
||||
_for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
|
||||
destination.span_scope,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span
|
||||
|
|
@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
|
|||
return self._inner.force_flush(timeout_millis)
|
||||
|
||||
|
||||
class _LangfuseRequestMetadataFilter(SpanProcessor):
|
||||
"""Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
|
||||
class _DestinationLangfuseAttributeFilter(SpanProcessor):
|
||||
"""Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
|
||||
exports them."""
|
||||
|
||||
def __init__(self, inner: SpanProcessor) -> None:
|
||||
def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
|
||||
self._inner: Final = inner
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
|
||||
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
|
||||
self._inner.on_start(span, parent_context)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._inner.on_end(without_langfuse_request_metadata(span))
|
||||
self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._inner.shutdown()
|
||||
|
|
@ -1491,13 +1508,18 @@ def _spec_processor(
|
|||
spec: ExporterSpec,
|
||||
use_simple_processor: bool | None,
|
||||
strip_message_content: bool,
|
||||
strip_langfuse_request_metadata: bool,
|
||||
strip_destination_langfuse_attributes: bool,
|
||||
langfuse_mapped: bool,
|
||||
) -> SpanProcessor:
|
||||
exporting: Final = _processor_for(
|
||||
_exporter_from_spec(spec),
|
||||
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
|
||||
)
|
||||
scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
|
||||
scoped: Final = (
|
||||
_DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
|
||||
if strip_destination_langfuse_attributes
|
||||
else exporting
|
||||
)
|
||||
return _MessageContentFilter(scoped) if strip_message_content else scoped
|
||||
|
||||
|
||||
|
|
@ -1530,6 +1552,7 @@ def build_tracer_provider(
|
|||
use_simple_processor: bool | None = None,
|
||||
tenant_overrides: bool = False,
|
||||
strip_message_content: bool = False,
|
||||
console_default: bool = False,
|
||||
) -> TracerProvider:
|
||||
"""Build the shared :class:`TracerProvider`.
|
||||
|
||||
|
|
@ -1548,11 +1571,16 @@ def build_tracer_provider(
|
|||
|
||||
``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
|
||||
alone; a collector or any other backend in the same config keeps the full tree.
|
||||
In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
|
||||
goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
|
||||
Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
|
||||
Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
|
||||
which any logger writes for a request with a Langfuse destination.
|
||||
``strip_message_content`` removes prompt and response content before every
|
||||
exporter, for a routed provider whose spans can carry content another
|
||||
destination asked for.
|
||||
|
||||
The console spec ``config._normalize`` folds in when nothing was configured
|
||||
prints only under ``console_default``, which the ``otel`` callback passes: a
|
||||
vendor preset or the proxy's background global logger was never asked to print.
|
||||
"""
|
||||
provider: Final = TracerProvider(resource=build_resource(config))
|
||||
provider.sampler = _RequestExclusionSampler(provider.sampler)
|
||||
|
|
@ -1564,19 +1592,20 @@ def build_tracer_provider(
|
|||
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
|
||||
return provider
|
||||
|
||||
langfuse_mapped: Final = _maps_langfuse(config)
|
||||
# ``config._normalize`` guarantees at least one spec (it folds the top-level
|
||||
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
|
||||
writes_langfuse: Final = "langfuse" in config.mapper_names or any(
|
||||
spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
|
||||
)
|
||||
for spec in config.exporters:
|
||||
if spec.requires_headers and not spec.headers:
|
||||
continue
|
||||
if is_unconfigured_placeholder(spec) and not console_default:
|
||||
continue
|
||||
processor = _spec_processor(
|
||||
spec,
|
||||
use_simple_processor,
|
||||
strip_message_content,
|
||||
strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
langfuse_mapped=langfuse_mapped,
|
||||
)
|
||||
owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
|
||||
scope = _operator_scope(config, spec)
|
||||
|
|
@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
|
|||
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
|
||||
config per v2 logger since each keeps its own provider and still writes its
|
||||
account, so an additive destination pointing at any of them is delivered once
|
||||
rather than twice. ``excluded_db_systems`` only filters what the fan-out
|
||||
delivers, never the operator's own exporters.
|
||||
rather than twice. The first is the config of the logger that owns ``provider``,
|
||||
whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
|
||||
the fan-out delivers, never the operator's own exporters.
|
||||
"""
|
||||
with _FAN_OUT_ATTACH_LOCK:
|
||||
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
|
||||
|
|
@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
|
|||
excluded_db_systems=excluded_db_systems,
|
||||
default_capture=_operator_capture_mode(configs),
|
||||
default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
|
||||
langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
|
||||
"""Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
|
||||
return "langfuse" in config.mapper_names
|
||||
|
||||
|
||||
def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
|
||||
"""The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
|
||||
return next(
|
||||
|
|
|
|||
|
|
@ -163,10 +163,12 @@ class TenantTracerCache:
|
|||
config: OpenTelemetryV2Config,
|
||||
callback_name: str | None,
|
||||
tracer_name: str,
|
||||
console_default: bool = False,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._callback_name = callback_name
|
||||
self._tracer_name = tracer_name
|
||||
self._console_default = console_default
|
||||
# Guards the three mutable structures below: ``pre_call`` can run on
|
||||
# thread-pool workers concurrently with the event loop, so cache
|
||||
# updates, span counts, and retirement must be atomic.
|
||||
|
|
@ -284,6 +286,7 @@ class TenantTracerCache:
|
|||
built: Final = build_tracer_provider(
|
||||
self._routed_config(credential_headers, project_headers, endpoint, service_name),
|
||||
strip_message_content=not self._config.capture_span_content,
|
||||
console_default=self._console_default,
|
||||
)
|
||||
self._providers[cache_key] = built
|
||||
return built
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
|
|||
_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _arize_project() -> Mapping[str, str]:
|
||||
def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
from litellm.integrations.arize.arize import ArizeLogger
|
||||
|
||||
project: Final = ArizeLogger.get_arize_config().project_name
|
||||
|
|
@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
|
|||
return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
|
||||
|
||||
|
||||
#: Resource attributes the operator chose for a backend, applied to every export the
|
||||
#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
|
||||
{"arize": _arize_project}
|
||||
def _stands_in_for_the_legacy_langfuse_logger() -> bool:
|
||||
"""Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
|
||||
of its own sends for a key's or team's ``langfuse_otel``."""
|
||||
return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
|
||||
|
||||
|
||||
def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
"""The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
|
||||
logger wrote it."""
|
||||
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
|
||||
|
||||
if not _stands_in_for_the_legacy_langfuse_logger():
|
||||
return _NO_ATTRS
|
||||
environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
|
||||
|
||||
|
||||
#: Resource attributes a backend gets on every export the team receives, over the
|
||||
#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
|
||||
MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
|
||||
)
|
||||
|
||||
#: Resource attributes a backend needs on every export to accept it, filled only where
|
||||
|
|
@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
|
|||
)
|
||||
|
||||
|
||||
def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
|
||||
def _resource_attributes(
|
||||
callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
|
||||
) -> Mapping[str, str]:
|
||||
backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**({"service.name": service_name} if service_name else {}),
|
||||
**(backend() if backend is not None else {}),
|
||||
**(backend(params) if backend is not None else {}),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -214,7 +233,7 @@ def destination_for(
|
|||
return OtelDestination(
|
||||
endpoint=endpoint,
|
||||
headers=MappingProxyType(dict(headers)),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name, params),
|
||||
resource_defaults=_resource_defaults(callback_name),
|
||||
callback_name=callback_name,
|
||||
protocol=protocol,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
|
||||
|
||||
|
||||
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
|
||||
|
|
@ -35,14 +35,3 @@ def credential_gated_exporters(
|
|||
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
|
||||
ExporterSpec(owner=owner, requires_headers=True),
|
||||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec this module appends, which would otherwise eat itself
|
||||
when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
|
|
|||
|
|
@ -468,15 +468,16 @@ def _bounded_requested_model_label(requested_model: object, router_originated: b
|
|||
llm_router: Final = _get_proxy_llm_router()
|
||||
if llm_router is None:
|
||||
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
if llm_router.is_recognized_model(requested_model):
|
||||
registry: Final = llm_router.registry()
|
||||
if registry.resolve(requested_model, kind="recognized"):
|
||||
return requested_model
|
||||
if requested_model in llm_router.team_public_model_names:
|
||||
if requested_model in registry.names(kind="team_public"):
|
||||
return requested_model
|
||||
if llm_router.pattern_router.route(requested_model) is not None:
|
||||
if registry.patterns().route(requested_model) is not None:
|
||||
return requested_model
|
||||
if any(
|
||||
team_pattern_router.route(requested_model) is not None
|
||||
for team_pattern_router in llm_router.team_pattern_routers.values()
|
||||
for team_pattern_router in registry.patterns(team=True).values()
|
||||
):
|
||||
return requested_model
|
||||
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
|
|
@ -1173,8 +1174,8 @@ class PrometheusLogger(CustomLogger):
|
|||
# Build label filters from valid configurations
|
||||
label_filters: Final = self._build_label_filters(parsed_configs)
|
||||
|
||||
# Pretty print the processed configuration
|
||||
self._pretty_print_prometheus_config(label_filters)
|
||||
if litellm.suppress_startup_output is not True:
|
||||
self._pretty_print_prometheus_config(label_filters)
|
||||
return label_filters
|
||||
|
||||
def _parse_exclude_config(self) -> tuple[frozenset[str], frozenset[str]]:
|
||||
|
|
@ -3410,7 +3411,7 @@ class PrometheusLogger(CustomLogger):
|
|||
OpenAI / Anthropic / Azure return remaining tokens/requests in response
|
||||
headers, but Bedrock and Vertex AI do not. This fallback computes
|
||||
``configured_limit - current_usage`` via
|
||||
``Router.get_remaining_model_group_usage`` so the same gauges are
|
||||
``Router.health().usage(model_group, kind="remaining")`` so the same gauges are
|
||||
emitted for every provider when tpm/rpm is configured on the
|
||||
deployment.
|
||||
"""
|
||||
|
|
@ -3437,7 +3438,7 @@ class PrometheusLogger(CustomLogger):
|
|||
return
|
||||
|
||||
try:
|
||||
remaining_usage: Final = await llm_router.get_remaining_model_group_usage(model_group)
|
||||
remaining_usage: Final = await llm_router.health().usage(model_group, kind="remaining")
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"Prometheus: get_remaining_model_group_usage failed for model_group=%s: %s",
|
||||
|
|
@ -5148,6 +5149,12 @@ def _get_combined_custom_metadata_from_standard_logging_payload(
|
|||
**(requester_metadata if isinstance(requester_metadata, dict) else {}),
|
||||
**(user_api_key_auth_metadata if isinstance(user_api_key_auth_metadata, dict) else {}),
|
||||
**(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
|
||||
"is_silent_experiment": (
|
||||
"true"
|
||||
if "is_silent_experiment" in standard_logging_metadata
|
||||
and standard_logging_metadata["is_silent_experiment"] is True
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ from litellm.litellm_core_utils.llm_judge import (
|
|||
)
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
|
||||
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
|
||||
|
|
@ -790,9 +789,7 @@ class ActiveShadowEvalJob(LiteLLMBaseModel):
|
|||
def _canonical_group(router: "Router | None", model_group: str) -> str:
|
||||
"""A model group in the one spelling both a job's scope and a request's model compare
|
||||
under: an alias resolves to its target so the two never fail to match on spelling."""
|
||||
return (
|
||||
resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
|
||||
) or model_group
|
||||
return (router.registry().resolve(model_group, kind="alias") if router is not None else None) or model_group
|
||||
|
||||
|
||||
def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import (
|
|||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
|
|
@ -78,6 +79,7 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -114,12 +116,24 @@ class _WebSearchSettingsView(TypedDict):
|
|||
websearch_interception_params: WebSearchInterceptionConfig
|
||||
|
||||
|
||||
def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str:
|
||||
agentic_view: Final[_AgenticLoopParamsView] = {
|
||||
"agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {}
|
||||
}
|
||||
recorded_model: Final = agentic_view["agentic_loop_params"].get("model")
|
||||
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
|
||||
return qualify_provider_stripped_model(
|
||||
recorded_model if isinstance(recorded_model, str) and recorded_model else model,
|
||||
custom_llm_provider if isinstance(custom_llm_provider, str) else "",
|
||||
)
|
||||
|
||||
|
||||
class _SearchToolLitellmParams(TypedDict, total=False):
|
||||
search_provider: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _SearchToolConfig(TypedDict, total=False):
|
||||
search_tool_name: str
|
||||
search_tool_name: ReadOnly[str]
|
||||
litellm_params: ReadOnly[_SearchToolLitellmParams | None]
|
||||
|
||||
|
||||
|
|
@ -1155,6 +1169,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
tool_calls=tool_calls,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
response_format=response_format,
|
||||
)
|
||||
return AgenticLoopPlan(
|
||||
|
|
@ -1181,6 +1196,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
tool_calls=tool_calls,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
|
|
@ -1195,6 +1211,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
tool_calls: list[dict],
|
||||
optional_params: dict,
|
||||
kwargs: dict,
|
||||
logging_obj: "LiteLLMLoggingObj | None",
|
||||
) -> AgenticLoopRequestPatch:
|
||||
"""Execute litellm.asearch() and build a Responses API rerun patch."""
|
||||
search_tasks: Final = [
|
||||
|
|
@ -1254,9 +1271,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
}
|
||||
}
|
||||
|
||||
full_model_name = model
|
||||
if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str):
|
||||
full_model_name = f"{kwargs['custom_llm_provider']}/{model}"
|
||||
full_model_name: Final = _caller_model(model, logging_obj, kwargs)
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d",
|
||||
|
|
@ -1754,10 +1769,13 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
return None
|
||||
|
||||
def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None":
|
||||
if llm_router is None or not hasattr(llm_router, "search_tools"):
|
||||
def _select_search_tool_from_router(self, llm_router: "Router | None") -> "_SearchToolConfig | None":
|
||||
if llm_router is None:
|
||||
return None
|
||||
search_tools: Final = tuple(getattr(llm_router, "search_tools", None) or ())
|
||||
settings: Final = llm_router.settings()
|
||||
if not settings.is_set("search_tools"):
|
||||
return None
|
||||
search_tools: Final = tuple(settings.search_tools or ())
|
||||
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
|
||||
|
||||
def _select_search_tool_from_list(
|
||||
|
|
@ -1814,6 +1832,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
tool_calls=tool_calls,
|
||||
optional_params=dict[str, object](optional_params),
|
||||
kwargs=dict[str, object](kwargs),
|
||||
logging_obj=logging_obj,
|
||||
response_format=response_format,
|
||||
)
|
||||
if request_patch.messages is None:
|
||||
|
|
@ -1836,7 +1855,8 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
messages: list[dict],
|
||||
tool_calls: list[dict],
|
||||
optional_params: dict,
|
||||
kwargs: dict,
|
||||
kwargs: dict[str, object],
|
||||
logging_obj: "LiteLLMLoggingObj | None",
|
||||
response_format: str = "openai",
|
||||
) -> AgenticLoopRequestPatch:
|
||||
"""Execute litellm.search() and build chat-completion rerun patch."""
|
||||
|
|
@ -1922,11 +1942,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params
|
||||
}
|
||||
|
||||
full_model_name = model
|
||||
if "custom_llm_provider" in kwargs:
|
||||
custom_llm_provider: Final = kwargs["custom_llm_provider"]
|
||||
if not model.startswith(custom_llm_provider) and "/" not in model:
|
||||
full_model_name = f"{custom_llm_provider}/{model}"
|
||||
full_model_name: Final = _caller_model(model, logging_obj, kwargs)
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Built chat completion request patch model=%s messages=%d",
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ class LiteLLMResponsesInteractionsHandler:
|
|||
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
||||
responses_response=responses_api_response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
async def async_interactions_api_handler(
|
||||
|
|
@ -138,4 +139,5 @@ class LiteLLMResponsesInteractionsHandler:
|
|||
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
||||
responses_response=responses_api_response,
|
||||
model=model,
|
||||
custom_llm_provider=responses_request.get("custom_llm_provider"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Final, cast
|
|||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
|
||||
from litellm.types.interactions import (
|
||||
InteractionInput,
|
||||
InteractionsAPIOptionalRequestParams,
|
||||
|
|
@ -190,6 +191,7 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
def transform_responses_response_to_interactions_response(
|
||||
responses_response: ResponsesAPIResponse,
|
||||
model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> InteractionsAPIResponse:
|
||||
"""
|
||||
Transform a Responses API response to an Interactions API response.
|
||||
|
|
@ -266,9 +268,11 @@ class LiteLLMResponsesInteractionsConfig:
|
|||
# (total_input_tokens, total_output_tokens)
|
||||
usage: Final[object] = getattr(responses_response, "usage", None)
|
||||
if usage:
|
||||
reported_cost: Final = billed_usage_cost(responses_response, custom_llm_provider)
|
||||
interactions_response_dict["usage"] = {
|
||||
"total_input_tokens": getattr(usage, "input_tokens", 0),
|
||||
"total_output_tokens": getattr(usage, "output_tokens", 0),
|
||||
**({} if reported_cost is None else {"cost": reported_cost}),
|
||||
}
|
||||
|
||||
# Add updated (same as created for now)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue