mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
chore: merge main and retain upstream telemetry tests
This commit is contained in:
commit
73eae699ee
25 changed files with 935 additions and 96 deletions
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GITHUB_ACTIONS:-}" = true ]; then
|
||||
echo "Integration contracts are owned by CircleCI" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
suite="${1:?integration suite required}"
|
||||
mode="${2:-standard}"
|
||||
side="${3:-}"
|
||||
requested_workers="${INTEGRATION_WORKERS:-}"
|
||||
license_env=()
|
||||
if [ -n "${LITELLM_LICENSE:-}" ]; then
|
||||
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
|
||||
fi
|
||||
if [ "$mode" = replica ]; then
|
||||
results="test-results/integration-${suite}-replica"
|
||||
elif [ "$mode" = parity ]; then
|
||||
|
|
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
|
|||
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
|
||||
test -f "$LITELLM_UI_PATH/index.html"
|
||||
fi
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
|
||||
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
|
||||
|
|
@ -107,7 +107,7 @@ sudo iptables -N integration_only
|
|||
guard_created=true
|
||||
sudo iptables -A integration_only -o lo -j ACCEPT
|
||||
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
for service in postgres-db redis-cache; do
|
||||
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
|
||||
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
|
||||
sudo iptables -A integration_only -d "$address" -j ACCEPT
|
||||
done
|
||||
|
|
@ -145,6 +145,9 @@ fi
|
|||
if [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
|
||||
fi
|
||||
if [ -n "$requested_workers" ]; then
|
||||
export INTEGRATION_WORKERS="$requested_workers"
|
||||
fi
|
||||
isolated_workers=false
|
||||
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
|
||||
isolated_workers=true
|
||||
|
|
@ -185,7 +188,7 @@ start_proxy() {
|
|||
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
|
||||
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
|
||||
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
|
||||
|
|
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|||
"${shared_services[@]}" "${worker_coverage[@]}" \
|
||||
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
||||
LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
||||
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
|
||||
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
|
||||
|
|
|
|||
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TextIO
|
||||
|
||||
SECRET_VARIABLE: Final = "SCANNED_SECRET"
|
||||
MIN_ENCODED_LENGTH: Final = 12
|
||||
MAX_ARCHIVE_DEPTH: Final = 3
|
||||
MASK: Final = "***"
|
||||
|
||||
Forms = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hit:
|
||||
location: str
|
||||
form: str
|
||||
|
||||
|
||||
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
|
||||
for offset in range(3):
|
||||
chunk: Final = raw[offset:]
|
||||
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
|
||||
|
||||
|
||||
def secret_forms(secret: str) -> Forms:
|
||||
encoded: Final = (
|
||||
("url-encoded", urllib.parse.quote(secret, safe="")),
|
||||
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
|
||||
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
|
||||
)
|
||||
by_value: Final = MappingProxyType(
|
||||
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
|
||||
)
|
||||
return (("raw", secret), *((name, value) for value, name in by_value.items()))
|
||||
|
||||
|
||||
def mask(text: str, forms: Forms) -> str:
|
||||
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
|
||||
|
||||
|
||||
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value in name:
|
||||
yield Hit(mask(location, forms), f"{form} in name")
|
||||
|
||||
|
||||
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value.encode() in data:
|
||||
yield Hit(mask(location, forms), form)
|
||||
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
|
||||
return
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
for member in archive.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
|
||||
|
||||
|
||||
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
|
||||
for directory, subdirectories, files in os.walk(root):
|
||||
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
|
||||
yield from ((Path(directory, name), False) for name in files)
|
||||
|
||||
|
||||
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
|
||||
location: Final = str(path.relative_to(root))
|
||||
if linked_directory:
|
||||
yield Hit(mask(location, forms), "directory symlink, not scanned")
|
||||
return
|
||||
yield from _name_hits(location, location, forms)
|
||||
yield from _content_hits(location, path.read_bytes(), forms, 0)
|
||||
|
||||
|
||||
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
|
||||
forms: Final = secret_forms(secret)
|
||||
entries: Final = tuple(_walk(root))
|
||||
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
|
||||
return len(entries), hits
|
||||
|
||||
|
||||
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
|
||||
if len(argv) != 1:
|
||||
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
|
||||
return 2
|
||||
secret: Final = environ.get(SECRET_VARIABLE, "")
|
||||
root: Final = Path(argv[0])
|
||||
if not secret:
|
||||
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
|
||||
return 0
|
||||
if not root.is_dir():
|
||||
print(f"{root} is not a directory, nothing to scan", file=out)
|
||||
return 0
|
||||
try:
|
||||
scanned, hits = scan(root, secret)
|
||||
except Exception as error:
|
||||
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
|
||||
return 1
|
||||
for hit in hits:
|
||||
print(f"secret found: {hit.location} ({hit.form})", file=out)
|
||||
if hits:
|
||||
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
|
||||
return 1
|
||||
print(f"no copy of the secret in {scanned} files under {root}", file=out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))
|
||||
199
.github/workflows/test-integration.yml
vendored
Normal file
199
.github/workflows/test-integration.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
name: integration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
schedule:
|
||||
- cron: "5 8-20/3 * * 1-6"
|
||||
timezone: America/Los_Angeles
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
suite:
|
||||
description: Integration group to run
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- providers
|
||||
- extensions
|
||||
- accounting
|
||||
- management
|
||||
- management-replica
|
||||
- database
|
||||
- database-replica
|
||||
- mcp
|
||||
- security
|
||||
- cost
|
||||
- sdk
|
||||
- browser
|
||||
default: all
|
||||
workers:
|
||||
description: Workers per group, blank keeps each group's default
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
integration:
|
||||
name: integration ${{ matrix.suite }}
|
||||
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
|
||||
environment: integration-license
|
||||
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: circle_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--shm-size 1g
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
|
||||
|
||||
- name: Install redis-server 7.2.16
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/redis-bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
|
||||
docker rm "$cid"
|
||||
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
|
||||
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install MCP SDK1 peer
|
||||
if: matrix.suite == 'mcp'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: |
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
tests/e2e/ui/package-lock.json
|
||||
|
||||
- name: Install browser dependencies and build the dashboard
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
(cd ui/litellm-dashboard && npm ci)
|
||||
(cd tests/e2e/ui && npm ci)
|
||||
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
|
||||
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
|
||||
|
||||
- name: Run integration contracts
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SUITE: ${{ matrix.suite }}
|
||||
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
|
||||
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: |
|
||||
case "$SUITE" in
|
||||
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
|
||||
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
|
||||
esac
|
||||
|
||||
- name: Verify every case ran exactly once
|
||||
if: ${{ !cancelled() && matrix.suite != 'browser' }}
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
|
||||
test -n "$record"
|
||||
mkdir -p "$RUNNER_TEMP/shards/0"
|
||||
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
|
||||
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
|
||||
|
||||
- name: Report MCP coverage
|
||||
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
coverage_dir=test-results/integration-mcp/coverage
|
||||
test -f "$coverage_dir/coverage.txt" && exit 0
|
||||
export COVERAGE_FILE="$PWD/$coverage_dir/data"
|
||||
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
|
||||
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
|
||||
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
|
||||
tail -n 1 "$coverage_dir/coverage.txt"
|
||||
|
||||
- name: Check results for the license
|
||||
id: secret-scan
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: python .github/scripts/assert_secret_absent.py test-results
|
||||
|
||||
- name: Upload results
|
||||
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
|
||||
timeout-minutes: 10
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: test-results/
|
||||
retention-days: 14
|
||||
|
|
@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
|
|||
close them, and each bucket is ordered by deadline, so a reap walks the entries
|
||||
that are due rather than the whole queue.
|
||||
|
||||
A bucket whose loop has been collected is dropped, and its clients are left to the
|
||||
collector, as they are when the queue is full.
|
||||
|
||||
The queue holds its clients weakly, so waiting out a grace window never keeps
|
||||
alive anything the collector would have reclaimed first.
|
||||
"""
|
||||
|
|
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
|
|||
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
|
||||
|
||||
_BucketKey = str | int
|
||||
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
|
||||
_LOOP_TOKEN_COUNTER: Final = itertools.count()
|
||||
_LOOP_TOKEN_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -65,12 +72,12 @@ class _PendingClose:
|
|||
collector would otherwise have reclaimed first.
|
||||
|
||||
``needs_loop`` is set for a client whose close is a coroutine; those can only
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_id``.
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_token``.
|
||||
A client that closes synchronously carries neither constraint.
|
||||
"""
|
||||
|
||||
client_ref: "weakref.ref[object]"
|
||||
loop_id: int | None
|
||||
loop_token: int | None
|
||||
needs_loop: bool
|
||||
close_after: float
|
||||
|
||||
|
|
@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
|
|||
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
|
||||
if not pending.needs_loop:
|
||||
return _CLOSABLE_ANYWHERE
|
||||
if pending.loop_id is None:
|
||||
if pending.loop_token is None:
|
||||
return _CLOSABLE_ON_ANY_LOOP
|
||||
return pending.loop_id
|
||||
return pending.loop_token
|
||||
|
||||
|
||||
def _running_loop_id() -> int | None:
|
||||
def running_loop_token() -> int | None:
|
||||
try:
|
||||
return id(asyncio.get_running_loop())
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
try:
|
||||
token: Final[int | None] = _LOOP_TOKENS.get(loop)
|
||||
if token is not None:
|
||||
return token
|
||||
with _LOOP_TOKEN_LOCK:
|
||||
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
|
||||
except TypeError:
|
||||
return id(loop)
|
||||
|
||||
|
||||
def _close_function(client: object) -> Callable[[], object] | None:
|
||||
|
|
@ -186,6 +201,8 @@ class EvictedClientCloser:
|
|||
self._clock = clock
|
||||
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
|
||||
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
|
||||
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
|
||||
self._pending_count = 0
|
||||
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
|
||||
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
|
||||
|
|
@ -216,13 +233,19 @@ class EvictedClientCloser:
|
|||
close_fn: Final = _close_function(client)
|
||||
if close_fn is None:
|
||||
return
|
||||
loop_token: Final = running_loop_token()
|
||||
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
self._discard_dead_loops()
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
if needs_loop and loop_token is not None:
|
||||
self._watch_loop(loop_token)
|
||||
self._enqueue(
|
||||
_PendingClose(
|
||||
client_ref=weakref.ref(client),
|
||||
loop_id=_running_loop_id(),
|
||||
needs_loop=inspect.iscoroutinefunction(close_fn),
|
||||
loop_token=loop_token,
|
||||
needs_loop=needs_loop,
|
||||
close_after=self._clock() + self._grace_seconds,
|
||||
)
|
||||
)
|
||||
|
|
@ -236,7 +259,7 @@ class EvictedClientCloser:
|
|||
if not self._pending_count:
|
||||
return
|
||||
now: Final = self._clock()
|
||||
for pending in self._take_due(_running_loop_id(), now):
|
||||
for pending in self._take_due(running_loop_token(), now):
|
||||
client = pending.client_ref()
|
||||
if client is None:
|
||||
continue
|
||||
|
|
@ -249,6 +272,29 @@ class EvictedClientCloser:
|
|||
def pending_count(self) -> int:
|
||||
return self._pending_count
|
||||
|
||||
def _watch_loop(self, loop_token: int) -> None:
|
||||
if loop_token in self._watched_loop_tokens:
|
||||
return
|
||||
try:
|
||||
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
|
||||
except TypeError:
|
||||
return
|
||||
self._watched_loop_tokens.add(loop_token)
|
||||
|
||||
def _discard_dead_loops(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
|
||||
def _discard_dead_loops_locked(self) -> None:
|
||||
while self._dead_loop_tokens:
|
||||
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
|
||||
|
||||
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
|
||||
self._watched_loop_tokens.discard(loop_token)
|
||||
bucket: Final = self._buckets.pop(loop_token, None)
|
||||
if bucket is not None:
|
||||
self._pending_count -= len(bucket)
|
||||
|
||||
def _enqueue(self, pending: _PendingClose) -> None:
|
||||
"""Append to the entry's bucket, dropping any dead entries it queues behind.
|
||||
|
||||
|
|
@ -257,6 +303,7 @@ class EvictedClientCloser:
|
|||
the front rather than having to be searched for.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
|
||||
while bucket and bucket[0].client_ref() is None:
|
||||
bucket.popleft()
|
||||
|
|
@ -264,9 +311,12 @@ class EvictedClientCloser:
|
|||
bucket.append(pending)
|
||||
self._pending_count += 1
|
||||
|
||||
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
|
||||
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (
|
||||
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
|
||||
)
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
|
||||
|
||||
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
|
|
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
|
|||
self.evicted_client_closer.schedule(evicted)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
def update_cache_key_with_event_loop(self, key):
|
||||
def update_cache_key_with_event_loop(self, key: str) -> str:
|
||||
"""
|
||||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
If none, use the key as is.
|
||||
"""
|
||||
try:
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
stringified_event_loop: Final = str(id(event_loop))
|
||||
return f"{key}-{stringified_event_loop}"
|
||||
except RuntimeError: # handle no current running event loop
|
||||
return key
|
||||
token: Final = running_loop_token()
|
||||
return f"{key}-{token}" if token is not None else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
def _scoped_key(self, key: str, loop_bound: bool) -> str:
|
||||
return self.update_cache_key_with_event_loop(key) if loop_bound else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
|
||||
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
|
||||
if litellm_owned_client:
|
||||
self.evicted_client_closer.mark_owned(value)
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
return super().set_cache(key, value, **kwargs)
|
||||
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
|
||||
|
||||
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
if litellm_owned_client:
|
||||
|
|
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
|
|||
key = self.update_cache_key_with_event_loop(key)
|
||||
return await super().async_set_cache(key, value, **kwargs)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
|
||||
scoped_key: Final = self._scoped_key(key, loop_bound)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
return super().get_cache(key, **kwargs)
|
||||
return super().get_cache(scoped_key, **kwargs)
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
async def async_get_cache(self, key: str, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
|
||||
return await super().async_get_cache(key, **kwargs)
|
||||
|
|
|
|||
|
|
@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
|
|||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
client: Final = get_httpx_client()
|
||||
try:
|
||||
response: Final = client.post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
response: Final = get_httpx_client().post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
finally:
|
||||
client.close()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
|
||||
|
|
@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
|
|||
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
|
||||
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
|
||||
)
|
||||
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
|
||||
hoisted: Final = hoist_additional_tools(
|
||||
input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params)
|
||||
)
|
||||
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
|
||||
if rewritten_types:
|
||||
verbose_logger.warning(
|
||||
|
|
|
|||
|
|
@ -1205,6 +1205,17 @@ class AsyncHTTPHandler:
|
|||
# Mirrors LiteLLMAiohttpTransport._background_close_tasks.
|
||||
_finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes
|
||||
|
||||
@classmethod
|
||||
async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None:
|
||||
try:
|
||||
await client.aclose()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
verbose_logger.debug("Error closing client at finalization: %s", exc)
|
||||
finally:
|
||||
task: Final = asyncio.current_task()
|
||||
if task is not None:
|
||||
cls._finalizer_close_tasks.discard(task)
|
||||
|
||||
@classmethod
|
||||
def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
|
||||
cls._finalizer_close_tasks.discard(task)
|
||||
|
|
@ -1272,8 +1283,8 @@ class AsyncHTTPHandler:
|
|||
# here is the cross-loop path the transport refuses.
|
||||
self._dispose_wrapped_aiohttp_session()
|
||||
return
|
||||
task: Final = loop.create_task(self._client.aclose())
|
||||
cls: Final = type(self)
|
||||
task: Final = loop.create_task(cls._close_client_at_finalization(self._client))
|
||||
cls._finalizer_close_tasks.add(task)
|
||||
task.add_done_callback(cls._on_finalizer_close_done)
|
||||
except Exception:
|
||||
|
|
@ -1973,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
cache = LLMClientCache()
|
||||
setattr(litellm, "in_memory_llm_clients_cache", cache)
|
||||
|
||||
_cached_client: Final = cache.get_cache(_cache_key_name)
|
||||
_cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False)
|
||||
if _cached_client:
|
||||
return _cached_client
|
||||
|
||||
|
|
@ -1989,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
value=_new_client,
|
||||
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
|
||||
litellm_owned_client=True,
|
||||
loop_bound=False,
|
||||
)
|
||||
return _new_client
|
||||
|
||||
|
|
|
|||
|
|
@ -275,6 +275,10 @@ def build_output_token_limit_response(
|
|||
return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
|
||||
|
||||
|
||||
def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool:
|
||||
return client_initialization_params.get("is_async") is not False
|
||||
|
||||
|
||||
class BaseOpenAILLM:
|
||||
"""
|
||||
Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
|
||||
|
|
@ -289,7 +293,9 @@ class BaseOpenAILLM:
|
|||
client_initialization_params=client_initialization_params,
|
||||
client_type=client_type,
|
||||
)
|
||||
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
|
||||
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(
|
||||
_cache_key, loop_bound=_is_loop_bound(client_initialization_params)
|
||||
)
|
||||
return _cached_client
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -330,6 +336,7 @@ class BaseOpenAILLM:
|
|||
value=openai_client,
|
||||
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
|
||||
litellm_owned_client=litellm_owned_client,
|
||||
loop_bound=_is_loop_bound(client_initialization_params),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, o
|
|||
def hoist_additional_tools(
|
||||
input: str | ResponseInputParam,
|
||||
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
|
||||
*,
|
||||
drop_params: bool,
|
||||
) -> HoistedAdditionalTools:
|
||||
existing: Final = tuple(tools or ())
|
||||
if isinstance(input, str):
|
||||
|
|
@ -103,6 +105,8 @@ def hoist_additional_tools(
|
|||
len(items),
|
||||
)
|
||||
remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
|
||||
if not drop_params:
|
||||
return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)
|
||||
unique: Final = tuple(
|
||||
cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts
|
||||
for tool in _unique_tools((*existing, *hoisted))
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ def with_provider_file_ids(
|
|||
}
|
||||
|
||||
|
||||
def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop_params: bool) -> Mapping[str, object]:
|
||||
responses_input: Final = responses_request.get("input")
|
||||
if not isinstance(responses_input, list):
|
||||
return responses_request
|
||||
|
|
@ -122,7 +122,7 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M
|
|||
top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does
|
||||
"tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ())
|
||||
)
|
||||
hoisted: Final = hoist_additional_tools(input_items, top_level_tools)
|
||||
hoisted: Final = hoist_additional_tools(input_items, top_level_tools, drop_params=drop_params)
|
||||
if not hoisted.hoisted:
|
||||
return responses_request
|
||||
return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
|
||||
|
|
@ -196,7 +196,7 @@ async def chat_count_request(
|
|||
chat_body: Final = await _with_previous_turns(
|
||||
responses_batch_body_to_chat_body(
|
||||
_with_compaction_as_summary_text(
|
||||
_with_emulated_file_search(_with_hoisted_additional_tools(responses_request))
|
||||
_with_emulated_file_search(_with_hoisted_additional_tools(responses_request, policy.drop_params))
|
||||
),
|
||||
custom_llm_provider,
|
||||
policy.drop_params,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
|
||||
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
|
||||
|
|
@ -73,7 +74,11 @@ class LiteLLMCompletionTransformationHandler:
|
|||
| BaseResponsesAPIStreamingIterator
|
||||
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
|
||||
):
|
||||
hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
|
||||
hoisted: Final = hoist_additional_tools(
|
||||
input,
|
||||
responses_api_request.get("tools"),
|
||||
drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")),
|
||||
)
|
||||
previous_response_id: Final = responses_api_request.get("previous_response_id")
|
||||
if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
|
||||
return self._async_response_api_handler_after_earlier_searches(
|
||||
|
|
|
|||
|
|
@ -27,8 +27,11 @@ tests. A product bug the test exposes is `pytest.skip("BUG: <symptom>")` at the
|
|||
fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit`
|
||||
|
||||
CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and
|
||||
the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here
|
||||
first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new
|
||||
the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same
|
||||
`run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday,
|
||||
and on a pull request when someone adds the `run-ci` label. Neither runs every pull request, so a new file here
|
||||
first runs on main after its PR merges, or earlier if the PR gets `run-ci`. Wanting a file to run before merge
|
||||
is no reason to change CI, and a new
|
||||
dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and
|
||||
every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that
|
||||
needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Integration contracts
|
||||
|
||||
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
|
||||
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite, and `.github/workflows/test-integration.yml` runs the same groups through `run_integration.sh` on GitHub Actions for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
|
||||
|
||||
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,9 @@ def bedrock_converse_peer(request: Request) -> Reply:
|
|||
)
|
||||
|
||||
|
||||
def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tuple[httpx.Response, tuple[Request, ...]]:
|
||||
def send_to_bedrock_converse(
|
||||
gateway: Gateway, history: list[JsonValue], drop_params: bool = True
|
||||
) -> tuple[httpx.Response, tuple[Request, ...]]:
|
||||
with wire_server(bedrock_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/converse/{BEDROCK_GPT}",
|
||||
|
|
@ -79,7 +81,15 @@ def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tupl
|
|||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": history, "store": False, "cache": {"no-cache": True}}
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": model,
|
||||
"input": history,
|
||||
"store": False,
|
||||
"cache": {"no-cache": True},
|
||||
"drop_params": drop_params,
|
||||
},
|
||||
)
|
||||
return response, wire.drain()
|
||||
|
||||
|
|
@ -113,6 +123,20 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_once(ga
|
|||
assert sorted(names) == ["collaboration__followup_task", "exec_command"], names
|
||||
|
||||
|
||||
def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_as_sent_without_drop_params(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
tools: Final = tool_list_item(function_tool("exec_command"))
|
||||
history: Final[list[JsonValue]] = [tools, user_turn(f"first {marker}"), assistant_turn("ok"), tools]
|
||||
|
||||
response, forwarded = send_to_bedrock_converse(gateway, history, drop_params=False)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert "already defined" in response.text, response.text
|
||||
assert [str(spec["name"]) for spec in converse_tool_specs(forwarded[0])] == ["exec_command", "exec_command"]
|
||||
|
||||
|
||||
def test_newest_copy_of_a_repeated_codex_tool_is_what_bedrock_converse_receives(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
history: Final[list[JsonValue]] = [
|
||||
|
|
@ -195,6 +219,7 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_mantle_once(gate
|
|||
user_turn(f"second {marker}"),
|
||||
],
|
||||
"store": False,
|
||||
"drop_params": True,
|
||||
},
|
||||
)
|
||||
forwarded: Final = wire.drain()
|
||||
|
|
|
|||
20
tests/unit/caching/_event_loop_reuse.py
Normal file
20
tests/unit/caching/_event_loop_reuse.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
from collections.abc import Callable
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import pytest
|
||||
|
||||
LoopResult = TypeVar("LoopResult")
|
||||
|
||||
|
||||
def runs_until_a_loop_address_repeats(
|
||||
run_on_new_loop: Callable[[], tuple[int, LoopResult]],
|
||||
earlier: tuple[tuple[int, LoopResult], ...] = (),
|
||||
limit: int = 50,
|
||||
) -> tuple[tuple[int, LoopResult], ...]:
|
||||
if len(earlier) == limit:
|
||||
pytest.skip(f"none of {limit} collected loops had its address handed to a later loop")
|
||||
latest: Final = run_on_new_loop()
|
||||
runs: Final = (*earlier, latest)
|
||||
if latest[0] in {loop_id for loop_id, _ in earlier}:
|
||||
return runs
|
||||
return runs_until_a_loop_address_repeats(run_on_new_loop, runs, limit)
|
||||
|
|
@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle.
|
|||
import asyncio
|
||||
import gc
|
||||
import weakref
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -18,6 +19,7 @@ from redis.asyncio import ConnectionPool, Redis
|
|||
|
||||
from litellm.caching.evicted_client_closer import EvictedClientCloser
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats
|
||||
|
||||
|
||||
class FakeClock:
|
||||
|
|
@ -41,6 +43,14 @@ class AsyncClient:
|
|||
self.closed = True
|
||||
|
||||
|
||||
class RecordingAsyncClient:
|
||||
def __init__(self) -> None:
|
||||
self.close_calls = 0
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
class SyncClient:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
|
@ -266,6 +276,100 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed():
|
|||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None:
|
||||
clock: Final = FakeClock()
|
||||
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
|
||||
|
||||
async def reap_then_schedule(client: RecordingAsyncClient) -> None:
|
||||
closer.reap()
|
||||
await asyncio.sleep(0)
|
||||
closer.schedule(client)
|
||||
|
||||
def run_on_new_loop() -> tuple[int, RecordingAsyncClient]:
|
||||
client: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(client)
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
loop_id: Final = id(loop)
|
||||
try:
|
||||
loop.run_until_complete(reap_then_schedule(client))
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
return loop_id, client
|
||||
|
||||
runs: Final = runs_until_a_loop_address_repeats(run_on_new_loop)
|
||||
|
||||
assert all(client.close_calls == 0 for _, client in runs)
|
||||
|
||||
|
||||
def test_async_client_queued_on_a_live_loop_is_closed_only_by_that_loop() -> None:
|
||||
clock: Final = FakeClock()
|
||||
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
|
||||
client: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(client)
|
||||
owner_loop: Final = asyncio.new_event_loop()
|
||||
other_loop: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule() -> None:
|
||||
closer.schedule(client)
|
||||
|
||||
async def reap() -> None:
|
||||
closer.reap()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
try:
|
||||
owner_loop.run_until_complete(schedule())
|
||||
other_loop.run_until_complete(reap())
|
||||
closes_after_other_loop_reaped: Final = client.close_calls
|
||||
pending_after_other_loop_reaped: Final = closer.pending_count
|
||||
owner_loop.run_until_complete(reap())
|
||||
finally:
|
||||
other_loop.close()
|
||||
owner_loop.close()
|
||||
|
||||
assert closes_after_other_loop_reaped == 0
|
||||
assert pending_after_other_loop_reaped == 1
|
||||
assert client.close_calls == 1
|
||||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
def test_closes_queued_on_a_collected_loop_free_capacity() -> None:
|
||||
clock: Final = FakeClock()
|
||||
closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock)
|
||||
first: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(first)
|
||||
loop_a: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule_first() -> None:
|
||||
closer.schedule(first)
|
||||
|
||||
try:
|
||||
loop_a.run_until_complete(schedule_first())
|
||||
finally:
|
||||
loop_a.close()
|
||||
del loop_a
|
||||
gc.collect()
|
||||
|
||||
second: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(second)
|
||||
loop_b: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule_and_reap_second() -> None:
|
||||
closer.schedule(second)
|
||||
closer.reap()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
try:
|
||||
loop_b.run_until_complete(schedule_and_reap_second())
|
||||
finally:
|
||||
loop_b.close()
|
||||
|
||||
assert first.close_calls == 0
|
||||
assert second.close_calls == 1
|
||||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
|
||||
clock = FakeClock()
|
||||
|
|
@ -289,8 +393,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_client_evicted_on_another_event_loop_is_left_alone():
|
||||
"""Closing a client bound to a different loop would schedule work on that loop."""
|
||||
async def test_a_client_evicted_on_a_collected_loop_is_dropped():
|
||||
clock = FakeClock()
|
||||
closer = make_closer(clock)
|
||||
client = AsyncClient()
|
||||
|
|
@ -310,7 +413,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone():
|
|||
await asyncio.sleep(0.05)
|
||||
|
||||
assert client.closed is False
|
||||
assert closer.pending_count == 1
|
||||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247
|
|||
|
||||
import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref
|
||||
import warnings
|
||||
from typing import Final
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
|
@ -29,6 +30,41 @@ from litellm.llms.custom_httpx.http_handler import(
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats
|
||||
|
||||
|
||||
async def _get_and_set_cache(cache: LLMClientCache) -> object | None:
|
||||
cached: Final = await cache.async_get_cache("client")
|
||||
await cache.async_set_cache("client", object())
|
||||
return cached
|
||||
|
||||
|
||||
def _cache_result_on_new_loop(cache: LLMClientCache) -> tuple[int, object | None]:
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
loop_id: Final = id(loop)
|
||||
try:
|
||||
result: Final = loop.run_until_complete(_get_and_set_cache(cache))
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
return loop_id, result
|
||||
|
||||
|
||||
async def _get_cache_key(cache: LLMClientCache) -> str:
|
||||
return cache.update_cache_key_with_event_loop("client")
|
||||
|
||||
|
||||
def _cache_key_on_new_loop(cache: LLMClientCache) -> tuple[int, str]:
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
loop_id: Final = id(loop)
|
||||
try:
|
||||
key: Final = loop.run_until_complete(_get_cache_key(cache))
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
return loop_id, key
|
||||
|
||||
|
||||
class MockAsyncClient:
|
||||
|
|
@ -51,6 +87,44 @@ class MockSyncClient:
|
|||
self.closed = True
|
||||
|
||||
|
||||
def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None:
|
||||
cache: Final = LLMClientCache()
|
||||
runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_result_on_new_loop(cache))
|
||||
|
||||
assert all(result is None for _, result in runs)
|
||||
|
||||
|
||||
def test_async_client_cache_keys_differ_for_every_collected_loop() -> None:
|
||||
cache: Final = LLMClientCache()
|
||||
runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_key_on_new_loop(cache))
|
||||
keys: Final = tuple(key for _, key in runs)
|
||||
|
||||
assert len(set(keys)) == len(keys)
|
||||
|
||||
|
||||
class _LoopWithoutWeakReferences:
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
def _cache_key_while_running(cache: LLMClientCache, loop: _LoopWithoutWeakReferences) -> str:
|
||||
asyncio._set_running_loop(loop)
|
||||
try:
|
||||
return cache.update_cache_key_with_event_loop("client")
|
||||
finally:
|
||||
asyncio._set_running_loop(None)
|
||||
|
||||
|
||||
def test_cache_keys_stay_per_loop_on_loops_without_weak_references() -> None:
|
||||
cache: Final = LLMClientCache()
|
||||
loop_a: Final = _LoopWithoutWeakReferences()
|
||||
loop_b: Final = _LoopWithoutWeakReferences()
|
||||
|
||||
first_key_on_a: Final = _cache_key_while_running(cache, loop_a)
|
||||
|
||||
assert _cache_key_while_running(cache, loop_a) == first_key_on_a
|
||||
assert _cache_key_while_running(cache, loop_b) != first_key_on_a
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_key_does_not_close_async_client():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3281,7 +3281,7 @@ ARIZE_DEST = OtelDestination(
|
|||
|
||||
class _ExporterCapture:
|
||||
"""Stands an in-memory exporter in for every exporter the provider builds, keyed
|
||||
by the headers it would have sent (``None`` for a headerless exporter), so a test
|
||||
by the headers it would have sent (``None`` for the stdout placeholder), so a test
|
||||
reads what each account received rather than which processors were wired."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
|
|
@ -3445,13 +3445,13 @@ class TestLangfuseDestinationFields:
|
|||
) -> None:
|
||||
"""With ``langfuse_otel`` only in a key's or team's logging, the startup logger is the
|
||||
generic one; the key's project still gets the user, session and tags, and the
|
||||
operator gets no unconfigured stdout copy."""
|
||||
operator's stdout placeholder still gets no Langfuse request metadata."""
|
||||
credential_less_proxy(monkeypatch)
|
||||
capture: Final = _ExporterCapture()
|
||||
for kind in ("otlp_grpc", "otlp_http", "console"):
|
||||
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
|
||||
config: Final = OpenTelemetryV2Config()
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True)
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
|
||||
attach_tenant_fan_out(provider, config)
|
||||
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
|
||||
call: Final = replace(
|
||||
|
|
@ -3477,7 +3477,7 @@ class TestLangfuseDestinationFields:
|
|||
"langfuse.trace.tags": ("agent:orders",),
|
||||
}
|
||||
assert "langfuse.observation.metadata" in to_langfuse
|
||||
assert None not in received
|
||||
assert "langfuse.observation.metadata" not in received[None]
|
||||
|
||||
@staticmethod
|
||||
def _received_by_account(
|
||||
|
|
@ -3491,7 +3491,7 @@ class TestLangfuseDestinationFields:
|
|||
capture: Final = _ExporterCapture()
|
||||
for kind in ("otlp_grpc", "otlp_http", "console"):
|
||||
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True)
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
|
||||
attach_tenant_fan_out(provider, config)
|
||||
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
|
||||
call: Final = replace(
|
||||
|
|
@ -3515,11 +3515,7 @@ class TestLangfuseDestinationFields:
|
|||
) -> None:
|
||||
"""The operator's own exporter and the key's Arize destination get the model call as they would with no
|
||||
Langfuse destination on the request."""
|
||||
received: Final = self._received_by_account(
|
||||
monkeypatch,
|
||||
OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]),
|
||||
(LANGFUSE_DEST, ARIZE_DEST),
|
||||
)
|
||||
received: Final = self._received_by_account(monkeypatch, OpenTelemetryV2Config(), (LANGFUSE_DEST, ARIZE_DEST))
|
||||
|
||||
assert received[LANGFUSE_DEST.header_string()]["user.id"] == "user-42"
|
||||
assert {
|
||||
|
|
@ -3532,9 +3528,7 @@ class TestLangfuseDestinationFields:
|
|||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
received: Final = self._received_by_account(
|
||||
monkeypatch,
|
||||
OpenTelemetryV2Config(mapper_names=["genai", "langfuse"], exporters=[ExporterSpec(kind="console")]),
|
||||
(LANGFUSE_DEST,),
|
||||
monkeypatch, OpenTelemetryV2Config(mapper_names=["genai", "langfuse"]), (LANGFUSE_DEST,)
|
||||
)
|
||||
|
||||
assert {key: received[None].get(key) for key in ("user.id", "session.id", "langfuse.trace.tags")} == {
|
||||
|
|
@ -3632,10 +3626,7 @@ class TestLangfuseDestinationFields:
|
|||
assert arize is not None
|
||||
|
||||
received: Final = self._received_by_account(
|
||||
monkeypatch,
|
||||
OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]),
|
||||
(destination, arize),
|
||||
with_content=True,
|
||||
monkeypatch, OpenTelemetryV2Config(), (destination, arize), with_content=True
|
||||
)
|
||||
|
||||
assert {headers: "langfuse.observation.input" in attributes for headers, attributes in received.items()} == {
|
||||
|
|
@ -3650,8 +3641,8 @@ class TestLangfuseDestinationFields:
|
|||
capture: Final = _ExporterCapture()
|
||||
for kind in ("otlp_grpc", "otlp_http", "console"):
|
||||
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
|
||||
config: Final = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")])
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True)
|
||||
config: Final = OpenTelemetryV2Config()
|
||||
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
|
||||
attach_tenant_fan_out(provider, config)
|
||||
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
|
||||
destination: Final = destination_for(
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.integrations.agentops.agentops import AgentOps, AgentOpsConfig
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -111,3 +115,17 @@ def test_agentops_initialization_no_auth():
|
|||
assert agentops.resource_attributes["deployment.environment"] == "test_env"
|
||||
assert agentops.resource_attributes["telemetry.sdk.name"] == "agentops"
|
||||
assert "project.id" not in agentops.resource_attributes
|
||||
|
||||
|
||||
def test_auth_fetch_leaves_the_shared_sync_client_open_for_other_callers(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
auth_endpoint: Final = "https://agentops.example/v3/auth/token"
|
||||
respx_mock.post(auth_endpoint).respond(json={"token": "jwt", "project_id": "project-1"})
|
||||
client_another_caller_is_using: Final = get_httpx_client().client
|
||||
|
||||
agentops: Final = AgentOps(config=AgentOpsConfig(api_key="key", auth_endpoint=auth_endpoint))
|
||||
|
||||
assert agentops.resource_attributes["project.id"] == "project-1"
|
||||
assert not client_another_caller_is_using.is_closed
|
||||
|
|
|
|||
|
|
@ -1388,6 +1388,55 @@ async def test_finalizer_with_running_loop_schedules_close_and_holds_task_ref():
|
|||
assert not (AsyncHTTPHandler._finalizer_close_tasks & scheduled)
|
||||
|
||||
|
||||
async def _drop_unused_handler() -> None:
|
||||
AsyncHTTPHandler(timeout=61.0)
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_finalizer_close_does_not_keep_a_loop_closed_right_after_it_alive():
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
loop.run_until_complete(_drop_unused_handler())
|
||||
loop.close()
|
||||
closed_loop: Final = weakref.ref(loop)
|
||||
del loop
|
||||
gc.collect()
|
||||
|
||||
assert closed_loop() is None
|
||||
|
||||
|
||||
class _TransportFailingToClose(httpx.AsyncBaseTransport):
|
||||
async def aclose(self) -> None:
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
|
||||
async def _drop_handler_whose_close_fails() -> None:
|
||||
AsyncHTTPHandler(timeout=61.0, transport=_TransportFailingToClose())
|
||||
gc.collect()
|
||||
|
||||
|
||||
def test_failed_finalizer_close_in_a_loops_last_step_reports_no_unretrieved_exception():
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
exception_handler: Final = MagicMock()
|
||||
loop.set_exception_handler(exception_handler)
|
||||
loop.run_until_complete(_drop_handler_whose_close_fails())
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
|
||||
exception_handler.assert_not_called()
|
||||
|
||||
|
||||
async def _sync_client_with_timeout_62() -> HTTPHandler:
|
||||
return get_httpx_client(params={"timeout": 62.0})
|
||||
|
||||
|
||||
def test_sync_client_is_shared_by_short_lived_loops_and_callers_outside_a_loop():
|
||||
built_in_first_loop: Final = asyncio.run(_sync_client_with_timeout_62())
|
||||
|
||||
assert asyncio.run(_sync_client_with_timeout_62()) is built_in_first_loop
|
||||
assert get_httpx_client(params={"timeout": 62.0}) is built_in_first_loop
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_close_helper_respects_session_ownership():
|
||||
"""The loop-less fallback closes only sessions the transport owns; a
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -154,6 +155,28 @@ def test_get_openai_client_initialization_param_fields(client_type):
|
|||
assert "self" not in result
|
||||
|
||||
|
||||
def _loop_test_client_params(is_async: bool) -> dict[str, object]:
|
||||
return {"api_key": "sk-loop-scope-test", "is_async": is_async}
|
||||
|
||||
|
||||
async def _cache_openai_client(client: openai.OpenAI | openai.AsyncOpenAI, is_async: bool) -> None:
|
||||
BaseOpenAILLM.set_cached_openai_client(client, "openai", _loop_test_client_params(is_async))
|
||||
|
||||
|
||||
async def _cached_openai_client(is_async: bool) -> object:
|
||||
return BaseOpenAILLM.get_cached_openai_client(_loop_test_client_params(is_async), "openai")
|
||||
|
||||
|
||||
def test_sync_openai_client_cached_in_one_loop_is_reused_by_the_next_while_async_is_not():
|
||||
sync_client: Final = openai.OpenAI(api_key="sk-loop-scope-test")
|
||||
async_client: Final = openai.AsyncOpenAI(api_key="sk-loop-scope-test")
|
||||
asyncio.run(_cache_openai_client(sync_client, is_async=False))
|
||||
asyncio.run(_cache_openai_client(async_client, is_async=True))
|
||||
|
||||
assert asyncio.run(_cached_openai_client(is_async=False)) is sync_client
|
||||
assert asyncio.run(_cached_openai_client(is_async=True)) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_type", ["openai", "azure"])
|
||||
def test_get_openai_client_cache_key(client_type):
|
||||
"""Verify get_openai_client_cache_key doesn't raise on tuple + tuple concatenation."""
|
||||
|
|
|
|||
|
|
@ -95,8 +95,8 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once() -> None:
|
||||
policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset())
|
||||
async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once_under_drop_params() -> None:
|
||||
policy: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset())
|
||||
tool_list: Final[dict[str, JsonValue]] = {"type": "additional_tools", "tools": [_WEATHER_TOOL]}
|
||||
|
||||
repeated: Final = await _claude_count(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ _USER_MESSAGE = {"type": "message", "role": "user", "content": "Run ls"}
|
|||
|
||||
|
||||
def test_string_input_passes_through_with_existing_tools():
|
||||
hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL])
|
||||
hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL], drop_params=False)
|
||||
|
||||
assert hoisted.input == "hello"
|
||||
assert hoisted.tools == (_TOP_LEVEL_TOOL,)
|
||||
|
|
@ -25,7 +25,7 @@ def test_string_input_passes_through_with_existing_tools():
|
|||
def test_input_without_additional_tools_items_is_returned_untouched():
|
||||
request_input = [_USER_MESSAGE]
|
||||
|
||||
hoisted = hoist_additional_tools(request_input, None)
|
||||
hoisted = hoist_additional_tools(request_input, None, drop_params=False)
|
||||
|
||||
assert hoisted.input is request_input
|
||||
assert hoisted.tools == ()
|
||||
|
|
@ -39,7 +39,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_
|
|||
{"type": "additional_tools", "id": "at_2", "role": "developer", "tools": [_WAIT_TOOL]},
|
||||
]
|
||||
|
||||
hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL])
|
||||
hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False)
|
||||
|
||||
assert hoisted.input == [_USER_MESSAGE]
|
||||
assert hoisted.tools == (_TOP_LEVEL_TOOL, _EXEC_TOOL, _WAIT_TOOL)
|
||||
|
|
@ -49,7 +49,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_
|
|||
def test_additional_tools_item_without_a_tools_list_is_stripped_and_contributes_nothing():
|
||||
request_input = [{"type": "additional_tools", "id": "at_1", "role": "developer", "tools": "exec"}, _USER_MESSAGE]
|
||||
|
||||
hoisted = hoist_additional_tools(request_input, None)
|
||||
hoisted = hoist_additional_tools(request_input, None, drop_params=False)
|
||||
|
||||
assert hoisted.input == [_USER_MESSAGE]
|
||||
assert hoisted.tools == ()
|
||||
|
|
@ -77,7 +77,7 @@ def test_tool_list_repeated_on_every_turn_keeps_each_tool_once_with_the_newest_d
|
|||
_USER_MESSAGE,
|
||||
]
|
||||
|
||||
hoisted: Final = hoist_additional_tools(request_input, None)
|
||||
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
|
||||
|
||||
assert hoisted.tools == (collaboration, _function("exec", "new"))
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ def test_namespace_sent_in_several_turns_keeps_every_distinct_member_once() -> N
|
|||
_tool_list(_namespace("mcp__docs__", _function("fetch", "new"), _function("list"))),
|
||||
]
|
||||
|
||||
hoisted: Final = hoist_additional_tools(request_input, None)
|
||||
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
|
||||
|
||||
assert hoisted.tools == (
|
||||
_namespace("mcp__docs__", _function("search"), _function("fetch", "new"), _function("list")),
|
||||
|
|
@ -109,7 +109,7 @@ def test_unnamed_tools_collapse_only_when_identical() -> None:
|
|||
}
|
||||
request_input: Final = [_tool_list(docs_server, wiki_server), _USER_MESSAGE, _tool_list(docs_server)]
|
||||
|
||||
hoisted: Final = hoist_additional_tools(request_input, None)
|
||||
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
|
||||
|
||||
assert hoisted.tools == (docs_server, wiki_server)
|
||||
|
||||
|
|
@ -117,6 +117,20 @@ def test_unnamed_tools_collapse_only_when_identical() -> None:
|
|||
def test_top_level_tool_repeated_in_a_tool_list_item_is_kept_once() -> None:
|
||||
request_input: Final = [_tool_list(_TOP_LEVEL_TOOL, _WAIT_TOOL), _USER_MESSAGE]
|
||||
|
||||
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL])
|
||||
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=True)
|
||||
|
||||
assert hoisted.tools == (_TOP_LEVEL_TOOL, _WAIT_TOOL)
|
||||
|
||||
|
||||
def test_tool_list_repeated_on_every_turn_is_sent_as_is_without_drop_params() -> None:
|
||||
request_input: Final = [
|
||||
_tool_list(_function("exec", "old")),
|
||||
_USER_MESSAGE,
|
||||
_tool_list(_function("exec", "new")),
|
||||
_USER_MESSAGE,
|
||||
]
|
||||
|
||||
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False)
|
||||
|
||||
assert hoisted.tools == (_TOP_LEVEL_TOOL, _function("exec", "old"), _function("exec", "new"))
|
||||
assert hoisted.input == [_USER_MESSAGE, _USER_MESSAGE]
|
||||
|
|
|
|||
124
tests/unit/test_assert_secret_absent.py
Normal file
124
tests/unit/test_assert_secret_absent.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import base64
|
||||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from collections.abc import Callable, Mapping
|
||||
from itertools import chain, product
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "assert_secret_absent.py"
|
||||
_spec: Final = importlib.util.spec_from_file_location("assert_secret_absent", _MODULE_PATH)
|
||||
scanner: Final = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_spec.name] = scanner
|
||||
_spec.loader.exec_module(scanner)
|
||||
|
||||
SECRET: Final = "lic/ense+KEY:9f8e~~??>>~?7d6c5b4a3210-zz"
|
||||
PREFIXES: Final = ("", "x", "xy")
|
||||
ENCODERS: Final = (base64.b64encode, base64.urlsafe_b64encode)
|
||||
|
||||
|
||||
def _zip(members: Mapping[str, bytes], compression: int = zipfile.ZIP_DEFLATED) -> bytes:
|
||||
buffer: Final = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w", compression) as archive:
|
||||
for name, data in members.items():
|
||||
archive.writestr(name, data)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _blob(prefix: str) -> bytes:
|
||||
return f"{prefix}{SECRET}tail".encode()
|
||||
|
||||
|
||||
def _planted_forms() -> tuple[str, ...]:
|
||||
encoded: Final = tuple(encode(_blob(prefix)).decode() for encode, prefix in product(ENCODERS, PREFIXES))
|
||||
return (SECRET, urllib.parse.quote(SECRET, safe=""), *encoded)
|
||||
|
||||
|
||||
def _assert_nothing_leaked(output: str) -> None:
|
||||
windows: Final = frozenset(
|
||||
chain.from_iterable((form[i : i + 8] for i in range(len(form) - 7)) for form in _planted_forms())
|
||||
)
|
||||
assert not [window for window in windows if window in output]
|
||||
|
||||
|
||||
def _run(root: Path, secret: str = SECRET) -> tuple[int, str]:
|
||||
out: Final = io.StringIO()
|
||||
code: Final = scanner.main((str(root),), {scanner.SECRET_VARIABLE: secret}, out)
|
||||
return code, out.getvalue()
|
||||
|
||||
|
||||
def _encoded_blob(prefix: str, encode: Callable[[bytes], bytes]) -> bytes:
|
||||
return b"trace " + encode(_blob(prefix)) + b" end"
|
||||
|
||||
|
||||
QUOTED: Final = urllib.parse.quote(SECRET, safe="")
|
||||
PLACEMENTS: Final = MappingProxyType(
|
||||
{
|
||||
"raw in a log": {"proxy.log": f"boot with key {SECRET} ok".encode()},
|
||||
"url-encoded in a url": {"upstream.log": f"GET /verify_license/{QUOTED}".encode()},
|
||||
"base64 aligned": {"a.json": _encoded_blob("", base64.b64encode)},
|
||||
"base64 one byte in": {"a.json": _encoded_blob("x", base64.b64encode)},
|
||||
"base64 two bytes in": {"a.json": _encoded_blob("xy", base64.b64encode)},
|
||||
"base64url two bytes in": {"a.json": _encoded_blob("xy", base64.urlsafe_b64encode)},
|
||||
"inside a zip member": {"trace.zip": _zip({"network.json": f'{{"auth": "{SECRET}"}}'.encode()})},
|
||||
"inside a nested zip": {"outer.zip": _zip({"inner.zip": _zip({"env.txt": SECRET.encode()})})},
|
||||
"named after it in a nested zip": {
|
||||
"outer.zip": _zip({"inner.zip": _zip({f"dump-{QUOTED}.txt": SECRET.encode()})})
|
||||
},
|
||||
"in a file name": {f"run-{QUOTED}.log": b"nothing inside"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("placement", PLACEMENTS)
|
||||
def test_every_planted_copy_blocks_the_upload_without_printing_the_secret(tmp_path: Path, placement: str) -> None:
|
||||
worker: Final = tmp_path / "gw0"
|
||||
worker.mkdir()
|
||||
for name, data in PLACEMENTS[placement].items():
|
||||
(worker / name).write_bytes(data)
|
||||
code, output = _run(tmp_path)
|
||||
assert code == 1
|
||||
assert "secret found:" in output
|
||||
_assert_nothing_leaked(output)
|
||||
|
||||
|
||||
def test_clean_results_pass_and_count_every_file(tmp_path: Path) -> None:
|
||||
(tmp_path / "junit.xml").write_text("<testsuite/>")
|
||||
(tmp_path / "gw0").mkdir()
|
||||
(tmp_path / "gw0" / "proxy.log").write_text("LITELLM_LICENSE is not set")
|
||||
code, output = _run(tmp_path)
|
||||
assert code == 0
|
||||
assert "no copy of the secret in 2 files" in output
|
||||
|
||||
|
||||
def test_an_unset_secret_has_nothing_to_find(tmp_path: Path) -> None:
|
||||
(tmp_path / "proxy.log").write_text(SECRET)
|
||||
assert _run(tmp_path, secret="")[0] == 0
|
||||
|
||||
|
||||
def test_a_symlinked_directory_fails_closed(tmp_path: Path) -> None:
|
||||
outside: Final = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
results: Final = tmp_path / "results"
|
||||
results.mkdir()
|
||||
(results / "linked").symlink_to(outside, target_is_directory=True)
|
||||
code, output = _run(results)
|
||||
assert code == 1
|
||||
assert "directory symlink" in output
|
||||
|
||||
|
||||
def test_an_unreadable_archive_fails_without_echoing_its_contents(tmp_path: Path) -> None:
|
||||
archive: Final = _zip({"env.txt": f"LITELLM_LICENSE={SECRET}".encode()}, zipfile.ZIP_STORED)
|
||||
at: Final = archive.index(SECRET.encode())
|
||||
(tmp_path / "trace.zip").write_bytes(archive[:at] + bytes([archive[at] ^ 0xFF]) + archive[at + 1 :])
|
||||
code, output = _run(tmp_path)
|
||||
assert code == 1
|
||||
assert "scan failed with BadZipFile" in output
|
||||
_assert_nothing_leaked(output)
|
||||
Loading…
Add table
Reference in a new issue