diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 00b3c5f8fdd..1380b07de53 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -if [ "${GITHUB_ACTIONS:-}" = true ]; then - echo "Integration contracts are owned by CircleCI" >&2 - exit 1 -fi - suite="${1:?integration suite required}" mode="${2:-standard}" side="${3:-}" +requested_workers="${INTEGRATION_WORKERS:-}" +license_env=() +if [ -n "${LITELLM_LICENSE:-}" ]; then + license_env=("LITELLM_LICENSE=$LITELLM_LICENSE") +fi if [ "$mode" = replica ]; then results="test-results/integration-${suite}-replica" elif [ "$mode" = parity ]; then @@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out" test -f "$LITELLM_UI_PATH/index.html" fi -export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')" +export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED" uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1 @@ -107,7 +107,7 @@ sudo iptables -N integration_only guard_created=true sudo iptables -A integration_only -o lo -j ACCEPT sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT -for service in postgres-db redis-cache; do +for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")" sudo iptables -A integration_only -d "$address" -j ACCEPT done @@ -145,6 +145,9 @@ fi if [ "$suite" = mcp ]; then export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1 fi +if [ -n "$requested_workers" ]; then + export INTEGRATION_WORKERS="$requested_workers" +fi isolated_workers=false if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then isolated_workers=true @@ -185,7 +188,7 @@ start_proxy() { "${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \ - LITELLM_LICENSE="${LITELLM_LICENSE:-}" \ + "${license_env[@]}" \ LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \ AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \ "${proxy_command[@]}" --config tests/integration/proxy_config.yaml \ @@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ "${shared_services[@]}" "${worker_coverage[@]}" \ INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \ LITELLM_MODE=PRODUCTION \ - LITELLM_LICENSE="${LITELLM_LICENSE:-}" \ + "${license_env[@]}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \ diff --git a/.github/scripts/assert_secret_absent.py b/.github/scripts/assert_secret_absent.py new file mode 100644 index 00000000000..2d001e833ac --- /dev/null +++ b/.github/scripts/assert_secret_absent.py @@ -0,0 +1,119 @@ +import base64 +import io +import os +import sys +import urllib.parse +import zipfile +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from functools import reduce +from itertools import chain +from pathlib import Path +from types import MappingProxyType +from typing import Final, TextIO + +SECRET_VARIABLE: Final = "SCANNED_SECRET" +MIN_ENCODED_LENGTH: Final = 12 +MAX_ARCHIVE_DEPTH: Final = 3 +MASK: Final = "***" + +Forms = tuple[tuple[str, str], ...] + + +@dataclass(frozen=True, slots=True) +class Hit: + location: str + form: str + + +def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]: + for offset in range(3): + chunk: Final = raw[offset:] + yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode() + + +def secret_forms(secret: str) -> Forms: + encoded: Final = ( + ("url-encoded", urllib.parse.quote(secret, safe="")), + *_aligned_encodings(secret.encode(), "base64", base64.b64encode), + *_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode), + ) + by_value: Final = MappingProxyType( + {value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH} + ) + return (("raw", secret), *((name, value) for value, name in by_value.items())) + + +def mask(text: str, forms: Forms) -> str: + return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text) + + +def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]: + for form, value in forms: + if value in name: + yield Hit(mask(location, forms), f"{form} in name") + + +def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]: + for form, value in forms: + if value.encode() in data: + yield Hit(mask(location, forms), form) + if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)): + return + with zipfile.ZipFile(io.BytesIO(data)) as archive: + for member in archive.infolist(): + if member.is_dir(): + continue + yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1) + + +def _walk(root: Path) -> Iterator[tuple[Path, bool]]: + for directory, subdirectories, files in os.walk(root): + yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink()) + yield from ((Path(directory, name), False) for name in files) + + +def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]: + location: Final = str(path.relative_to(root)) + if linked_directory: + yield Hit(mask(location, forms), "directory symlink, not scanned") + return + yield from _name_hits(location, location, forms) + yield from _content_hits(location, path.read_bytes(), forms, 0) + + +def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]: + forms: Final = secret_forms(secret) + entries: Final = tuple(_walk(root)) + hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries)) + return len(entries), hits + + +def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int: + if len(argv) != 1: + print(f"usage: assert_secret_absent.py , with the secret in ${SECRET_VARIABLE}", file=out) + return 2 + secret: Final = environ.get(SECRET_VARIABLE, "") + root: Final = Path(argv[0]) + if not secret: + print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out) + return 0 + if not root.is_dir(): + print(f"{root} is not a directory, nothing to scan", file=out) + return 0 + try: + scanned, hits = scan(root, secret) + except Exception as error: + print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out) + return 1 + for hit in hits: + print(f"secret found: {hit.location} ({hit.form})", file=out) + if hits: + print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out) + return 1 + print(f"no copy of the secret in {scanned} files under {root}", file=out) + return 0 + + +if __name__ == "__main__": + sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout)) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml new file mode 100644 index 00000000000..0c5b70a67fe --- /dev/null +++ b/.github/workflows/test-integration.yml @@ -0,0 +1,199 @@ +name: integration + +on: + pull_request: + types: [labeled] + schedule: + - cron: "5 8-20/3 * * 1-6" + timezone: America/Los_Angeles + workflow_dispatch: + inputs: + suite: + description: Integration group to run + type: choice + options: + - all + - providers + - extensions + - accounting + - management + - management-replica + - database + - database-replica + - mcp + - security + - cost + - sdk + - browser + default: all + workers: + description: Workers per group, blank keeps each group's default + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + integration: + name: integration ${{ matrix.suite }} + if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci' + environment: integration-license + runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }} + env: + UV_PYTHON: "3.12" + INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }} + services: + postgres: + image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 + command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: circle_test + ports: + - 5432:5432 + options: >- + --shm-size 1g + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + redis: + image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + timeout-minutes: 3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ env.UV_PYTHON }} + + - name: Set up uv + timeout-minutes: 3 + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + timeout-minutes: 5 + uses: ./.github/actions/cache-cargo-build + + - name: Cache Prisma binaries + timeout-minutes: 5 + uses: ./.github/actions/cache-prisma-binaries + + - name: Install dependencies + timeout-minutes: 20 + run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras + + - name: Install redis-server 7.2.16 + timeout-minutes: 3 + run: | + mkdir -p "$RUNNER_TEMP/redis-bin" + cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" + docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server" + docker rm "$cid" + "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16' + echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH" + + - name: Install MCP SDK1 peer + if: matrix.suite == 'mcp' + timeout-minutes: 5 + run: | + uv venv --python 3.12 .venv-mcp-peer + uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' + echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV" + + - name: Set up Node.js + if: matrix.suite == 'browser' + timeout-minutes: 5 + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version-file: ui/litellm-dashboard/.nvmrc + cache: "npm" + cache-dependency-path: | + ui/litellm-dashboard/package-lock.json + tests/e2e/ui/package-lock.json + + - name: Install browser dependencies and build the dashboard + if: matrix.suite == 'browser' + timeout-minutes: 20 + run: | + (cd ui/litellm-dashboard && npm ci) + (cd tests/e2e/ui && npm ci) + sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \ + node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium + node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium + (cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build) + + - name: Run integration contracts + timeout-minutes: 45 + env: + SUITE: ${{ matrix.suite }} + INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }} + LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} + run: | + case "$SUITE" in + *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;; + *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; + esac + + - name: Verify every case ran exactly once + if: ${{ !cancelled() && matrix.suite != 'browser' }} + timeout-minutes: 2 + run: | + record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)" + test -n "$record" + mkdir -p "$RUNNER_TEMP/shards/0" + cp "$record" "$RUNNER_TEMP/shards/0/execution.json" + .venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1 + + - name: Report MCP coverage + if: ${{ !cancelled() && matrix.suite == 'mcp' }} + timeout-minutes: 5 + run: | + coverage_dir=test-results/integration-mcp/coverage + test -f "$coverage_dir/coverage.txt" && exit 0 + export COVERAGE_FILE="$PWD/$coverage_dir/data" + .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml + .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt" + .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html" + tail -n 1 "$coverage_dir/coverage.txt" + + - name: Check results for the license + id: secret-scan + if: ${{ !cancelled() }} + timeout-minutes: 5 + env: + SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }} + run: python .github/scripts/assert_secret_absent.py test-results + + - name: Upload results + if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }} + timeout-minutes: 10 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }} + path: test-results/ + retention-days: 14 diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index 555f6a25324..862f9ddb82e 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to close them, and each bucket is ordered by deadline, so a reap walks the entries that are due rather than the whole queue. +A bucket whose loop has been collected is dropped, and its clients are left to the +collector, as they are when the queue is full. + The queue holds its clients weakly, so waiting out a grace window never keeps alive anything the collector would have reclaimed first. """ @@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first. import asyncio import contextlib import inspect +import itertools import threading import time import weakref @@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere" _CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop" _BucketKey = str | int +_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary() +_LOOP_TOKEN_COUNTER: Final = itertools.count() +_LOOP_TOKEN_LOCK: Final = threading.Lock() @dataclass(frozen=True, slots=True) @@ -65,12 +72,12 @@ class _PendingClose: collector would otherwise have reclaimed first. ``needs_loop`` is set for a client whose close is a coroutine; those can only - be closed from the event loop they were evicted on, recorded in ``loop_id``. + be closed from the event loop they were evicted on, recorded in ``loop_token``. A client that closes synchronously carries neither constraint. """ client_ref: "weakref.ref[object]" - loop_id: int | None + loop_token: int | None needs_loop: bool close_after: float @@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey: """Which reaps can close this entry: any at all, any running a loop, or one loop's.""" if not pending.needs_loop: return _CLOSABLE_ANYWHERE - if pending.loop_id is None: + if pending.loop_token is None: return _CLOSABLE_ON_ANY_LOOP - return pending.loop_id + return pending.loop_token -def _running_loop_id() -> int | None: +def running_loop_token() -> int | None: try: - return id(asyncio.get_running_loop()) + loop: Final = asyncio.get_running_loop() except RuntimeError: return None + try: + token: Final[int | None] = _LOOP_TOKENS.get(loop) + if token is not None: + return token + with _LOOP_TOKEN_LOCK: + return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER)) + except TypeError: + return id(loop) def _close_function(client: object) -> Callable[[], object] | None: @@ -186,6 +201,8 @@ class EvictedClientCloser: self._clock = clock self._owned: weakref.WeakSet[object] = weakref.WeakSet() self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues + self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock + self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded self._pending_count = 0 self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes @@ -216,13 +233,19 @@ class EvictedClientCloser: close_fn: Final = _close_function(client) if close_fn is None: return + loop_token: Final = running_loop_token() + needs_loop: Final = inspect.iscoroutinefunction(close_fn) if self._pending_count >= self._max_pending: - return + self._discard_dead_loops() + if self._pending_count >= self._max_pending: + return + if needs_loop and loop_token is not None: + self._watch_loop(loop_token) self._enqueue( _PendingClose( client_ref=weakref.ref(client), - loop_id=_running_loop_id(), - needs_loop=inspect.iscoroutinefunction(close_fn), + loop_token=loop_token, + needs_loop=needs_loop, close_after=self._clock() + self._grace_seconds, ) ) @@ -236,7 +259,7 @@ class EvictedClientCloser: if not self._pending_count: return now: Final = self._clock() - for pending in self._take_due(_running_loop_id(), now): + for pending in self._take_due(running_loop_token(), now): client = pending.client_ref() if client is None: continue @@ -249,6 +272,29 @@ class EvictedClientCloser: def pending_count(self) -> int: return self._pending_count + def _watch_loop(self, loop_token: int) -> None: + if loop_token in self._watched_loop_tokens: + return + try: + weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token) + except TypeError: + return + self._watched_loop_tokens.add(loop_token) + + def _discard_dead_loops(self) -> None: + with self._queue_lock: + self._discard_dead_loops_locked() + + def _discard_dead_loops_locked(self) -> None: + while self._dead_loop_tokens: + self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft()) + + def _discard_dead_loop_token_locked(self, loop_token: int) -> None: + self._watched_loop_tokens.discard(loop_token) + bucket: Final = self._buckets.pop(loop_token, None) + if bucket is not None: + self._pending_count -= len(bucket) + def _enqueue(self, pending: _PendingClose) -> None: """Append to the entry's bucket, dropping any dead entries it queues behind. @@ -257,6 +303,7 @@ class EvictedClientCloser: the front rather than having to be searched for. """ with self._queue_lock: + self._discard_dead_loops_locked() bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque()) while bucket and bucket[0].client_ref() is None: bucket.popleft() @@ -264,9 +311,12 @@ class EvictedClientCloser: bucket.append(pending) self._pending_count += 1 - def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]: - buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id) + def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]: + buckets = ( + (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token) + ) with self._queue_lock: + self._discard_dead_loops_locked() return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]: diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index a89e43b78b4..ccc4b314e44 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -2,10 +2,9 @@ Add the event loop to the cache key, to prevent event loop closed errors. """ -import asyncio from typing import Final -from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer +from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token from .in_memory_cache import InMemoryCache @@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache): self.evicted_client_closer.schedule(evicted) self.evicted_client_closer.reap() - def update_cache_key_with_event_loop(self, key): + def update_cache_key_with_event_loop(self, key: str) -> str: """ Add the event loop to the cache key, to prevent event loop closed errors. If none, use the key as is. """ - try: - event_loop: Final = asyncio.get_running_loop() - stringified_event_loop: Final = str(id(event_loop)) - return f"{key}-{stringified_event_loop}" - except RuntimeError: # handle no current running event loop - return key + token: Final = running_loop_token() + return f"{key}-{token}" if token is not None else key - def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): + def _scoped_key(self, key: str, loop_bound: bool) -> str: + return self.update_cache_key_with_event_loop(key) if loop_bound else key + + def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs): """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted.""" if litellm_owned_client: self.evicted_client_closer.mark_owned(value) - key = self.update_cache_key_with_event_loop(key) - return super().set_cache(key, value, **kwargs) + return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs) async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): if litellm_owned_client: @@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache): key = self.update_cache_key_with_event_loop(key) return await super().async_set_cache(key, value, **kwargs) - def get_cache(self, key, **kwargs): - key = self.update_cache_key_with_event_loop(key) + def get_cache(self, key: str, loop_bound: bool = True, **kwargs): + scoped_key: Final = self._scoped_key(key, loop_bound) self.evicted_client_closer.reap() - return super().get_cache(key, **kwargs) + return super().get_cache(scoped_key, **kwargs) - async def async_get_cache(self, key, **kwargs): + async def async_get_cache(self, key: str, **kwargs): key = self.update_cache_key_with_event_loop(key) return await super().async_get_cache(key, **kwargs) diff --git a/litellm/integrations/agentops/agentops.py b/litellm/integrations/agentops/agentops.py index 3364520a8ae..3666cd88a9d 100644 --- a/litellm/integrations/agentops/agentops.py +++ b/litellm/integrations/agentops/agentops.py @@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry): "Connection": "keep-alive", } - client: Final = get_httpx_client() - try: - response: Final = client.post( - url=auth_endpoint, - headers=headers, - json={"api_key": api_key}, - timeout=10, - ) + response: Final = get_httpx_client().post( + url=auth_endpoint, + headers=headers, + json={"api_key": api_key}, + timeout=10, + ) - if response.status_code != 200: - raise Exception(f"Failed to fetch auth token: {response.text}") + if response.status_code != 200: + raise Exception(f"Failed to fetch auth token: {response.text}") - return _DECODED_JSON.validate_python(response.json()) - finally: - client.close() + return _DECODED_JSON.validate_python(response.json()) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 3532cfbd963..05855c307e9 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -22,6 +22,7 @@ import httpx import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items @@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI params: Final = cast( # cast-ok: the base signature leaves the params dict untyped "ResponsesAPIOptionalRequestParams", response_api_optional_request_params ) - hoisted: Final = hoist_additional_tools(input, params.get("tools")) + hoisted: Final = hoist_additional_tools( + input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params) + ) normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input) if rewritten_types: verbose_logger.warning( diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index a8be217709f..5620971a8a2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1205,6 +1205,17 @@ class AsyncHTTPHandler: # Mirrors LiteLLMAiohttpTransport._background_close_tasks. _finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes + @classmethod + async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None: + try: + await client.aclose() + except (OSError, RuntimeError) as exc: + verbose_logger.debug("Error closing client at finalization: %s", exc) + finally: + task: Final = asyncio.current_task() + if task is not None: + cls._finalizer_close_tasks.discard(task) + @classmethod def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None: cls._finalizer_close_tasks.discard(task) @@ -1272,8 +1283,8 @@ class AsyncHTTPHandler: # here is the cross-loop path the transport refuses. self._dispose_wrapped_aiohttp_session() return - task: Final = loop.create_task(self._client.aclose()) cls: Final = type(self) + task: Final = loop.create_task(cls._close_client_at_finalization(self._client)) cls._finalizer_close_tasks.add(task) task.add_done_callback(cls._on_finalizer_close_done) except Exception: @@ -1973,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: cache = LLMClientCache() setattr(litellm, "in_memory_llm_clients_cache", cache) - _cached_client: Final = cache.get_cache(_cache_key_name) + _cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False) if _cached_client: return _cached_client @@ -1989,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: value=_new_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, litellm_owned_client=True, + loop_bound=False, ) return _new_client diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index a9d2dcdffb8..6c416124d1a 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -275,6 +275,10 @@ def build_output_token_limit_response( return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk)) +def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool: + return client_initialization_params.get("is_async") is not False + + class BaseOpenAILLM: """ Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings @@ -289,7 +293,9 @@ class BaseOpenAILLM: client_initialization_params=client_initialization_params, client_type=client_type, ) - _cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key) + _cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache( + _cache_key, loop_bound=_is_loop_bound(client_initialization_params) + ) return _cached_client @staticmethod @@ -330,6 +336,7 @@ class BaseOpenAILLM: value=openai_client, ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, litellm_owned_client=litellm_owned_client, + loop_bound=_is_loop_bound(client_initialization_params), ) @staticmethod diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index ce530447c0d..9c7a3a5f72e 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -89,6 +89,8 @@ def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, o def hoist_additional_tools( input: str | ResponseInputParam, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, + *, + drop_params: bool, ) -> HoistedAdditionalTools: existing: Final = tuple(tools or ()) if isinstance(input, str): @@ -103,6 +105,8 @@ def hoist_additional_tools( len(items), ) remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)] + if not drop_params: + return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted) unique: Final = tuple( cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts for tool in _unique_tools((*existing, *hoisted)) diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 8e8e4066bb8..a24a26aac42 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -114,7 +114,7 @@ def with_provider_file_ids( } -def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> Mapping[str, object]: +def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop_params: bool) -> Mapping[str, object]: responses_input: Final = responses_request.get("input") if not isinstance(responses_input, list): return responses_request @@ -122,7 +122,7 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) ) - hoisted: Final = hoist_additional_tools(input_items, top_level_tools) + hoisted: Final = hoist_additional_tools(input_items, top_level_tools, drop_params=drop_params) if not hoisted.hoisted: return responses_request return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} @@ -196,7 +196,7 @@ async def chat_count_request( chat_body: Final = await _with_previous_turns( responses_batch_body_to_chat_body( _with_compaction_as_summary_text( - _with_emulated_file_search(_with_hoisted_additional_tools(responses_request)) + _with_emulated_file_search(_with_hoisted_additional_tools(responses_request, policy.drop_params)) ), custom_llm_provider, policy.drop_params, diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 141e97d902d..1f56ddf863e 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -9,6 +9,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary @@ -73,7 +74,11 @@ class LiteLLMCompletionTransformationHandler: | BaseResponsesAPIStreamingIterator | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] ): - hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools")) + hoisted: Final = hoist_additional_tools( + input, + responses_api_request.get("tools"), + drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")), + ) previous_response_id: Final = responses_api_request.get("previous_response_id") if _is_async and previous_response_id and has_client_tool_search(hoisted.tools): return self._async_response_api_handler_after_earlier_searches( diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 5bdc538b2d9..e5068d24e88 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -27,8 +27,11 @@ tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and -the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here -first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new +the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same +`run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday, +and on a pull request when someone adds the `run-ci` label. Neither runs every pull request, so a new file here +first runs on main after its PR merges, or earlier if the PR gets `run-ci`. Wanting a file to run before merge +is no reason to change CI, and a new dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest diff --git a/tests/integration/README.md b/tests/integration/README.md index c32fda2f944..c5d52922de6 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,6 +1,6 @@ # Integration contracts -These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls +These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite, and `.github/workflows/test-integration.yml` runs the same groups through `run_integration.sh` on GitHub Actions for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport diff --git a/tests/integration/providers/test_responses_codex_repeated_additional_tools_wire.py b/tests/integration/providers/test_responses_codex_repeated_additional_tools_wire.py index e03b58188f8..7d930cbe44a 100644 --- a/tests/integration/providers/test_responses_codex_repeated_additional_tools_wire.py +++ b/tests/integration/providers/test_responses_codex_repeated_additional_tools_wire.py @@ -70,7 +70,9 @@ def bedrock_converse_peer(request: Request) -> Reply: ) -def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tuple[httpx.Response, tuple[Request, ...]]: +def send_to_bedrock_converse( + gateway: Gateway, history: list[JsonValue], drop_params: bool = True +) -> tuple[httpx.Response, tuple[Request, ...]]: with wire_server(bedrock_converse_peer) as wire, gateway.scenario() as scenario: model: Final = scenario.model( model=f"bedrock/converse/{BEDROCK_GPT}", @@ -79,7 +81,15 @@ def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tupl aws_bedrock_runtime_endpoint=wire.url, ) response: Final = gateway.request( - "POST", "/v1/responses", {"model": model, "input": history, "store": False, "cache": {"no-cache": True}} + "POST", + "/v1/responses", + { + "model": model, + "input": history, + "store": False, + "cache": {"no-cache": True}, + "drop_params": drop_params, + }, ) return response, wire.drain() @@ -113,6 +123,20 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_once(ga assert sorted(names) == ["collaboration__followup_task", "exec_command"], names +def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_as_sent_without_drop_params( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + tools: Final = tool_list_item(function_tool("exec_command")) + history: Final[list[JsonValue]] = [tools, user_turn(f"first {marker}"), assistant_turn("ok"), tools] + + response, forwarded = send_to_bedrock_converse(gateway, history, drop_params=False) + + assert response.status_code == 400, response.text + assert "already defined" in response.text, response.text + assert [str(spec["name"]) for spec in converse_tool_specs(forwarded[0])] == ["exec_command", "exec_command"] + + def test_newest_copy_of_a_repeated_codex_tool_is_what_bedrock_converse_receives(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex history: Final[list[JsonValue]] = [ @@ -195,6 +219,7 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_mantle_once(gate user_turn(f"second {marker}"), ], "store": False, + "drop_params": True, }, ) forwarded: Final = wire.drain() diff --git a/tests/unit/caching/_event_loop_reuse.py b/tests/unit/caching/_event_loop_reuse.py new file mode 100644 index 00000000000..97cf8fe31f0 --- /dev/null +++ b/tests/unit/caching/_event_loop_reuse.py @@ -0,0 +1,20 @@ +from collections.abc import Callable +from typing import Final, TypeVar + +import pytest + +LoopResult = TypeVar("LoopResult") + + +def runs_until_a_loop_address_repeats( + run_on_new_loop: Callable[[], tuple[int, LoopResult]], + earlier: tuple[tuple[int, LoopResult], ...] = (), + limit: int = 50, +) -> tuple[tuple[int, LoopResult], ...]: + if len(earlier) == limit: + pytest.skip(f"none of {limit} collected loops had its address handed to a later loop") + latest: Final = run_on_new_loop() + runs: Final = (*earlier, latest) + if latest[0] in {loop_id for loop_id, _ in earlier}: + return runs + return runs_until_a_loop_address_repeats(run_on_new_loop, runs, limit) diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index 7679e276621..936eb37a733 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle. import asyncio import gc import weakref +from typing import Final from unittest.mock import AsyncMock import httpx @@ -18,6 +19,7 @@ from redis.asyncio import ConnectionPool, Redis from litellm.caching.evicted_client_closer import EvictedClientCloser from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats class FakeClock: @@ -41,6 +43,14 @@ class AsyncClient: self.closed = True +class RecordingAsyncClient: + def __init__(self) -> None: + self.close_calls = 0 + + async def aclose(self) -> None: + self.close_calls += 1 + + class SyncClient: def __init__(self) -> None: self.closed = False @@ -266,6 +276,100 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): assert closer.pending_count == 0 +def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock) + + async def reap_then_schedule(client: RecordingAsyncClient) -> None: + closer.reap() + await asyncio.sleep(0) + closer.schedule(client) + + def run_on_new_loop() -> tuple[int, RecordingAsyncClient]: + client: Final = RecordingAsyncClient() + closer.mark_owned(client) + loop: Final = asyncio.new_event_loop() + loop_id: Final = id(loop) + try: + loop.run_until_complete(reap_then_schedule(client)) + finally: + loop.close() + del loop + gc.collect() + return loop_id, client + + runs: Final = runs_until_a_loop_address_repeats(run_on_new_loop) + + assert all(client.close_calls == 0 for _, client in runs) + + +def test_async_client_queued_on_a_live_loop_is_closed_only_by_that_loop() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock) + client: Final = RecordingAsyncClient() + closer.mark_owned(client) + owner_loop: Final = asyncio.new_event_loop() + other_loop: Final = asyncio.new_event_loop() + + async def schedule() -> None: + closer.schedule(client) + + async def reap() -> None: + closer.reap() + await asyncio.sleep(0) + + try: + owner_loop.run_until_complete(schedule()) + other_loop.run_until_complete(reap()) + closes_after_other_loop_reaped: Final = client.close_calls + pending_after_other_loop_reaped: Final = closer.pending_count + owner_loop.run_until_complete(reap()) + finally: + other_loop.close() + owner_loop.close() + + assert closes_after_other_loop_reaped == 0 + assert pending_after_other_loop_reaped == 1 + assert client.close_calls == 1 + assert closer.pending_count == 0 + + +def test_closes_queued_on_a_collected_loop_free_capacity() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock) + first: Final = RecordingAsyncClient() + closer.mark_owned(first) + loop_a: Final = asyncio.new_event_loop() + + async def schedule_first() -> None: + closer.schedule(first) + + try: + loop_a.run_until_complete(schedule_first()) + finally: + loop_a.close() + del loop_a + gc.collect() + + second: Final = RecordingAsyncClient() + closer.mark_owned(second) + loop_b: Final = asyncio.new_event_loop() + + async def schedule_and_reap_second() -> None: + closer.schedule(second) + closer.reap() + await asyncio.sleep(0) + + try: + loop_b.run_until_complete(schedule_and_reap_second()) + finally: + loop_b.close() + + assert first.close_calls == 0 + assert second.close_calls == 1 + assert closer.pending_count == 0 + + @pytest.mark.asyncio async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): clock = FakeClock() @@ -289,8 +393,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): @pytest.mark.asyncio -async def test_a_client_evicted_on_another_event_loop_is_left_alone(): - """Closing a client bound to a different loop would schedule work on that loop.""" +async def test_a_client_evicted_on_a_collected_loop_is_dropped(): clock = FakeClock() closer = make_closer(clock) client = AsyncClient() @@ -310,7 +413,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone(): await asyncio.sleep(0.05) assert client.closed is False - assert closer.pending_count == 1 + assert closer.pending_count == 0 @pytest.mark.asyncio diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index a3efd5d97aa..465ba2f9150 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247 import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref import warnings +from typing import Final import aiohttp import pytest @@ -29,6 +30,41 @@ from litellm.llms.custom_httpx.http_handler import( from litellm.types.utils import LlmProviders from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats + + +async def _get_and_set_cache(cache: LLMClientCache) -> object | None: + cached: Final = await cache.async_get_cache("client") + await cache.async_set_cache("client", object()) + return cached + + +def _cache_result_on_new_loop(cache: LLMClientCache) -> tuple[int, object | None]: + loop: Final = asyncio.new_event_loop() + loop_id: Final = id(loop) + try: + result: Final = loop.run_until_complete(_get_and_set_cache(cache)) + finally: + loop.close() + del loop + gc.collect() + return loop_id, result + + +async def _get_cache_key(cache: LLMClientCache) -> str: + return cache.update_cache_key_with_event_loop("client") + + +def _cache_key_on_new_loop(cache: LLMClientCache) -> tuple[int, str]: + loop: Final = asyncio.new_event_loop() + loop_id: Final = id(loop) + try: + key: Final = loop.run_until_complete(_get_cache_key(cache)) + finally: + loop.close() + del loop + gc.collect() + return loop_id, key class MockAsyncClient: @@ -51,6 +87,44 @@ class MockSyncClient: self.closed = True +def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None: + cache: Final = LLMClientCache() + runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_result_on_new_loop(cache)) + + assert all(result is None for _, result in runs) + + +def test_async_client_cache_keys_differ_for_every_collected_loop() -> None: + cache: Final = LLMClientCache() + runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_key_on_new_loop(cache)) + keys: Final = tuple(key for _, key in runs) + + assert len(set(keys)) == len(keys) + + +class _LoopWithoutWeakReferences: + __slots__ = () + + +def _cache_key_while_running(cache: LLMClientCache, loop: _LoopWithoutWeakReferences) -> str: + asyncio._set_running_loop(loop) + try: + return cache.update_cache_key_with_event_loop("client") + finally: + asyncio._set_running_loop(None) + + +def test_cache_keys_stay_per_loop_on_loops_without_weak_references() -> None: + cache: Final = LLMClientCache() + loop_a: Final = _LoopWithoutWeakReferences() + loop_b: Final = _LoopWithoutWeakReferences() + + first_key_on_a: Final = _cache_key_while_running(cache, loop_a) + + assert _cache_key_while_running(cache, loop_a) == first_key_on_a + assert _cache_key_while_running(cache, loop_b) != first_key_on_a + + @pytest.mark.asyncio async def test_remove_key_does_not_close_async_client(): """ diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index fe32831a459..6dd9ff35026 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -3281,7 +3281,7 @@ ARIZE_DEST = OtelDestination( class _ExporterCapture: """Stands an in-memory exporter in for every exporter the provider builds, keyed - by the headers it would have sent (``None`` for a headerless exporter), so a test + by the headers it would have sent (``None`` for the stdout placeholder), so a test reads what each account received rather than which processors were wired.""" def __init__(self) -> None: @@ -3445,13 +3445,13 @@ class TestLangfuseDestinationFields: ) -> None: """With ``langfuse_otel`` only in a key's or team's logging, the startup logger is the generic one; the key's project still gets the user, session and tags, and the - operator gets no unconfigured stdout copy.""" + operator's stdout placeholder still gets no Langfuse request metadata.""" credential_less_proxy(monkeypatch) capture: Final = _ExporterCapture() for kind in ("otlp_grpc", "otlp_http", "console"): monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter config: Final = OpenTelemetryV2Config() - provider: Final = build_tracer_provider(config, tenant_overrides=True) + provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True) attach_tenant_fan_out(provider, config) emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config) call: Final = replace( @@ -3477,7 +3477,7 @@ class TestLangfuseDestinationFields: "langfuse.trace.tags": ("agent:orders",), } assert "langfuse.observation.metadata" in to_langfuse - assert None not in received + assert "langfuse.observation.metadata" not in received[None] @staticmethod def _received_by_account( @@ -3491,7 +3491,7 @@ class TestLangfuseDestinationFields: capture: Final = _ExporterCapture() for kind in ("otlp_grpc", "otlp_http", "console"): monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter - provider: Final = build_tracer_provider(config, tenant_overrides=True) + provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True) attach_tenant_fan_out(provider, config) emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config) call: Final = replace( @@ -3515,11 +3515,7 @@ class TestLangfuseDestinationFields: ) -> None: """The operator's own exporter and the key's Arize destination get the model call as they would with no Langfuse destination on the request.""" - received: Final = self._received_by_account( - monkeypatch, - OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]), - (LANGFUSE_DEST, ARIZE_DEST), - ) + received: Final = self._received_by_account(monkeypatch, OpenTelemetryV2Config(), (LANGFUSE_DEST, ARIZE_DEST)) assert received[LANGFUSE_DEST.header_string()]["user.id"] == "user-42" assert { @@ -3532,9 +3528,7 @@ class TestLangfuseDestinationFields: self, monkeypatch: pytest.MonkeyPatch ) -> None: received: Final = self._received_by_account( - monkeypatch, - OpenTelemetryV2Config(mapper_names=["genai", "langfuse"], exporters=[ExporterSpec(kind="console")]), - (LANGFUSE_DEST,), + monkeypatch, OpenTelemetryV2Config(mapper_names=["genai", "langfuse"]), (LANGFUSE_DEST,) ) assert {key: received[None].get(key) for key in ("user.id", "session.id", "langfuse.trace.tags")} == { @@ -3632,10 +3626,7 @@ class TestLangfuseDestinationFields: assert arize is not None received: Final = self._received_by_account( - monkeypatch, - OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]), - (destination, arize), - with_content=True, + monkeypatch, OpenTelemetryV2Config(), (destination, arize), with_content=True ) assert {headers: "langfuse.observation.input" in attributes for headers, attributes in received.items()} == { @@ -3650,8 +3641,8 @@ class TestLangfuseDestinationFields: capture: Final = _ExporterCapture() for kind in ("otlp_grpc", "otlp_http", "console"): monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter - config: Final = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]) - provider: Final = build_tracer_provider(config, tenant_overrides=True) + config: Final = OpenTelemetryV2Config() + provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True) attach_tenant_fan_out(provider, config) emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config) destination: Final = destination_for( diff --git a/tests/unit/integrations/test_agentops.py b/tests/unit/integrations/test_agentops.py index 5d4055ac75f..629722f8602 100644 --- a/tests/unit/integrations/test_agentops.py +++ b/tests/unit/integrations/test_agentops.py @@ -1,10 +1,14 @@ import os +from typing import Final from unittest.mock import MagicMock, patch import pytest +import respx - +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.integrations.agentops.agentops import AgentOps, AgentOpsConfig +from litellm.llms.custom_httpx.http_handler import get_httpx_client @pytest.fixture @@ -111,3 +115,17 @@ def test_agentops_initialization_no_auth(): assert agentops.resource_attributes["deployment.environment"] == "test_env" assert agentops.resource_attributes["telemetry.sdk.name"] == "agentops" assert "project.id" not in agentops.resource_attributes + + +def test_auth_fetch_leaves_the_shared_sync_client_open_for_other_callers( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + auth_endpoint: Final = "https://agentops.example/v3/auth/token" + respx_mock.post(auth_endpoint).respond(json={"token": "jwt", "project_id": "project-1"}) + client_another_caller_is_using: Final = get_httpx_client().client + + agentops: Final = AgentOps(config=AgentOpsConfig(api_key="key", auth_endpoint=auth_endpoint)) + + assert agentops.resource_attributes["project.id"] == "project-1" + assert not client_another_caller_is_using.is_closed diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index cd3b377b943..0163cb9b26d 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -1388,6 +1388,55 @@ async def test_finalizer_with_running_loop_schedules_close_and_holds_task_ref(): assert not (AsyncHTTPHandler._finalizer_close_tasks & scheduled) +async def _drop_unused_handler() -> None: + AsyncHTTPHandler(timeout=61.0) + gc.collect() + + +def test_finalizer_close_does_not_keep_a_loop_closed_right_after_it_alive(): + loop: Final = asyncio.new_event_loop() + loop.run_until_complete(_drop_unused_handler()) + loop.close() + closed_loop: Final = weakref.ref(loop) + del loop + gc.collect() + + assert closed_loop() is None + + +class _TransportFailingToClose(httpx.AsyncBaseTransport): + async def aclose(self) -> None: + raise RuntimeError("close failed") + + +async def _drop_handler_whose_close_fails() -> None: + AsyncHTTPHandler(timeout=61.0, transport=_TransportFailingToClose()) + gc.collect() + + +def test_failed_finalizer_close_in_a_loops_last_step_reports_no_unretrieved_exception(): + loop: Final = asyncio.new_event_loop() + exception_handler: Final = MagicMock() + loop.set_exception_handler(exception_handler) + loop.run_until_complete(_drop_handler_whose_close_fails()) + loop.close() + del loop + gc.collect() + + exception_handler.assert_not_called() + + +async def _sync_client_with_timeout_62() -> HTTPHandler: + return get_httpx_client(params={"timeout": 62.0}) + + +def test_sync_client_is_shared_by_short_lived_loops_and_callers_outside_a_loop(): + built_in_first_loop: Final = asyncio.run(_sync_client_with_timeout_62()) + + assert asyncio.run(_sync_client_with_timeout_62()) is built_in_first_loop + assert get_httpx_client(params={"timeout": 62.0}) is built_in_first_loop + + @pytest.mark.asyncio async def test_sync_close_helper_respects_session_ownership(): """The loop-less fallback closes only sessions the transport owns; a diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 206fd13260c..46c14fcfd8f 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -1,3 +1,4 @@ +import asyncio from typing import Final from unittest.mock import MagicMock, patch @@ -154,6 +155,28 @@ def test_get_openai_client_initialization_param_fields(client_type): assert "self" not in result +def _loop_test_client_params(is_async: bool) -> dict[str, object]: + return {"api_key": "sk-loop-scope-test", "is_async": is_async} + + +async def _cache_openai_client(client: openai.OpenAI | openai.AsyncOpenAI, is_async: bool) -> None: + BaseOpenAILLM.set_cached_openai_client(client, "openai", _loop_test_client_params(is_async)) + + +async def _cached_openai_client(is_async: bool) -> object: + return BaseOpenAILLM.get_cached_openai_client(_loop_test_client_params(is_async), "openai") + + +def test_sync_openai_client_cached_in_one_loop_is_reused_by_the_next_while_async_is_not(): + sync_client: Final = openai.OpenAI(api_key="sk-loop-scope-test") + async_client: Final = openai.AsyncOpenAI(api_key="sk-loop-scope-test") + asyncio.run(_cache_openai_client(sync_client, is_async=False)) + asyncio.run(_cache_openai_client(async_client, is_async=True)) + + assert asyncio.run(_cached_openai_client(is_async=False)) is sync_client + assert asyncio.run(_cached_openai_client(is_async=True)) is None + + @pytest.mark.parametrize("client_type", ["openai", "azure"]) def test_get_openai_client_cache_key(client_type): """Verify get_openai_client_cache_key doesn't raise on tuple + tuple concatenation.""" diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py index a07076fb64f..2080a091160 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_count_request.py +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -95,8 +95,8 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools @pytest.mark.asyncio -async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once() -> None: - policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) +async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once_under_drop_params() -> None: + policy: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) tool_list: Final[dict[str, JsonValue]] = {"type": "additional_tools", "tools": [_WEATHER_TOOL]} repeated: Final = await _claude_count( diff --git a/tests/unit/responses/test_additional_tools.py b/tests/unit/responses/test_additional_tools.py index 8844d07b105..290670a7c0a 100644 --- a/tests/unit/responses/test_additional_tools.py +++ b/tests/unit/responses/test_additional_tools.py @@ -15,7 +15,7 @@ _USER_MESSAGE = {"type": "message", "role": "user", "content": "Run ls"} def test_string_input_passes_through_with_existing_tools(): - hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL]) + hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL], drop_params=False) assert hoisted.input == "hello" assert hoisted.tools == (_TOP_LEVEL_TOOL,) @@ -25,7 +25,7 @@ def test_string_input_passes_through_with_existing_tools(): def test_input_without_additional_tools_items_is_returned_untouched(): request_input = [_USER_MESSAGE] - hoisted = hoist_additional_tools(request_input, None) + hoisted = hoist_additional_tools(request_input, None, drop_params=False) assert hoisted.input is request_input assert hoisted.tools == () @@ -39,7 +39,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_ {"type": "additional_tools", "id": "at_2", "role": "developer", "tools": [_WAIT_TOOL]}, ] - hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL]) + hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False) assert hoisted.input == [_USER_MESSAGE] assert hoisted.tools == (_TOP_LEVEL_TOOL, _EXEC_TOOL, _WAIT_TOOL) @@ -49,7 +49,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_ def test_additional_tools_item_without_a_tools_list_is_stripped_and_contributes_nothing(): request_input = [{"type": "additional_tools", "id": "at_1", "role": "developer", "tools": "exec"}, _USER_MESSAGE] - hoisted = hoist_additional_tools(request_input, None) + hoisted = hoist_additional_tools(request_input, None, drop_params=False) assert hoisted.input == [_USER_MESSAGE] assert hoisted.tools == () @@ -77,7 +77,7 @@ def test_tool_list_repeated_on_every_turn_keeps_each_tool_once_with_the_newest_d _USER_MESSAGE, ] - hoisted: Final = hoist_additional_tools(request_input, None) + hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True) assert hoisted.tools == (collaboration, _function("exec", "new")) @@ -89,7 +89,7 @@ def test_namespace_sent_in_several_turns_keeps_every_distinct_member_once() -> N _tool_list(_namespace("mcp__docs__", _function("fetch", "new"), _function("list"))), ] - hoisted: Final = hoist_additional_tools(request_input, None) + hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True) assert hoisted.tools == ( _namespace("mcp__docs__", _function("search"), _function("fetch", "new"), _function("list")), @@ -109,7 +109,7 @@ def test_unnamed_tools_collapse_only_when_identical() -> None: } request_input: Final = [_tool_list(docs_server, wiki_server), _USER_MESSAGE, _tool_list(docs_server)] - hoisted: Final = hoist_additional_tools(request_input, None) + hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True) assert hoisted.tools == (docs_server, wiki_server) @@ -117,6 +117,20 @@ def test_unnamed_tools_collapse_only_when_identical() -> None: def test_top_level_tool_repeated_in_a_tool_list_item_is_kept_once() -> None: request_input: Final = [_tool_list(_TOP_LEVEL_TOOL, _WAIT_TOOL), _USER_MESSAGE] - hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL]) + hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=True) assert hoisted.tools == (_TOP_LEVEL_TOOL, _WAIT_TOOL) + + +def test_tool_list_repeated_on_every_turn_is_sent_as_is_without_drop_params() -> None: + request_input: Final = [ + _tool_list(_function("exec", "old")), + _USER_MESSAGE, + _tool_list(_function("exec", "new")), + _USER_MESSAGE, + ] + + hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False) + + assert hoisted.tools == (_TOP_LEVEL_TOOL, _function("exec", "old"), _function("exec", "new")) + assert hoisted.input == [_USER_MESSAGE, _USER_MESSAGE] diff --git a/tests/unit/test_assert_secret_absent.py b/tests/unit/test_assert_secret_absent.py new file mode 100644 index 00000000000..240229c01dc --- /dev/null +++ b/tests/unit/test_assert_secret_absent.py @@ -0,0 +1,124 @@ +import base64 +import importlib.util +import io +import sys +import urllib.parse +import zipfile +from collections.abc import Callable, Mapping +from itertools import chain, product +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "assert_secret_absent.py" +_spec: Final = importlib.util.spec_from_file_location("assert_secret_absent", _MODULE_PATH) +scanner: Final = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = scanner +_spec.loader.exec_module(scanner) + +SECRET: Final = "lic/ense+KEY:9f8e~~??>>~?7d6c5b4a3210-zz" +PREFIXES: Final = ("", "x", "xy") +ENCODERS: Final = (base64.b64encode, base64.urlsafe_b64encode) + + +def _zip(members: Mapping[str, bytes], compression: int = zipfile.ZIP_DEFLATED) -> bytes: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression) as archive: + for name, data in members.items(): + archive.writestr(name, data) + return buffer.getvalue() + + +def _blob(prefix: str) -> bytes: + return f"{prefix}{SECRET}tail".encode() + + +def _planted_forms() -> tuple[str, ...]: + encoded: Final = tuple(encode(_blob(prefix)).decode() for encode, prefix in product(ENCODERS, PREFIXES)) + return (SECRET, urllib.parse.quote(SECRET, safe=""), *encoded) + + +def _assert_nothing_leaked(output: str) -> None: + windows: Final = frozenset( + chain.from_iterable((form[i : i + 8] for i in range(len(form) - 7)) for form in _planted_forms()) + ) + assert not [window for window in windows if window in output] + + +def _run(root: Path, secret: str = SECRET) -> tuple[int, str]: + out: Final = io.StringIO() + code: Final = scanner.main((str(root),), {scanner.SECRET_VARIABLE: secret}, out) + return code, out.getvalue() + + +def _encoded_blob(prefix: str, encode: Callable[[bytes], bytes]) -> bytes: + return b"trace " + encode(_blob(prefix)) + b" end" + + +QUOTED: Final = urllib.parse.quote(SECRET, safe="") +PLACEMENTS: Final = MappingProxyType( + { + "raw in a log": {"proxy.log": f"boot with key {SECRET} ok".encode()}, + "url-encoded in a url": {"upstream.log": f"GET /verify_license/{QUOTED}".encode()}, + "base64 aligned": {"a.json": _encoded_blob("", base64.b64encode)}, + "base64 one byte in": {"a.json": _encoded_blob("x", base64.b64encode)}, + "base64 two bytes in": {"a.json": _encoded_blob("xy", base64.b64encode)}, + "base64url two bytes in": {"a.json": _encoded_blob("xy", base64.urlsafe_b64encode)}, + "inside a zip member": {"trace.zip": _zip({"network.json": f'{{"auth": "{SECRET}"}}'.encode()})}, + "inside a nested zip": {"outer.zip": _zip({"inner.zip": _zip({"env.txt": SECRET.encode()})})}, + "named after it in a nested zip": { + "outer.zip": _zip({"inner.zip": _zip({f"dump-{QUOTED}.txt": SECRET.encode()})}) + }, + "in a file name": {f"run-{QUOTED}.log": b"nothing inside"}, + } +) + + +@pytest.mark.parametrize("placement", PLACEMENTS) +def test_every_planted_copy_blocks_the_upload_without_printing_the_secret(tmp_path: Path, placement: str) -> None: + worker: Final = tmp_path / "gw0" + worker.mkdir() + for name, data in PLACEMENTS[placement].items(): + (worker / name).write_bytes(data) + code, output = _run(tmp_path) + assert code == 1 + assert "secret found:" in output + _assert_nothing_leaked(output) + + +def test_clean_results_pass_and_count_every_file(tmp_path: Path) -> None: + (tmp_path / "junit.xml").write_text("") + (tmp_path / "gw0").mkdir() + (tmp_path / "gw0" / "proxy.log").write_text("LITELLM_LICENSE is not set") + code, output = _run(tmp_path) + assert code == 0 + assert "no copy of the secret in 2 files" in output + + +def test_an_unset_secret_has_nothing_to_find(tmp_path: Path) -> None: + (tmp_path / "proxy.log").write_text(SECRET) + assert _run(tmp_path, secret="")[0] == 0 + + +def test_a_symlinked_directory_fails_closed(tmp_path: Path) -> None: + outside: Final = tmp_path / "outside" + outside.mkdir() + results: Final = tmp_path / "results" + results.mkdir() + (results / "linked").symlink_to(outside, target_is_directory=True) + code, output = _run(results) + assert code == 1 + assert "directory symlink" in output + + +def test_an_unreadable_archive_fails_without_echoing_its_contents(tmp_path: Path) -> None: + archive: Final = _zip({"env.txt": f"LITELLM_LICENSE={SECRET}".encode()}, zipfile.ZIP_STORED) + at: Final = archive.index(SECRET.encode()) + (tmp_path / "trace.zip").write_bytes(archive[:at] + bytes([archive[at] ^ 0xFF]) + archive[at + 1 :]) + code, output = _run(tmp_path) + assert code == 1 + assert "scan failed with BadZipFile" in output + _assert_nothing_leaked(output)