chore: merge main and retain upstream telemetry tests

This commit is contained in:
Tin 2026-10-29 14:55:28 -07:00
commit 73eae699ee
25 changed files with 935 additions and 96 deletions

View file

@ -1,14 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "${GITHUB_ACTIONS:-}" = true ]; then
echo "Integration contracts are owned by CircleCI" >&2
exit 1
fi
suite="${1:?integration suite required}"
mode="${2:-standard}"
side="${3:-}"
requested_workers="${INTEGRATION_WORKERS:-}"
license_env=()
if [ -n "${LITELLM_LICENSE:-}" ]; then
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
fi
if [ "$mode" = replica ]; then
results="test-results/integration-${suite}-replica"
elif [ "$mode" = parity ]; then
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
test -f "$LITELLM_UI_PATH/index.html"
fi
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@ -107,7 +107,7 @@ sudo iptables -N integration_only
guard_created=true
sudo iptables -A integration_only -o lo -j ACCEPT
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
for service in postgres-db redis-cache; do
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
sudo iptables -A integration_only -d "$address" -j ACCEPT
done
@ -145,6 +145,9 @@ fi
if [ "$suite" = mcp ]; then
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
fi
if [ -n "$requested_workers" ]; then
export INTEGRATION_WORKERS="$requested_workers"
fi
isolated_workers=false
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
isolated_workers=true
@ -185,7 +188,7 @@ start_proxy() {
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
LITELLM_MODE=PRODUCTION \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

119
.github/scripts/assert_secret_absent.py vendored Normal file
View file

@ -0,0 +1,119 @@
import base64
import io
import os
import sys
import urllib.parse
import zipfile
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TextIO
SECRET_VARIABLE: Final = "SCANNED_SECRET"
MIN_ENCODED_LENGTH: Final = 12
MAX_ARCHIVE_DEPTH: Final = 3
MASK: Final = "***"
Forms = tuple[tuple[str, str], ...]
@dataclass(frozen=True, slots=True)
class Hit:
location: str
form: str
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
for offset in range(3):
chunk: Final = raw[offset:]
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
def secret_forms(secret: str) -> Forms:
encoded: Final = (
("url-encoded", urllib.parse.quote(secret, safe="")),
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
)
by_value: Final = MappingProxyType(
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
)
return (("raw", secret), *((name, value) for value, name in by_value.items()))
def mask(text: str, forms: Forms) -> str:
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
for form, value in forms:
if value in name:
yield Hit(mask(location, forms), f"{form} in name")
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
for form, value in forms:
if value.encode() in data:
yield Hit(mask(location, forms), form)
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
return
with zipfile.ZipFile(io.BytesIO(data)) as archive:
for member in archive.infolist():
if member.is_dir():
continue
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
for directory, subdirectories, files in os.walk(root):
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
yield from ((Path(directory, name), False) for name in files)
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
location: Final = str(path.relative_to(root))
if linked_directory:
yield Hit(mask(location, forms), "directory symlink, not scanned")
return
yield from _name_hits(location, location, forms)
yield from _content_hits(location, path.read_bytes(), forms, 0)
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
forms: Final = secret_forms(secret)
entries: Final = tuple(_walk(root))
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
return len(entries), hits
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
if len(argv) != 1:
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
return 2
secret: Final = environ.get(SECRET_VARIABLE, "")
root: Final = Path(argv[0])
if not secret:
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
return 0
if not root.is_dir():
print(f"{root} is not a directory, nothing to scan", file=out)
return 0
try:
scanned, hits = scan(root, secret)
except Exception as error:
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
return 1
for hit in hits:
print(f"secret found: {hit.location} ({hit.form})", file=out)
if hits:
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
return 1
print(f"no copy of the secret in {scanned} files under {root}", file=out)
return 0
if __name__ == "__main__":
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

199
.github/workflows/test-integration.yml vendored Normal file
View file

@ -0,0 +1,199 @@
name: integration
on:
pull_request:
types: [labeled]
schedule:
- cron: "5 8-20/3 * * 1-6"
timezone: America/Los_Angeles
workflow_dispatch:
inputs:
suite:
description: Integration group to run
type: choice
options:
- all
- providers
- extensions
- accounting
- management
- management-replica
- database
- database-replica
- mcp
- security
- cost
- sdk
- browser
default: all
workers:
description: Workers per group, blank keeps each group's default
type: string
default: ""
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integration:
name: integration ${{ matrix.suite }}
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
environment: integration-license
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: circle_test
ports:
- 5432:5432
options: >-
--shm-size 1g
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 20
redis:
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
timeout-minutes: 5
uses: ./.github/actions/cache-prisma-binaries
- name: Install dependencies
timeout-minutes: 20
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Install redis-server 7.2.16
timeout-minutes: 3
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Install MCP SDK1 peer
if: matrix.suite == 'mcp'
timeout-minutes: 5
run: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
- name: Set up Node.js
if: matrix.suite == 'browser'
timeout-minutes: 5
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: |
ui/litellm-dashboard/package-lock.json
tests/e2e/ui/package-lock.json
- name: Install browser dependencies and build the dashboard
if: matrix.suite == 'browser'
timeout-minutes: 20
run: |
(cd ui/litellm-dashboard && npm ci)
(cd tests/e2e/ui && npm ci)
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
- name: Run integration contracts
timeout-minutes: 45
env:
SUITE: ${{ matrix.suite }}
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
run: |
case "$SUITE" in
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
esac
- name: Verify every case ran exactly once
if: ${{ !cancelled() && matrix.suite != 'browser' }}
timeout-minutes: 2
run: |
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
test -n "$record"
mkdir -p "$RUNNER_TEMP/shards/0"
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
- name: Report MCP coverage
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
timeout-minutes: 5
run: |
coverage_dir=test-results/integration-mcp/coverage
test -f "$coverage_dir/coverage.txt" && exit 0
export COVERAGE_FILE="$PWD/$coverage_dir/data"
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
tail -n 1 "$coverage_dir/coverage.txt"
- name: Check results for the license
id: secret-scan
if: ${{ !cancelled() }}
timeout-minutes: 5
env:
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
run: python .github/scripts/assert_secret_absent.py test-results
- name: Upload results
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
timeout-minutes: 10
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
path: test-results/
retention-days: 14

View file

@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
close them, and each bucket is ordered by deadline, so a reap walks the entries
that are due rather than the whole queue.
A bucket whose loop has been collected is dropped, and its clients are left to the
collector, as they are when the queue is full.
The queue holds its clients weakly, so waiting out a grace window never keeps
alive anything the collector would have reclaimed first.
"""
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
import asyncio
import contextlib
import inspect
import itertools
import threading
import time
import weakref
@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
_BucketKey = str | int
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
_LOOP_TOKEN_COUNTER: Final = itertools.count()
_LOOP_TOKEN_LOCK: Final = threading.Lock()
@dataclass(frozen=True, slots=True)
@ -65,12 +72,12 @@ class _PendingClose:
collector would otherwise have reclaimed first.
``needs_loop`` is set for a client whose close is a coroutine; those can only
be closed from the event loop they were evicted on, recorded in ``loop_id``.
be closed from the event loop they were evicted on, recorded in ``loop_token``.
A client that closes synchronously carries neither constraint.
"""
client_ref: "weakref.ref[object]"
loop_id: int | None
loop_token: int | None
needs_loop: bool
close_after: float
@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
if not pending.needs_loop:
return _CLOSABLE_ANYWHERE
if pending.loop_id is None:
if pending.loop_token is None:
return _CLOSABLE_ON_ANY_LOOP
return pending.loop_id
return pending.loop_token
def _running_loop_id() -> int | None:
def running_loop_token() -> int | None:
try:
return id(asyncio.get_running_loop())
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return None
try:
token: Final[int | None] = _LOOP_TOKENS.get(loop)
if token is not None:
return token
with _LOOP_TOKEN_LOCK:
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
except TypeError:
return id(loop)
def _close_function(client: object) -> Callable[[], object] | None:
@ -186,6 +201,8 @@ class EvictedClientCloser:
self._clock = clock
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
self._pending_count = 0
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
@ -216,13 +233,19 @@ class EvictedClientCloser:
close_fn: Final = _close_function(client)
if close_fn is None:
return
loop_token: Final = running_loop_token()
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
if self._pending_count >= self._max_pending:
return
self._discard_dead_loops()
if self._pending_count >= self._max_pending:
return
if needs_loop and loop_token is not None:
self._watch_loop(loop_token)
self._enqueue(
_PendingClose(
client_ref=weakref.ref(client),
loop_id=_running_loop_id(),
needs_loop=inspect.iscoroutinefunction(close_fn),
loop_token=loop_token,
needs_loop=needs_loop,
close_after=self._clock() + self._grace_seconds,
)
)
@ -236,7 +259,7 @@ class EvictedClientCloser:
if not self._pending_count:
return
now: Final = self._clock()
for pending in self._take_due(_running_loop_id(), now):
for pending in self._take_due(running_loop_token(), now):
client = pending.client_ref()
if client is None:
continue
@ -249,6 +272,29 @@ class EvictedClientCloser:
def pending_count(self) -> int:
return self._pending_count
def _watch_loop(self, loop_token: int) -> None:
if loop_token in self._watched_loop_tokens:
return
try:
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
except TypeError:
return
self._watched_loop_tokens.add(loop_token)
def _discard_dead_loops(self) -> None:
with self._queue_lock:
self._discard_dead_loops_locked()
def _discard_dead_loops_locked(self) -> None:
while self._dead_loop_tokens:
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
self._watched_loop_tokens.discard(loop_token)
bucket: Final = self._buckets.pop(loop_token, None)
if bucket is not None:
self._pending_count -= len(bucket)
def _enqueue(self, pending: _PendingClose) -> None:
"""Append to the entry's bucket, dropping any dead entries it queues behind.
@ -257,6 +303,7 @@ class EvictedClientCloser:
the front rather than having to be searched for.
"""
with self._queue_lock:
self._discard_dead_loops_locked()
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
while bucket and bucket[0].client_ref() is None:
bucket.popleft()
@ -264,9 +311,12 @@ class EvictedClientCloser:
bucket.append(pending)
self._pending_count += 1
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
)
with self._queue_lock:
self._discard_dead_loops_locked()
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

View file

@ -2,10 +2,9 @@
Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Final
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
from .in_memory_cache import InMemoryCache
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
self.evicted_client_closer.schedule(evicted)
self.evicted_client_closer.reap()
def update_cache_key_with_event_loop(self, key):
def update_cache_key_with_event_loop(self, key: str) -> str:
"""
Add the event loop to the cache key, to prevent event loop closed errors.
If none, use the key as is.
"""
try:
event_loop: Final = asyncio.get_running_loop()
stringified_event_loop: Final = str(id(event_loop))
return f"{key}-{stringified_event_loop}"
except RuntimeError: # handle no current running event loop
return key
token: Final = running_loop_token()
return f"{key}-{token}" if token is not None else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
def _scoped_key(self, key: str, loop_bound: bool) -> str:
return self.update_cache_key_with_event_loop(key) if loop_bound else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
if litellm_owned_client:
self.evicted_client_closer.mark_owned(value)
key = self.update_cache_key_with_event_loop(key)
return super().set_cache(key, value, **kwargs)
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
if litellm_owned_client:
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
key = self.update_cache_key_with_event_loop(key)
return await super().async_set_cache(key, value, **kwargs)
def get_cache(self, key, **kwargs):
key = self.update_cache_key_with_event_loop(key)
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
scoped_key: Final = self._scoped_key(key, loop_bound)
self.evicted_client_closer.reap()
return super().get_cache(key, **kwargs)
return super().get_cache(scoped_key, **kwargs)
async def async_get_cache(self, key, **kwargs):
async def async_get_cache(self, key: str, **kwargs):
key = self.update_cache_key_with_event_loop(key)
return await super().async_get_cache(key, **kwargs)

View file

@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
"Connection": "keep-alive",
}
client: Final = get_httpx_client()
try:
response: Final = client.post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
response: Final = get_httpx_client().post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()
return _DECODED_JSON.validate_python(response.json())

View file

@ -22,6 +22,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
hoisted: Final = hoist_additional_tools(
input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params)
)
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
if rewritten_types:
verbose_logger.warning(

View file

@ -1205,6 +1205,17 @@ class AsyncHTTPHandler:
# Mirrors LiteLLMAiohttpTransport._background_close_tasks.
_finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes
@classmethod
async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None:
try:
await client.aclose()
except (OSError, RuntimeError) as exc:
verbose_logger.debug("Error closing client at finalization: %s", exc)
finally:
task: Final = asyncio.current_task()
if task is not None:
cls._finalizer_close_tasks.discard(task)
@classmethod
def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
cls._finalizer_close_tasks.discard(task)
@ -1272,8 +1283,8 @@ class AsyncHTTPHandler:
# here is the cross-loop path the transport refuses.
self._dispose_wrapped_aiohttp_session()
return
task: Final = loop.create_task(self._client.aclose())
cls: Final = type(self)
task: Final = loop.create_task(cls._close_client_at_finalization(self._client))
cls._finalizer_close_tasks.add(task)
task.add_done_callback(cls._on_finalizer_close_done)
except Exception:
@ -1973,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
cache = LLMClientCache()
setattr(litellm, "in_memory_llm_clients_cache", cache)
_cached_client: Final = cache.get_cache(_cache_key_name)
_cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False)
if _cached_client:
return _cached_client
@ -1989,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
value=_new_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=True,
loop_bound=False,
)
return _new_client

View file

@ -275,6 +275,10 @@ def build_output_token_limit_response(
return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool:
return client_initialization_params.get("is_async") is not False
class BaseOpenAILLM:
"""
Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
@ -289,7 +293,9 @@ class BaseOpenAILLM:
client_initialization_params=client_initialization_params,
client_type=client_type,
)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(
_cache_key, loop_bound=_is_loop_bound(client_initialization_params)
)
return _cached_client
@staticmethod
@ -330,6 +336,7 @@ class BaseOpenAILLM:
value=openai_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=litellm_owned_client,
loop_bound=_is_loop_bound(client_initialization_params),
)
@staticmethod

View file

@ -89,6 +89,8 @@ def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, o
def hoist_additional_tools(
input: str | ResponseInputParam,
tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
*,
drop_params: bool,
) -> HoistedAdditionalTools:
existing: Final = tuple(tools or ())
if isinstance(input, str):
@ -103,6 +105,8 @@ def hoist_additional_tools(
len(items),
)
remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
if not drop_params:
return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)
unique: Final = tuple(
cast("ALL_RESPONSES_API_TOOL_PARAMS", tool) # cast-ok: a merged namespace is rebuilt from the same tool dicts
for tool in _unique_tools((*existing, *hoisted))

View file

@ -114,7 +114,7 @@ def with_provider_file_ids(
}
def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> Mapping[str, object]:
def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop_params: bool) -> Mapping[str, object]:
responses_input: Final = responses_request.get("input")
if not isinstance(responses_input, list):
return responses_request
@ -122,7 +122,7 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object]) -> M
top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does
"tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ())
)
hoisted: Final = hoist_additional_tools(input_items, top_level_tools)
hoisted: Final = hoist_additional_tools(input_items, top_level_tools, drop_params=drop_params)
if not hoisted.hoisted:
return responses_request
return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
@ -196,7 +196,7 @@ async def chat_count_request(
chat_body: Final = await _with_previous_turns(
responses_batch_body_to_chat_body(
_with_compaction_as_summary_text(
_with_emulated_file_search(_with_hoisted_additional_tools(responses_request))
_with_emulated_file_search(_with_hoisted_additional_tools(responses_request, policy.drop_params))
),
custom_llm_provider,
policy.drop_params,

View file

@ -9,6 +9,7 @@ from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
@ -73,7 +74,11 @@ class LiteLLMCompletionTransformationHandler:
| BaseResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
hoisted: Final = hoist_additional_tools(
input,
responses_api_request.get("tools"),
drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")),
)
previous_response_id: Final = responses_api_request.get("previous_response_id")
if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
return self._async_response_api_handler_after_earlier_searches(

View file

@ -27,8 +27,11 @@ tests. A product bug the test exposes is `pytest.skip("BUG: <symptom>")` at the
fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit`
CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and
the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here
first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new
the files in `GITHUB_FILES` in `run.py`. `test-integration.yml` runs the same groups through the same
`run_integration.sh` on GitHub Actions, on main every three hours from 8am to 8pm Pacific Monday to Saturday,
and on a pull request when someone adds the `run-ci` label. Neither runs every pull request, so a new file here
first runs on main after its PR merges, or earlier if the PR gets `run-ci`. Wanting a file to run before merge
is no reason to change CI, and a new
dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and
every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that
needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest

View file

@ -1,6 +1,6 @@
# Integration contracts
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite, and `.github/workflows/test-integration.yml` runs the same groups through `run_integration.sh` on GitHub Actions for `run-ci` pull requests and scheduled main runs. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport

View file

@ -70,7 +70,9 @@ def bedrock_converse_peer(request: Request) -> Reply:
)
def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tuple[httpx.Response, tuple[Request, ...]]:
def send_to_bedrock_converse(
gateway: Gateway, history: list[JsonValue], drop_params: bool = True
) -> tuple[httpx.Response, tuple[Request, ...]]:
with wire_server(bedrock_converse_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/converse/{BEDROCK_GPT}",
@ -79,7 +81,15 @@ def send_to_bedrock_converse(gateway: Gateway, history: list[JsonValue]) -> tupl
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": history, "store": False, "cache": {"no-cache": True}}
"POST",
"/v1/responses",
{
"model": model,
"input": history,
"store": False,
"cache": {"no-cache": True},
"drop_params": drop_params,
},
)
return response, wire.drain()
@ -113,6 +123,20 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_once(ga
assert sorted(names) == ["collaboration__followup_task", "exec_command"], names
def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_converse_as_sent_without_drop_params(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
tools: Final = tool_list_item(function_tool("exec_command"))
history: Final[list[JsonValue]] = [tools, user_turn(f"first {marker}"), assistant_turn("ok"), tools]
response, forwarded = send_to_bedrock_converse(gateway, history, drop_params=False)
assert response.status_code == 400, response.text
assert "already defined" in response.text, response.text
assert [str(spec["name"]) for spec in converse_tool_specs(forwarded[0])] == ["exec_command", "exec_command"]
def test_newest_copy_of_a_repeated_codex_tool_is_what_bedrock_converse_receives(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
history: Final[list[JsonValue]] = [
@ -195,6 +219,7 @@ def test_codex_tool_list_repeated_on_every_turn_reaches_bedrock_mantle_once(gate
user_turn(f"second {marker}"),
],
"store": False,
"drop_params": True,
},
)
forwarded: Final = wire.drain()

View file

@ -0,0 +1,20 @@
from collections.abc import Callable
from typing import Final, TypeVar
import pytest
LoopResult = TypeVar("LoopResult")
def runs_until_a_loop_address_repeats(
run_on_new_loop: Callable[[], tuple[int, LoopResult]],
earlier: tuple[tuple[int, LoopResult], ...] = (),
limit: int = 50,
) -> tuple[tuple[int, LoopResult], ...]:
if len(earlier) == limit:
pytest.skip(f"none of {limit} collected loops had its address handed to a later loop")
latest: Final = run_on_new_loop()
runs: Final = (*earlier, latest)
if latest[0] in {loop_id for loop_id, _ in earlier}:
return runs
return runs_until_a_loop_address_repeats(run_on_new_loop, runs, limit)

View file

@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle.
import asyncio
import gc
import weakref
from typing import Final
from unittest.mock import AsyncMock
import httpx
@ -18,6 +19,7 @@ from redis.asyncio import ConnectionPool, Redis
from litellm.caching.evicted_client_closer import EvictedClientCloser
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats
class FakeClock:
@ -41,6 +43,14 @@ class AsyncClient:
self.closed = True
class RecordingAsyncClient:
def __init__(self) -> None:
self.close_calls = 0
async def aclose(self) -> None:
self.close_calls += 1
class SyncClient:
def __init__(self) -> None:
self.closed = False
@ -266,6 +276,100 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed():
assert closer.pending_count == 0
def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None:
clock: Final = FakeClock()
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
async def reap_then_schedule(client: RecordingAsyncClient) -> None:
closer.reap()
await asyncio.sleep(0)
closer.schedule(client)
def run_on_new_loop() -> tuple[int, RecordingAsyncClient]:
client: Final = RecordingAsyncClient()
closer.mark_owned(client)
loop: Final = asyncio.new_event_loop()
loop_id: Final = id(loop)
try:
loop.run_until_complete(reap_then_schedule(client))
finally:
loop.close()
del loop
gc.collect()
return loop_id, client
runs: Final = runs_until_a_loop_address_repeats(run_on_new_loop)
assert all(client.close_calls == 0 for _, client in runs)
def test_async_client_queued_on_a_live_loop_is_closed_only_by_that_loop() -> None:
clock: Final = FakeClock()
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
client: Final = RecordingAsyncClient()
closer.mark_owned(client)
owner_loop: Final = asyncio.new_event_loop()
other_loop: Final = asyncio.new_event_loop()
async def schedule() -> None:
closer.schedule(client)
async def reap() -> None:
closer.reap()
await asyncio.sleep(0)
try:
owner_loop.run_until_complete(schedule())
other_loop.run_until_complete(reap())
closes_after_other_loop_reaped: Final = client.close_calls
pending_after_other_loop_reaped: Final = closer.pending_count
owner_loop.run_until_complete(reap())
finally:
other_loop.close()
owner_loop.close()
assert closes_after_other_loop_reaped == 0
assert pending_after_other_loop_reaped == 1
assert client.close_calls == 1
assert closer.pending_count == 0
def test_closes_queued_on_a_collected_loop_free_capacity() -> None:
clock: Final = FakeClock()
closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock)
first: Final = RecordingAsyncClient()
closer.mark_owned(first)
loop_a: Final = asyncio.new_event_loop()
async def schedule_first() -> None:
closer.schedule(first)
try:
loop_a.run_until_complete(schedule_first())
finally:
loop_a.close()
del loop_a
gc.collect()
second: Final = RecordingAsyncClient()
closer.mark_owned(second)
loop_b: Final = asyncio.new_event_loop()
async def schedule_and_reap_second() -> None:
closer.schedule(second)
closer.reap()
await asyncio.sleep(0)
try:
loop_b.run_until_complete(schedule_and_reap_second())
finally:
loop_b.close()
assert first.close_calls == 0
assert second.close_calls == 1
assert closer.pending_count == 0
@pytest.mark.asyncio
async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
clock = FakeClock()
@ -289,8 +393,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
@pytest.mark.asyncio
async def test_a_client_evicted_on_another_event_loop_is_left_alone():
"""Closing a client bound to a different loop would schedule work on that loop."""
async def test_a_client_evicted_on_a_collected_loop_is_dropped():
clock = FakeClock()
closer = make_closer(clock)
client = AsyncClient()
@ -310,7 +413,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone():
await asyncio.sleep(0.05)
assert client.closed is False
assert closer.pending_count == 1
assert closer.pending_count == 0
@pytest.mark.asyncio

View file

@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247
import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref
import warnings
from typing import Final
import aiohttp
import pytest
@ -29,6 +30,41 @@ from litellm.llms.custom_httpx.http_handler import(
from litellm.types.utils import LlmProviders
from litellm.utils import _invalidate_model_cost_lowercase_map
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from tests.unit.caching._event_loop_reuse import runs_until_a_loop_address_repeats
async def _get_and_set_cache(cache: LLMClientCache) -> object | None:
cached: Final = await cache.async_get_cache("client")
await cache.async_set_cache("client", object())
return cached
def _cache_result_on_new_loop(cache: LLMClientCache) -> tuple[int, object | None]:
loop: Final = asyncio.new_event_loop()
loop_id: Final = id(loop)
try:
result: Final = loop.run_until_complete(_get_and_set_cache(cache))
finally:
loop.close()
del loop
gc.collect()
return loop_id, result
async def _get_cache_key(cache: LLMClientCache) -> str:
return cache.update_cache_key_with_event_loop("client")
def _cache_key_on_new_loop(cache: LLMClientCache) -> tuple[int, str]:
loop: Final = asyncio.new_event_loop()
loop_id: Final = id(loop)
try:
key: Final = loop.run_until_complete(_get_cache_key(cache))
finally:
loop.close()
del loop
gc.collect()
return loop_id, key
class MockAsyncClient:
@ -51,6 +87,44 @@ class MockSyncClient:
self.closed = True
def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None:
cache: Final = LLMClientCache()
runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_result_on_new_loop(cache))
assert all(result is None for _, result in runs)
def test_async_client_cache_keys_differ_for_every_collected_loop() -> None:
cache: Final = LLMClientCache()
runs: Final = runs_until_a_loop_address_repeats(lambda: _cache_key_on_new_loop(cache))
keys: Final = tuple(key for _, key in runs)
assert len(set(keys)) == len(keys)
class _LoopWithoutWeakReferences:
__slots__ = ()
def _cache_key_while_running(cache: LLMClientCache, loop: _LoopWithoutWeakReferences) -> str:
asyncio._set_running_loop(loop)
try:
return cache.update_cache_key_with_event_loop("client")
finally:
asyncio._set_running_loop(None)
def test_cache_keys_stay_per_loop_on_loops_without_weak_references() -> None:
cache: Final = LLMClientCache()
loop_a: Final = _LoopWithoutWeakReferences()
loop_b: Final = _LoopWithoutWeakReferences()
first_key_on_a: Final = _cache_key_while_running(cache, loop_a)
assert _cache_key_while_running(cache, loop_a) == first_key_on_a
assert _cache_key_while_running(cache, loop_b) != first_key_on_a
@pytest.mark.asyncio
async def test_remove_key_does_not_close_async_client():
"""

View file

@ -3281,7 +3281,7 @@ ARIZE_DEST = OtelDestination(
class _ExporterCapture:
"""Stands an in-memory exporter in for every exporter the provider builds, keyed
by the headers it would have sent (``None`` for a headerless exporter), so a test
by the headers it would have sent (``None`` for the stdout placeholder), so a test
reads what each account received rather than which processors were wired."""
def __init__(self) -> None:
@ -3445,13 +3445,13 @@ class TestLangfuseDestinationFields:
) -> None:
"""With ``langfuse_otel`` only in a key's or team's logging, the startup logger is the
generic one; the key's project still gets the user, session and tags, and the
operator gets no unconfigured stdout copy."""
operator's stdout placeholder still gets no Langfuse request metadata."""
credential_less_proxy(monkeypatch)
capture: Final = _ExporterCapture()
for kind in ("otlp_grpc", "otlp_http", "console"):
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
config: Final = OpenTelemetryV2Config()
provider: Final = build_tracer_provider(config, tenant_overrides=True)
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
attach_tenant_fan_out(provider, config)
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
call: Final = replace(
@ -3477,7 +3477,7 @@ class TestLangfuseDestinationFields:
"langfuse.trace.tags": ("agent:orders",),
}
assert "langfuse.observation.metadata" in to_langfuse
assert None not in received
assert "langfuse.observation.metadata" not in received[None]
@staticmethod
def _received_by_account(
@ -3491,7 +3491,7 @@ class TestLangfuseDestinationFields:
capture: Final = _ExporterCapture()
for kind in ("otlp_grpc", "otlp_http", "console"):
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
provider: Final = build_tracer_provider(config, tenant_overrides=True)
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
attach_tenant_fan_out(provider, config)
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
call: Final = replace(
@ -3515,11 +3515,7 @@ class TestLangfuseDestinationFields:
) -> None:
"""The operator's own exporter and the key's Arize destination get the model call as they would with no
Langfuse destination on the request."""
received: Final = self._received_by_account(
monkeypatch,
OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]),
(LANGFUSE_DEST, ARIZE_DEST),
)
received: Final = self._received_by_account(monkeypatch, OpenTelemetryV2Config(), (LANGFUSE_DEST, ARIZE_DEST))
assert received[LANGFUSE_DEST.header_string()]["user.id"] == "user-42"
assert {
@ -3532,9 +3528,7 @@ class TestLangfuseDestinationFields:
self, monkeypatch: pytest.MonkeyPatch
) -> None:
received: Final = self._received_by_account(
monkeypatch,
OpenTelemetryV2Config(mapper_names=["genai", "langfuse"], exporters=[ExporterSpec(kind="console")]),
(LANGFUSE_DEST,),
monkeypatch, OpenTelemetryV2Config(mapper_names=["genai", "langfuse"]), (LANGFUSE_DEST,)
)
assert {key: received[None].get(key) for key in ("user.id", "session.id", "langfuse.trace.tags")} == {
@ -3632,10 +3626,7 @@ class TestLangfuseDestinationFields:
assert arize is not None
received: Final = self._received_by_account(
monkeypatch,
OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")]),
(destination, arize),
with_content=True,
monkeypatch, OpenTelemetryV2Config(), (destination, arize), with_content=True
)
assert {headers: "langfuse.observation.input" in attributes for headers, attributes in received.items()} == {
@ -3650,8 +3641,8 @@ class TestLangfuseDestinationFields:
capture: Final = _ExporterCapture()
for kind in ("otlp_grpc", "otlp_http", "console"):
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build) # pyright: ignore[reportPrivateUsage] # the factory table is the seam for capturing each exporter
config: Final = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="console")])
provider: Final = build_tracer_provider(config, tenant_overrides=True)
config: Final = OpenTelemetryV2Config()
provider: Final = build_tracer_provider(config, tenant_overrides=True, console_default=True)
attach_tenant_fan_out(provider, config)
emitter: Final = SpanEmitter(get_tracer(provider, "litellm"), config)
destination: Final = destination_for(

View file

@ -1,10 +1,14 @@
import os
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.integrations.agentops.agentops import AgentOps, AgentOpsConfig
from litellm.llms.custom_httpx.http_handler import get_httpx_client
@pytest.fixture
@ -111,3 +115,17 @@ def test_agentops_initialization_no_auth():
assert agentops.resource_attributes["deployment.environment"] == "test_env"
assert agentops.resource_attributes["telemetry.sdk.name"] == "agentops"
assert "project.id" not in agentops.resource_attributes
def test_auth_fetch_leaves_the_shared_sync_client_open_for_other_callers(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
auth_endpoint: Final = "https://agentops.example/v3/auth/token"
respx_mock.post(auth_endpoint).respond(json={"token": "jwt", "project_id": "project-1"})
client_another_caller_is_using: Final = get_httpx_client().client
agentops: Final = AgentOps(config=AgentOpsConfig(api_key="key", auth_endpoint=auth_endpoint))
assert agentops.resource_attributes["project.id"] == "project-1"
assert not client_another_caller_is_using.is_closed

View file

@ -1388,6 +1388,55 @@ async def test_finalizer_with_running_loop_schedules_close_and_holds_task_ref():
assert not (AsyncHTTPHandler._finalizer_close_tasks & scheduled)
async def _drop_unused_handler() -> None:
AsyncHTTPHandler(timeout=61.0)
gc.collect()
def test_finalizer_close_does_not_keep_a_loop_closed_right_after_it_alive():
loop: Final = asyncio.new_event_loop()
loop.run_until_complete(_drop_unused_handler())
loop.close()
closed_loop: Final = weakref.ref(loop)
del loop
gc.collect()
assert closed_loop() is None
class _TransportFailingToClose(httpx.AsyncBaseTransport):
async def aclose(self) -> None:
raise RuntimeError("close failed")
async def _drop_handler_whose_close_fails() -> None:
AsyncHTTPHandler(timeout=61.0, transport=_TransportFailingToClose())
gc.collect()
def test_failed_finalizer_close_in_a_loops_last_step_reports_no_unretrieved_exception():
loop: Final = asyncio.new_event_loop()
exception_handler: Final = MagicMock()
loop.set_exception_handler(exception_handler)
loop.run_until_complete(_drop_handler_whose_close_fails())
loop.close()
del loop
gc.collect()
exception_handler.assert_not_called()
async def _sync_client_with_timeout_62() -> HTTPHandler:
return get_httpx_client(params={"timeout": 62.0})
def test_sync_client_is_shared_by_short_lived_loops_and_callers_outside_a_loop():
built_in_first_loop: Final = asyncio.run(_sync_client_with_timeout_62())
assert asyncio.run(_sync_client_with_timeout_62()) is built_in_first_loop
assert get_httpx_client(params={"timeout": 62.0}) is built_in_first_loop
@pytest.mark.asyncio
async def test_sync_close_helper_respects_session_ownership():
"""The loop-less fallback closes only sessions the transport owns; a

View file

@ -1,3 +1,4 @@
import asyncio
from typing import Final
from unittest.mock import MagicMock, patch
@ -154,6 +155,28 @@ def test_get_openai_client_initialization_param_fields(client_type):
assert "self" not in result
def _loop_test_client_params(is_async: bool) -> dict[str, object]:
return {"api_key": "sk-loop-scope-test", "is_async": is_async}
async def _cache_openai_client(client: openai.OpenAI | openai.AsyncOpenAI, is_async: bool) -> None:
BaseOpenAILLM.set_cached_openai_client(client, "openai", _loop_test_client_params(is_async))
async def _cached_openai_client(is_async: bool) -> object:
return BaseOpenAILLM.get_cached_openai_client(_loop_test_client_params(is_async), "openai")
def test_sync_openai_client_cached_in_one_loop_is_reused_by_the_next_while_async_is_not():
sync_client: Final = openai.OpenAI(api_key="sk-loop-scope-test")
async_client: Final = openai.AsyncOpenAI(api_key="sk-loop-scope-test")
asyncio.run(_cache_openai_client(sync_client, is_async=False))
asyncio.run(_cache_openai_client(async_client, is_async=True))
assert asyncio.run(_cached_openai_client(is_async=False)) is sync_client
assert asyncio.run(_cached_openai_client(is_async=True)) is None
@pytest.mark.parametrize("client_type", ["openai", "azure"])
def test_get_openai_client_cache_key(client_type):
"""Verify get_openai_client_cache_key doesn't raise on tuple + tuple concatenation."""

View file

@ -95,8 +95,8 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools
@pytest.mark.asyncio
async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once() -> None:
policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset())
async def test_claude_count_request_counts_a_tool_list_repeated_on_every_turn_once_under_drop_params() -> None:
policy: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset())
tool_list: Final[dict[str, JsonValue]] = {"type": "additional_tools", "tools": [_WEATHER_TOOL]}
repeated: Final = await _claude_count(

View file

@ -15,7 +15,7 @@ _USER_MESSAGE = {"type": "message", "role": "user", "content": "Run ls"}
def test_string_input_passes_through_with_existing_tools():
hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL])
hoisted = hoist_additional_tools("hello", [_TOP_LEVEL_TOOL], drop_params=False)
assert hoisted.input == "hello"
assert hoisted.tools == (_TOP_LEVEL_TOOL,)
@ -25,7 +25,7 @@ def test_string_input_passes_through_with_existing_tools():
def test_input_without_additional_tools_items_is_returned_untouched():
request_input = [_USER_MESSAGE]
hoisted = hoist_additional_tools(request_input, None)
hoisted = hoist_additional_tools(request_input, None, drop_params=False)
assert hoisted.input is request_input
assert hoisted.tools == ()
@ -39,7 +39,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_
{"type": "additional_tools", "id": "at_2", "role": "developer", "tools": [_WAIT_TOOL]},
]
hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL])
hoisted = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False)
assert hoisted.input == [_USER_MESSAGE]
assert hoisted.tools == (_TOP_LEVEL_TOOL, _EXEC_TOOL, _WAIT_TOOL)
@ -49,7 +49,7 @@ def test_additional_tools_items_are_stripped_and_appended_after_top_level_tools_
def test_additional_tools_item_without_a_tools_list_is_stripped_and_contributes_nothing():
request_input = [{"type": "additional_tools", "id": "at_1", "role": "developer", "tools": "exec"}, _USER_MESSAGE]
hoisted = hoist_additional_tools(request_input, None)
hoisted = hoist_additional_tools(request_input, None, drop_params=False)
assert hoisted.input == [_USER_MESSAGE]
assert hoisted.tools == ()
@ -77,7 +77,7 @@ def test_tool_list_repeated_on_every_turn_keeps_each_tool_once_with_the_newest_d
_USER_MESSAGE,
]
hoisted: Final = hoist_additional_tools(request_input, None)
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
assert hoisted.tools == (collaboration, _function("exec", "new"))
@ -89,7 +89,7 @@ def test_namespace_sent_in_several_turns_keeps_every_distinct_member_once() -> N
_tool_list(_namespace("mcp__docs__", _function("fetch", "new"), _function("list"))),
]
hoisted: Final = hoist_additional_tools(request_input, None)
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
assert hoisted.tools == (
_namespace("mcp__docs__", _function("search"), _function("fetch", "new"), _function("list")),
@ -109,7 +109,7 @@ def test_unnamed_tools_collapse_only_when_identical() -> None:
}
request_input: Final = [_tool_list(docs_server, wiki_server), _USER_MESSAGE, _tool_list(docs_server)]
hoisted: Final = hoist_additional_tools(request_input, None)
hoisted: Final = hoist_additional_tools(request_input, None, drop_params=True)
assert hoisted.tools == (docs_server, wiki_server)
@ -117,6 +117,20 @@ def test_unnamed_tools_collapse_only_when_identical() -> None:
def test_top_level_tool_repeated_in_a_tool_list_item_is_kept_once() -> None:
request_input: Final = [_tool_list(_TOP_LEVEL_TOOL, _WAIT_TOOL), _USER_MESSAGE]
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL])
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=True)
assert hoisted.tools == (_TOP_LEVEL_TOOL, _WAIT_TOOL)
def test_tool_list_repeated_on_every_turn_is_sent_as_is_without_drop_params() -> None:
request_input: Final = [
_tool_list(_function("exec", "old")),
_USER_MESSAGE,
_tool_list(_function("exec", "new")),
_USER_MESSAGE,
]
hoisted: Final = hoist_additional_tools(request_input, [_TOP_LEVEL_TOOL], drop_params=False)
assert hoisted.tools == (_TOP_LEVEL_TOOL, _function("exec", "old"), _function("exec", "new"))
assert hoisted.input == [_USER_MESSAGE, _USER_MESSAGE]

View file

@ -0,0 +1,124 @@
import base64
import importlib.util
import io
import sys
import urllib.parse
import zipfile
from collections.abc import Callable, Mapping
from itertools import chain, product
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
_REPO_ROOT: Final = Path(__file__).resolve().parents[2]
_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "assert_secret_absent.py"
_spec: Final = importlib.util.spec_from_file_location("assert_secret_absent", _MODULE_PATH)
scanner: Final = importlib.util.module_from_spec(_spec)
sys.modules[_spec.name] = scanner
_spec.loader.exec_module(scanner)
SECRET: Final = "lic/ense+KEY:9f8e~~??>>~?7d6c5b4a3210-zz"
PREFIXES: Final = ("", "x", "xy")
ENCODERS: Final = (base64.b64encode, base64.urlsafe_b64encode)
def _zip(members: Mapping[str, bytes], compression: int = zipfile.ZIP_DEFLATED) -> bytes:
buffer: Final = io.BytesIO()
with zipfile.ZipFile(buffer, "w", compression) as archive:
for name, data in members.items():
archive.writestr(name, data)
return buffer.getvalue()
def _blob(prefix: str) -> bytes:
return f"{prefix}{SECRET}tail".encode()
def _planted_forms() -> tuple[str, ...]:
encoded: Final = tuple(encode(_blob(prefix)).decode() for encode, prefix in product(ENCODERS, PREFIXES))
return (SECRET, urllib.parse.quote(SECRET, safe=""), *encoded)
def _assert_nothing_leaked(output: str) -> None:
windows: Final = frozenset(
chain.from_iterable((form[i : i + 8] for i in range(len(form) - 7)) for form in _planted_forms())
)
assert not [window for window in windows if window in output]
def _run(root: Path, secret: str = SECRET) -> tuple[int, str]:
out: Final = io.StringIO()
code: Final = scanner.main((str(root),), {scanner.SECRET_VARIABLE: secret}, out)
return code, out.getvalue()
def _encoded_blob(prefix: str, encode: Callable[[bytes], bytes]) -> bytes:
return b"trace " + encode(_blob(prefix)) + b" end"
QUOTED: Final = urllib.parse.quote(SECRET, safe="")
PLACEMENTS: Final = MappingProxyType(
{
"raw in a log": {"proxy.log": f"boot with key {SECRET} ok".encode()},
"url-encoded in a url": {"upstream.log": f"GET /verify_license/{QUOTED}".encode()},
"base64 aligned": {"a.json": _encoded_blob("", base64.b64encode)},
"base64 one byte in": {"a.json": _encoded_blob("x", base64.b64encode)},
"base64 two bytes in": {"a.json": _encoded_blob("xy", base64.b64encode)},
"base64url two bytes in": {"a.json": _encoded_blob("xy", base64.urlsafe_b64encode)},
"inside a zip member": {"trace.zip": _zip({"network.json": f'{{"auth": "{SECRET}"}}'.encode()})},
"inside a nested zip": {"outer.zip": _zip({"inner.zip": _zip({"env.txt": SECRET.encode()})})},
"named after it in a nested zip": {
"outer.zip": _zip({"inner.zip": _zip({f"dump-{QUOTED}.txt": SECRET.encode()})})
},
"in a file name": {f"run-{QUOTED}.log": b"nothing inside"},
}
)
@pytest.mark.parametrize("placement", PLACEMENTS)
def test_every_planted_copy_blocks_the_upload_without_printing_the_secret(tmp_path: Path, placement: str) -> None:
worker: Final = tmp_path / "gw0"
worker.mkdir()
for name, data in PLACEMENTS[placement].items():
(worker / name).write_bytes(data)
code, output = _run(tmp_path)
assert code == 1
assert "secret found:" in output
_assert_nothing_leaked(output)
def test_clean_results_pass_and_count_every_file(tmp_path: Path) -> None:
(tmp_path / "junit.xml").write_text("<testsuite/>")
(tmp_path / "gw0").mkdir()
(tmp_path / "gw0" / "proxy.log").write_text("LITELLM_LICENSE is not set")
code, output = _run(tmp_path)
assert code == 0
assert "no copy of the secret in 2 files" in output
def test_an_unset_secret_has_nothing_to_find(tmp_path: Path) -> None:
(tmp_path / "proxy.log").write_text(SECRET)
assert _run(tmp_path, secret="")[0] == 0
def test_a_symlinked_directory_fails_closed(tmp_path: Path) -> None:
outside: Final = tmp_path / "outside"
outside.mkdir()
results: Final = tmp_path / "results"
results.mkdir()
(results / "linked").symlink_to(outside, target_is_directory=True)
code, output = _run(results)
assert code == 1
assert "directory symlink" in output
def test_an_unreadable_archive_fails_without_echoing_its_contents(tmp_path: Path) -> None:
archive: Final = _zip({"env.txt": f"LITELLM_LICENSE={SECRET}".encode()}, zipfile.ZIP_STORED)
at: Final = archive.index(SECRET.encode())
(tmp_path / "trace.zip").write_bytes(archive[:at] + bytes([archive[at] ^ 0xFF]) + archive[at + 1 :])
code, output = _run(tmp_path)
assert code == 1
assert "scan failed with BadZipFile" in output
_assert_nothing_leaked(output)