diff --git a/.github/workflows/python-benchmark.yml b/.github/workflows/python-benchmark.yml new file mode 100644 index 00000000000..a04d9a29177 --- /dev/null +++ b/.github/workflows/python-benchmark.yml @@ -0,0 +1,137 @@ +name: Python Benchmarks + +on: + push: + branches-ignore: [gh-pages] + workflow_dispatch: + inputs: + target_ref: + description: "SHA, branch, or refs/pull/123/head; blank uses the workflow commit" + type: string + required: false + +permissions: {} + +concurrency: + group: python-benchmark-pages + queue: max + +jobs: + benchmark: + name: Measure target commit + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + outputs: + target_sha: ${{ steps.target.outputs.commit }} + script_sha: ${{ steps.harness.outputs.commit }} + steps: + - name: Fetch benchmark harness from default branch + id: harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.event.repository.default_branch }} + path: harness + persist-credentials: false + + - name: Fetch target code + id: target + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.target_ref || github.sha }} + path: target + persist-credentials: false + + - name: Run benchmark + working-directory: harness + shell: bash + run: | + bash scripts/benchmark.sh \ + "$GITHUB_WORKSPACE/target" \ + "$RUNNER_TEMP/results.json" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: benchmark-results + path: ${{ runner.temp }}/results.json + if-no-files-found: error + retention-days: 7 + + publish: + name: Store benchmark history + needs: benchmark + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write # Commit benchmark results to gh-pages + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: benchmark-results + path: ${{ runner.temp }}/incoming + + - name: Validate metrics and attach provenance + shell: bash + env: + SCRIPT_SHA: ${{ needs.benchmark.outputs.script_sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: | + test -f "$RUNNER_TEMP/incoming/results.json" + test ! -L "$RUNNER_TEMP/incoming/results.json" + test "$(wc -c < "$RUNNER_TEMP/incoming/results.json")" -le 1048576 + jq -e ' + type == "array" and length > 0 and length <= 1000 and + all(.[]; type == "object" and + (.name | type == "string" and length > 0 and length <= 200 and + test("^[a-zA-Z0-9_. /():%-]+$")) and + (.unit | type == "string" and length > 0 and length <= 32 and + test("^[a-zA-Z0-9_./% -]+$")) and + (.value | type == "number" and isfinite)) + ' "$RUNNER_TEMP/incoming/results.json" > /dev/null + jq --arg extra "harness=$SCRIPT_SHA; run=$RUN_URL" \ + 'map({name, unit, value, extra: $extra})' \ + "$RUNNER_TEMP/incoming/results.json" > "$RUNNER_TEMP/results.json" + + - name: Store results and update dashboard + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba + with: + name: Benchmarks + tool: customSmallerIsBetter + output-file-path: ${{ runner.temp }}/results.json + ref: ${{ needs.benchmark.outputs.target_sha }} + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + auto-push: true + + - name: Export dashboard + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/site" + git archive gh-pages:benchmarks | tar -x -C "$RUNNER_TEMP/site" + + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 + with: + path: ${{ runner.temp }}/site + + deploy: + name: Deploy benchmark dashboard + needs: publish + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + pages: write # Deploy the prepared Pages artifact + id-token: write # Authenticate the Pages deployment + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 + id: deployment diff --git a/scripts/README.bench_sdk.md b/scripts/README.bench_sdk.md new file mode 100644 index 00000000000..f5ee95ae613 --- /dev/null +++ b/scripts/README.bench_sdk.md @@ -0,0 +1,116 @@ +# SDK footprint and startup benchmark + +Keep the harness in this repository, outside `litellm/`. One invocation measures one explicitly selected source and writes one artifact directory. It never checks out a revision or updates a shared results file, so the same harness can measure older revisions from separate checkouts + +## Run locally + +Requires Linux or macOS, Python 3.10+, and uv. Select the same exact Python patch version on every comparison runner. The script declares pinned controller dependencies using inline script metadata, without installing LiteLLM's development environment + +```bash +uv run --no-project --python 3.11 scripts/bench_sdk.py --local . --output /tmp/sdk-current +``` + +Exactly one of `--local`, `--package`, and `--wheel` is required. `--local` without a path means the current working directory. `--package` accepts an exact published version, not a range or `latest`. Both package and wheel modes require a compatible binary wheel and never fall back to compiling an sdist + +Local mode copies source into a private temporary directory before invoking `pip wheel`, with PEP 517 build isolation enabled. Git checkouts include tracked working changes and untracked files that are not ignored; deleted and ignored files, Git metadata, and old ignored build products are excluded. Internal symlinks stay inside the copy. Non-Git source directories exclude common environment and build directories. Normal build outputs land in the copied source, and Cargo uses a private target directory. This is workspace isolation, not a security sandbox for untrusted build code. Use wheel mode when measuring a release artifact with generated assets absent from the source snapshot + +A checkout must have its normal wheel build prerequisites, including Rust when its build backend requires it. Building and resolving dependencies need network access, but neither operation is a latency metric + +For a quick smoke check: + +```bash +uv run --no-project --python 3.11 scripts/bench_sdk.py \ + --local . --output /tmp/sdk-smoke --samples 3 --install-samples 1 +``` + +For a published package, another checkout, or a prebuilt wheel: + +```bash +uv run --no-project --python 3.11 scripts/bench_sdk.py \ + --package 1.98.0 --output /tmp/sdk-published + +uv run --no-project --python 3.11 scripts/bench_sdk.py \ + --local /tmp/older-checkout --output /tmp/sdk-older + +uv run --no-project --python 3.11 scripts/bench_sdk.py \ + --wheel /tmp/litellm-version-platform.whl --extras proxy --output /tmp/sdk-proxy +``` + +Use the actual wheel filename in the last command. Extras change the installed dependency set; the workload still exercises SDK completion, not proxy server startup + +## Rust build contract + +The current root `pyproject.toml` declares `maturin==1.9.4` as the PEP 517 backend and points it at `litellm-rust/crates/python-bridge/Cargo.toml`, with module name `litellm.rust_bridge._native`. Local mode therefore compiles the extension as part of the wheel build before installing or timing anything. Older checkouts use their own declared backend. There is no separate hand-maintained `cargo build` command in this harness + +Python build isolation installs the backend, not a pinned Rust compiler or system linker. Those are runner prerequisites. The current Docker builder installs Rust and then installs the package via `uv sync`. CircleCI's Linux setup pins Rust 1.97.1, while its Windows setup uses floating `stable`. The workspace declares `rust-version = "1.88"`, but the bridge does not set `rust-version.workspace = true`. A consistent compiler pin across runners and inheritance of the intended minimum are remaining build reproducibility gaps, outside this benchmark change. [Cargo inheritance rules](https://doc.rust-lang.org/cargo/reference/workspaces.html#the-package-table) + +Result metadata identifies the source mode, whether a build occurred, and native files present in the selected wheel. Build output is retained in `run.log`. Cargo's compiler and registry caches may be shared, but compiled target output, source snapshots, Python environments, and measured runtime state are private to each invocation + +## What it measures + +| Result | Meaning | +| --- | --- | +| `sizes.root_wheel_bytes` | Compressed LiteLLM wheel | +| `sizes.resolved_wheelhouse_bytes` | LiteLLM and every selected dependency wheel, deduplicated by SHA-256 | +| `sizes.installed_delta_bytes` | File bytes after minus before each pristine install, including bytecode and entry points; symlinks excluded | +| `timings.offline_install` | Hash-verified installation and bytecode compilation from the wheelhouse; environment creation excluded | +| `timings.import` | Timer immediately around `import litellm` in each fresh process | +| `timings.configuration` | Public completion arguments and telemetry configuration after import | +| `timings.first_request` | First synchronous, non-streaming completion through a real HTTP client | +| `timings.second_request` | Second completion in the same process, with connection reuse available | +| `timings.import_to_first_response` | Start of import through the first response | +| `timings.launch_to_import` | Parent launch timestamp through the child's import-complete timestamp | +| `timings.launch_to_first_response` | Parent launch timestamp through the child's first response, excluding interpreter shutdown | +| `timings.python_startup_exit` | pyperf command running `python -I -B -c pass` | +| `timings.import_process_exit` | pyperf command running the guarded import probe, including process startup and shutdown | + +Durations are seconds. Every timing has raw samples, count, median, mean, sample standard deviation, median absolute deviation, minimum, and maximum. A single sample has no standard deviation. No percentile or statistical significance claim is made from small sample counts + +`diagnostics.json` records externally sampled RSS/USS and loaded modules at import, configuration, first response, and second response. These four snapshots come from a separate process with a controller handshake, not a timing run. Module lists are relative to the diagnostic baseline and cumulative. USS is null when unavailable. `importtime.log` is also a separate diagnostic run + +## Isolation and repeatability + +The controller contains pip, pyperf, psutil, and Pydantic for input validation. Target virtual environments have only the resolved wheel installation, without pip or benchmark packages. Every runtime sample starts a new interpreter with `-I`, an empty working directory, and an allowlisted environment. The current checkout and user site-packages cannot supply the import + +pip compiles bytecode during installation. Runtime probes use `-B`, which reads existing bytecode but does not write more. Warmups are discarded and the OS filesystem cache is not flushed. A private home directory starts empty for each invocation and is shared across its samples. This measures process-cold startup with warm filesystem caches, not first-ever token cache initialization or serverless platform startup + +The fake provider runs in the controller process on a dynamically assigned loopback port. It returns one fixed response, checks the request, and verifies exactly two requests per workflow probe. The SDK receives dummy credentials, zero retries, a request timeout, and the bundled model cost map setting. A Python audit hook blocks non-loopback socket operations and external DNS lookups; any blocked access fails the benchmark. This is a guard against accidental Python networking, not a security sandbox for native code or untrusted revisions + +Import timing preloads only `sys` and `time`, not JSON, HTTP clients, psutil, or pyperf. Process-to-stage times use the system-wide monotonic performance clock shared by processes on the supported platforms. The controller server, audit hook, and minimal probe scaffolding have overhead, so compare like-for-like runs + +## Dependencies and backfills + +Each run saves the exact wheelhouse, SHA-256 hashes, a hashed `requirements.lock`, dependency-only `constraints.txt`, pip installation reports, and an installed inventory. Dependencies must have binary wheels for the running Python/platform. An unavailable wheel fails explicitly instead of silently compiling an sdist + +By default each source resolves its own declared requirements against the current index. That captures dependency changes but does not reconstruct the index as it existed at an old commit. For an implementation comparison, pass the same dependency constraints to both revisions: + +```bash +uv run --no-project --python 3.11 scripts/bench_sdk.py \ + --local /tmp/candidate --constraints /tmp/sdk-current/constraints.txt \ + --output /tmp/sdk-candidate-pinned +``` + +An incompatible constraint fails instead of being relaxed. To replay without dependency downloads, use the saved root wheel as `--wheel` and the saved wheelhouse as `--wheelhouse`. This freezes the artifact universe on that Python/platform. `--package VERSION --wheelhouse PATH` also resolves that exact version from the archive. `--wheelhouse` does not make a local source build offline: the build backend and Rust dependencies can still need downloads. The controller tools must already be available locally + +Run multiple invocations against separate checkouts and separate output directories for backfills. Every invocation has private venvs, build output, pip cache, home, and an ephemeral provider port. Existing output paths are refused. Use separate runners for timing comparisons; simultaneous CPU or disk work on one host contaminates the numbers. Revision selection, job matrices, aggregation, and publishing belong in the future GitHub Actions layer + +## Output and validation + +Standard output contains only the complete JSON result; progress goes to stderr. `result.json` is written atomically only after every measurement succeeds. Failures exit nonzero and retain logs and artifacts, without a success result. Temporary target environments are always removed + +The result records Python/platform, target and harness Git revisions and dirty flags, harness file hashes, tool versions, runtime environment settings, wheel metadata, and raw workflow samples. Archive the entire output directory, not just `result.json` + +The pyperf files can be inspected with `pyperf check`, `pyperf stats`, and `pyperf compare_to`. Treat local smoke results as verification that the harness works, not evidence of a regression. Use repeated runs on the same idle runner before setting any thresholds + +Run the focused tests in a controller environment with the same pinned packages: + +```bash +uv run --no-project --with pip==26.2.1 --with pyperf==2.10.0 --with psutil==7.2.2 --with pydantic==2.13.4 \ + python -m unittest discover -s scripts -p test_bench_sdk.py -v +``` + +The tests use a small synthetic package to verify the harness without downloading LiteLLM dependencies. Real measurements always use the supplied LiteLLM wheel + +This initial suite does not measure streaming/async paths, proxy boot, every provider, native allocation profiles, online installation latency, or cloud platform startup + +Method references: [pip repeatable installs](https://pip.pypa.io/en/stable/topics/repeatable-installs/), [pip managing a separate interpreter](https://pip.pypa.io/en/stable/topics/python-option/), [pyperf command](https://pyperf.readthedocs.io/en/latest/cli.html#pyperf-command) diff --git a/scripts/bench_sdk.py b/scripts/bench_sdk.py new file mode 100644 index 00000000000..8a0ed72e7e5 --- /dev/null +++ b/scripts/bench_sdk.py @@ -0,0 +1,465 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["pip==26.2.1", "pyperf==2.10.0", "psutil==7.2.2", "pydantic==2.13.4"] +# /// +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time +import zipfile +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from email.parser import BytesParser +from functools import partial +from pathlib import Path +from typing import Final, Literal, TextIO + +from bench_sdk_runtime import PROBE, command, probe, provider, require, runtime_environment, startup, summary +from pydantic import TypeAdapter + + +@dataclass(frozen=True, slots=True) +class Wheel: + filename: str + name: str + version: str + sha256: str + compressed_bytes: int + uncompressed_bytes: int + tags: tuple[str, ...] + extras: tuple[str, ...] + native_files: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class Source: + kind: Literal["local", "package", "wheel"] + value: str + + +@dataclass(frozen=True, slots=True) +class Options: + source: Source + output: Path + extras: str + samples: int + install_samples: int + warmups: int + timeout: int + constraints: Path | None + wheelhouse: Path | None + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + hasher: Final = hashlib.sha256() + for chunk in iter(partial(stream.read, 1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def wheel_info(path: Path) -> Wheel: + with zipfile.ZipFile(path) as archive: + metadata_files: Final = tuple(name for name in archive.namelist() if name.endswith(".dist-info/METADATA")) + require(len(metadata_files) == 1, f"Expected one wheel METADATA file in {path}") + metadata: Final = BytesParser().parsebytes(archive.read(metadata_files[0])) + wheel_metadata: Final = BytesParser().parsebytes( + archive.read(metadata_files[0].removesuffix("METADATA") + "WHEEL") + ) + return Wheel( + path.name, + re.sub(r"[-_.]+", "-", str(metadata["Name"])).lower(), + str(metadata["Version"]), + digest(path), + path.stat().st_size, + sum(item.file_size for item in archive.infolist()), + tuple(wheel_metadata.get_all("Tag", ())), + tuple(metadata.get_all("Provides-Extra", ())), + tuple(name for name in archive.namelist() if name.endswith((".so", ".pyd", ".dylib"))), + ) + + +def file_bytes(root: Path) -> int: + return sum(path.stat().st_size for path in root.rglob("*") if path.is_file() and not path.is_symlink()) + + +def git_metadata(source: Path) -> dict[str, object]: + if not source.is_dir(): + return {"path": str(source), "commit": None, "dirty": None} + head: Final = subprocess.run( + ("git", "-C", str(source), "rev-parse", "HEAD"), + capture_output=True, + text=True, + ) + if head.returncode: + return {"path": str(source), "commit": None, "dirty": None} + status: Final = subprocess.run( + ("git", "-C", str(source), "status", "--porcelain"), + capture_output=True, + text=True, + check=True, + ) + return {"path": str(source), "commit": head.stdout.strip(), "dirty": bool(status.stdout.strip())} + + +def lock_text(wheels: Sequence[Wheel], extras: str) -> str: + return "".join( + f"{wheel.name}{f'[{extras}]' if wheel.name == 'litellm' and extras else ''}" + f"=={wheel.version} --hash=sha256:{wheel.sha256}\n" + for wheel in sorted(wheels, key=lambda item: item.name) + ) + + +def snapshot(source: Path, destination: Path) -> None: + files: Final = subprocess.run( + ("git", "-C", str(source), "ls-files", "--cached", "--others", "--exclude-standard", "-z"), + capture_output=True, + check=False, + ) + if files.returncode: + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns(".git", ".venv", "venv", "__pycache__", "target", "dist", "build"), + ) + return + destination.mkdir() + for original, copied in ( + (source / relative, destination / relative) + for relative in frozenset(os.fsdecode(name) for name in files.stdout.split(b"\0") if name) + ): + if not original.exists() and not original.is_symlink(): + continue + copied.parent.mkdir(parents=True, exist_ok=True) + if original.is_symlink(): + require(original.resolve().is_relative_to(source.resolve()), f"Symlink escapes local source: {original}") + copied.symlink_to( + os.path.relpath(destination / original.resolve().relative_to(source.resolve()), copied.parent) + ) + continue + require(original.is_file(), f"Local snapshot needs a populated source tree, not a Git submodule: {original}") + shutil.copy2(original, copied) + + +def prepare( + source: Source, + output: Path, + work: Path, + extras: str, + constraints: Path | None, + wheelhouse: Path | None, + environment: Mapping[str, str], + log: TextIO, +) -> tuple[Wheel, ...]: + destination: Final = output / "wheelhouse" + destination.mkdir() + root_wheels: Final = work / "root-wheel" + root_wheels.mkdir() + offline: Final = ("--no-index", "--find-links", str(wheelhouse)) if wheelhouse else () + if source.kind == "local": + copied: Final = work / "source" + snapshot(Path(source.value).resolve(), copied) + command( + ( + sys.executable, + "-m", + "pip", + "wheel", + "--no-deps", + "--no-cache-dir", + "--wheel-dir", + str(root_wheels), + str(copied), + ), + work, + environment, + log, + timeout=1800, + ) + elif source.kind == "package": + command( + ( + sys.executable, + "-m", + "pip", + "download", + "--no-deps", + "--only-binary=:all:", + "--dest", + str(root_wheels), + *offline, + f"litellm=={source.value}", + ), + work, + environment, + log, + ) + else: + wheel: Final = Path(source.value).resolve() + shutil.copy2(wheel, root_wheels / wheel.name) + roots: Final = tuple(root_wheels.glob("*.whl")) + require(len(roots) == 1 and wheel_info(roots[0]).name == "litellm", "Source must produce one LiteLLM wheel") + require(not extras or set(extras.split(",")).issubset(wheel_info(roots[0]).extras), "Unknown installation extra") + command( + ( + sys.executable, + "-m", + "pip", + "download", + "--only-binary=:all:", + "--dest", + str(destination), + *(("--constraint", str(constraints)) if constraints else ()), + *offline, + f"{roots[0]}{f'[{extras}]' if extras else ''}", + ), + work, + environment, + log, + ) + wheels: Final = tuple(wheel_info(path) for path in sorted(destination.glob("*.whl"))) + require(len({wheel.name for wheel in wheels}) == len(wheels), "Multiple wheels for one distribution") + (output / "requirements.lock").write_text(lock_text(wheels, extras)) + (output / "constraints.txt").write_text( + "".join(f"{wheel.name}=={wheel.version}\n" for wheel in wheels if wheel.name != "litellm") + ) + return wheels + + +def install_sample( + index: int, + output: Path, + work: Path, + environment: Mapping[str, str], + log: TextIO, +) -> tuple[Path, float, int, int]: + target: Final = work / f"target-{index}" + command((sys.executable, "-I", "-m", "venv", "--without-pip", str(target)), work, environment, log) + before: Final = file_bytes(target) + started: Final = time.perf_counter_ns() + command( + ( + sys.executable, + "-m", + "pip", + "--python", + str(target), + "install", + "--no-index", + "--no-cache-dir", + "--only-binary=:all:", + "--require-hashes", + "--compile", + "--find-links", + str(output / "wheelhouse"), + "--report", + str(output / f"install-{index}.json"), + "-r", + str(output / "requirements.lock"), + ), + work, + environment, + log, + ) + elapsed: Final = (time.perf_counter_ns() - started) / 1e9 + after: Final = file_bytes(target) + return target / "bin" / "python", elapsed, before, after + + +def positive_int(value: str) -> int: + parsed: Final = int(value) + require(parsed > 0, "Counts and timeouts must be positive") + return parsed + + +def arguments() -> Options: + parser: Final = argparse.ArgumentParser( + description="Benchmark a local LiteLLM build, a pinned published package, or an existing wheel" + ) + sources: Final = parser.add_mutually_exclusive_group(required=True) + sources.add_argument( + "--local", + dest="source", + type=lambda value: Source("local", value), + nargs="?", + const=Source("local", "."), + help="Build a private copy of a local checkout (omit path for cwd)", + ) + sources.add_argument( + "--package", + dest="source", + type=lambda value: Source("package", value), + help="Download exactly this published LiteLLM version; never build from source", + ) + sources.add_argument( + "--wheel", + dest="source", + type=lambda value: Source("wheel", value), + help="Use this existing wheel; never build from source", + ) + parser.add_argument("--output", type=Path, required=True, help="New artifact directory; existing paths are refused") + parser.add_argument("--extras", default="", help="Comma-separated installation extras, e.g. proxy") + parser.add_argument("--samples", type=positive_int, default=10, help="Fresh-process timing samples (default: 10)") + parser.add_argument( + "--install-samples", type=positive_int, default=3, help="Pristine offline installs (default: 3)" + ) + parser.add_argument("--warmups", type=positive_int, default=1, help="Untimed workflow warmups (default: 1)") + parser.add_argument("--timeout", type=positive_int, default=120, help="Seconds allowed per runtime probe") + parser.add_argument("--constraints", type=Path, help="Pinned dependency constraints for implementation comparisons") + parser.add_argument("--wheelhouse", type=Path, help="Resolve dependencies offline from an archived wheelhouse") + return TypeAdapter(Options).validate_python(vars(parser.parse_args())) + + +def benchmark(options: Options, output: Path, work: Path, log: TextIO) -> dict[str, object]: + source: Final = options.source + if source.kind == "package": + require(bool(re.fullmatch(r"[0-9][A-Za-z0-9.!+_-]*", source.value)), "--package requires an exact version") + elif source.kind == "local": + require(Path(source.value).is_dir(), "--local must name a source directory") + else: + require(Path(source.value).is_file() and source.value.endswith(".whl"), "--wheel must name a wheel file") + require(bool(re.fullmatch(r"[A-Za-z0-9_,-]*", options.extras)), "Invalid extras") + provenance: Final = { + "kind": source.kind, + "requested": source.value, + "built_from_source": source.kind == "local", + **(git_metadata(Path(source.value).resolve()) if source.kind != "package" else {}), + } + harness: Final = { + **git_metadata(Path(__file__).resolve().parent), + "files": {name: digest(Path(__file__).with_name(name)) for name in ("bench_sdk.py", "bench_sdk_runtime.py")}, + "tools": {name: importlib.metadata.version(name) for name in ("pip", "pyperf", "psutil", "pydantic")}, + } + home: Final = work / "home" + home.mkdir() + runtime: Final = work / "runtime" + runtime.mkdir() + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith(("PIP_", "PYTHON"))}, + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INPUT": "1", + "PIP_CACHE_DIR": str(work / "pip-cache"), + "CARGO_TARGET_DIR": str(work / "cargo-target"), + } + sys.stderr.write(f"Source: {source.kind} ({source.value})\n") + sys.stderr.write( + "Building a private source copy, then resolving binary dependencies...\n" + if source.kind == "local" + else "Resolving binary wheels only; source builds disabled...\n" + ) + wheels: Final = prepare( + source, + output, + work, + options.extras, + options.constraints.resolve() if options.constraints else None, + options.wheelhouse.resolve() if options.wheelhouse else None, + environment, + log, + ) + sys.stderr.write("Measuring pristine offline installs...\n") + installs: Final = tuple( + install_sample(index, output, work, environment, log) for index in range(options.install_samples) + ) + python: Final = installs[-1][0] + command((sys.executable, "-m", "pip", "--python", str(python), "check"), work, environment, log) + inventory: Final = command( + (sys.executable, "-m", "pip", "--python", str(python), "inspect"), work, environment, log + ) + (output / "installed.json").write_text(inventory) + runtime_env: Final = runtime_environment(home) + sys.stderr.write("Measuring fresh-process imports and local responses...\n") + with provider() as (base_url, requests): + for _ in range(options.warmups): + probe(python, base_url, runtime, runtime_env, log, requests, timeout=options.timeout) + samples: Final = tuple( + probe(python, base_url, runtime, runtime_env, log, requests, timeout=options.timeout)[0] + for _ in range(options.samples) + ) + _, diagnostics = probe( + python, base_url, runtime, runtime_env, log, requests, diagnostic=True, timeout=options.timeout + ) + (output / "diagnostics.json").write_text(json.dumps(diagnostics, indent=2) + "\n") + startup_metrics: Final = startup(python, output, runtime, runtime_env, log, options.samples, options.timeout) + with (output / "importtime.log").open("w") as profile: + command( + (str(python), "-I", "-B", "-X", "importtime", "-c", PROBE, "import_exit"), + runtime, + runtime_env, + profile, + options.timeout, + ) + log.flush() + root: Final = next(wheel for wheel in wheels if wheel.name == "litellm") + return { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "source": provenance, + "harness": harness, + "environment": { + "python": sys.version, + "implementation": platform.python_implementation(), + "platform": platform.platform(), + "machine": platform.machine(), + "cpu_count": os.cpu_count(), + "extras": options.extras, + "runtime_variables": runtime_env, + "bytecode": "pip --compile, probes -B (read existing pyc, never write)", + "filesystem_cache": "warmup runs; OS page cache not flushed", + "network": "Python socket audit guard allows loopback only; not an OS sandbox", + "scenario": "synchronous non-streaming completion; fixed loopback provider; retries disabled", + "warmups": options.warmups, + "constraints_sha256": digest(options.constraints) if options.constraints else None, + }, + "artifacts": {"root": asdict(root), "wheels": tuple(asdict(wheel) for wheel in wheels)}, + "sizes": { + "root_wheel_bytes": root.compressed_bytes, + "root_uncompressed_bytes": root.uncompressed_bytes, + "resolved_wheelhouse_bytes": sum({wheel.sha256: wheel.compressed_bytes for wheel in wheels}.values()), + "dependency_wheel_bytes": sum(wheel.compressed_bytes for wheel in wheels if wheel.name != "litellm"), + "environment_before_bytes": tuple(before for _, _, before, _ in installs), + "environment_after_bytes": tuple(after for _, _, _, after in installs), + "installed_delta_bytes": tuple(after - before for _, _, before, after in installs), + }, + "timings": { + "offline_install": summary(tuple(elapsed for _, elapsed, _, _ in installs), "seconds"), + **{ + key.removesuffix("_ns"): summary(tuple(sample[key] / 1e9 for sample in samples), "seconds") + for key in samples[0] + }, + **startup_metrics, + }, + "raw_workflow_samples_ns": samples, + "memory": tuple({key: value for key, value in stage.items() if key != "modules"} for stage in diagnostics), + "diagnostics": "diagnostics.json", + } + + +def main() -> None: + options: Final = arguments() + require(sys.platform in ("linux", "darwin"), "This benchmark currently supports Linux and macOS") + output: Final = options.output.resolve() + require(not output.exists(), f"Output already exists: {output}") + output.mkdir(parents=True) + with tempfile.TemporaryDirectory(prefix="litellm-sdk-bench-") as temporary, (output / "run.log").open("w") as log: + result: Final = benchmark(options, output, Path(temporary), log) + temporary_result: Final = output / "result.json.tmp" + temporary_result.write_text(json.dumps(result, indent=2) + "\n") + temporary_result.replace(output / "result.json") + sys.stdout.write(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/bench_sdk_runtime.py b/scripts/bench_sdk_runtime.py new file mode 100644 index 00000000000..978e5d7e96e --- /dev/null +++ b/scripts/bench_sdk_runtime.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import json +import os +import signal +import socket +import statistics +import subprocess +import sys +import threading +import time +from collections.abc import Generator, Iterator, Mapping, Sequence +from contextlib import contextmanager +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from queue import SimpleQueue +from socketserver import BaseServer +from typing import Final, Protocol, TextIO, cast + +import psutil +import pyperf # pyright: ignore[reportMissingTypeStubs] # Upstream does not publish typing stubs +from pydantic import BaseModel, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class MemoryInfo(BaseModel): + rss: int + uss: int | None = None + + +def memory_info(pid: int) -> MemoryInfo: + process: Final = psutil.Process(pid) + try: + return MemoryInfo.model_validate(process.memory_full_info(), from_attributes=True) + except (psutil.AccessDenied, AttributeError): + return MemoryInfo.model_validate(process.memory_info(), from_attributes=True) + + +class BenchmarkValues(Protocol): + def get_values(self) -> Sequence[float]: ... + + +PROBE: Final = r""" +import sys +import time + +def guard(event, args): + if event in ("socket.connect", "socket.sendto"): + address = args[-1] + allowed = isinstance(address, tuple) and address[0] in ("127.0.0.1", "::1") + elif event in ("socket.getaddrinfo", "socket.gethostbyname", "socket.gethostbyaddr"): + allowed = args[0] in ("127.0.0.1", "::1", "localhost") + else: + return + if not allowed: + import os + sys.stderr.write("BENCH_EGRESS_BLOCKED\n") + sys.stderr.flush() + os._exit(1) + +sys.addaudithook(guard) +mode = sys.argv[1] +if mode == "diagnostic": + import json + baseline = frozenset(sys.modules) + +def stage(name): + if mode == "diagnostic": + print("BENCH:" + json.dumps({"stage": name, "modules": sorted(set(sys.modules) - baseline)}), flush=True) + if sys.stdin.readline() != "continue\n": + sys.exit("Missing diagnostic acknowledgement") + +started = time.perf_counter_ns() +import litellm +imported = time.perf_counter_ns() + +if mode == "import_exit": + sys.exit(0) + +stage("after_import") +litellm.telemetry = False +arguments = dict( + model="openai/benchmark-model", + messages=[{"role": "user", "content": "Hi"}], + api_base=sys.argv[2], + api_key="benchmark-dummy-key", + num_retries=0, + timeout=10, + max_tokens=1, +) +configured = time.perf_counter_ns() +stage("after_configuration") +first_started = time.perf_counter_ns() +first = litellm.completion(**arguments) +first_finished = time.perf_counter_ns() +if first.choices[0].message.content != "ok": + sys.exit("Unexpected first response") +stage("after_first_response") +second_started = time.perf_counter_ns() +second = litellm.completion(**arguments) +second_finished = time.perf_counter_ns() +if second.choices[0].message.content != "ok": + sys.exit("Unexpected second response") +stage("after_second_response") +if mode != "diagnostic": + import json + print("BENCH:" + json.dumps(dict( + import_ns=imported-started, + configuration_ns=configured-imported, + first_request_ns=first_finished-first_started, + second_request_ns=second_finished-second_started, + import_to_first_response_ns=first_finished-started, + imported_at_ns=imported, + first_response_at_ns=first_finished, + ))) +""" + + +def require(condition: bool, message: str) -> None: + if not condition: + sys.exit(message) + + +def stop_process(process: subprocess.Popen[str]) -> None: + if process.poll() is None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + + +def command( + arguments: Sequence[str], directory: Path, environment: Mapping[str, str], log: TextIO, timeout: float = 900 +) -> str: + log.write(f"\n$ {arguments!r}\n") + log.flush() + with subprocess.Popen( + arguments, + cwd=directory, + env=environment, + stdout=subprocess.PIPE, + stderr=log, + text=True, + start_new_session=True, + ) as process: + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + stop_process(process) + process.communicate() + sys.exit(f"Command timed out after {timeout}s; see {log.name}") + log.write(output) + log.flush() + require(process.returncode == 0, f"Command failed ({process.returncode}); see {log.name}") + return output + + +def runtime_environment(home: Path) -> dict[str, str]: + return { + "PATH": os.defpath, + "HOME": str(home), + "TMPDIR": str(home), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "LITELLM_LOG": "ERROR", + "TOKENIZERS_PARALLELISM": "false", + "HF_HUB_OFFLINE": "1", + "AWS_EC2_METADATA_DISABLED": "true", + } + + +class ProviderHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + disable_nagle_algorithm = True + + def __init__( + self, + request: socket.socket, + client_address: tuple[str, int], + server: BaseServer, + *, + requests: SimpleQueue[bool], + ) -> None: + self.requests = requests + super().__init__(request, client_address, server) + + def log_message(self, format: str, *args: object) -> None: + return + + def log_request(self, code: int | str = "-", size: int | str = "-") -> None: + self.requests.put(code == 200) + + def do_POST(self) -> None: + body: Final = JSON_OBJECT.validate_json(self.rfile.read(int(self.headers.get("Content-Length", "0")))) + valid: Final = ( + self.path == "/v1/chat/completions" + and self.headers.get("Authorization") == "Bearer benchmark-dummy-key" + and body.get("model") == "benchmark-model" + and body.get("messages") == [{"role": "user", "content": "Hi"}] + and not body.get("stream", False) + ) + if not valid: + self.send_error(400, "Unexpected benchmark request") + return + payload: Final = json.dumps( + { + "id": "chatcmpl-benchmark", + "object": "chat.completion", + "created": 0, + "model": "benchmark-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +@contextmanager +def provider() -> Generator[tuple[str, SimpleQueue[bool]]]: + requests: Final[SimpleQueue[bool]] = SimpleQueue() + with ThreadingHTTPServer(("127.0.0.1", 0), partial(ProviderHandler, requests=requests)) as server: + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1", requests + finally: + server.shutdown() + thread.join() + + +def summary(values: Sequence[float], unit: str) -> dict[str, object]: + require(bool(values), "Cannot summarize an empty sample") + median: Final = statistics.median(values) + return { + "unit": unit, + "samples": tuple(values), + "count": len(values), + "median": median, + "mean": statistics.mean(values), + "stdev": statistics.stdev(values) if len(values) > 1 else None, + "mad": statistics.median(abs(value - median) for value in values), + "min": min(values), + "max": max(values), + } + + +def probe( + python: Path, + base_url: str, + directory: Path, + environment: Mapping[str, str], + log: TextIO, + requests: SimpleQueue[bool], + *, + diagnostic: bool = False, + timeout: float = 120, +) -> tuple[dict[str, int], tuple[dict[str, object], ...]]: + require(requests.empty(), "Unexpected provider request between probes") + mode: Final = "diagnostic" if diagnostic else "timing" + launched: Final = time.perf_counter_ns() + with subprocess.Popen( + (str(python), "-I", "-B", "-c", PROBE, mode, base_url), + cwd=directory, + env=environment, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=log, + text=True, + start_new_session=True, + ) as process: + timer: Final = threading.Timer(timeout, stop_process, args=(process,)) + timer.start() + try: + if process.stdout is None or process.stdin is None: + sys.exit("Probe pipes unavailable") + incoming: Final = process.stdout + outgoing: Final = process.stdin + + def record(line: str) -> dict[str, object]: + payload: Final = JSON_OBJECT.validate_json(line.removeprefix("BENCH:")) + if diagnostic: + memory: Final = memory_info(process.pid) + return acknowledge({**payload, "rss_bytes": memory.rss, "uss_bytes": memory.uss}) + return payload + + def acknowledge(payload: dict[str, object]) -> dict[str, object]: + outgoing.write("continue\n") + outgoing.flush() + return payload + + lines: Final = cast(Iterator[str], iter(incoming.readline, "")) + records: Final = tuple(record(line) for line in lines if line.startswith("BENCH:")) + process.wait() + require(process.returncode == 0, f"Probe failed or timed out; see {log.name}") + require(requests.qsize() == 2, f"Expected two provider requests, received {requests.qsize()}") + require(all(requests.get() for _ in range(2)), "Provider rejected a request") + if diagnostic: + require(len(records) == 4, "Incomplete diagnostic probe") + return {}, records + require(len(records) == 1, "Missing timing record") + timings: Final = {key: int(value) for key, value in records[0].items() if isinstance(value, int)} + return { + **{key: value for key, value in timings.items() if not key.endswith("_at_ns")}, + "launch_to_import_ns": timings["imported_at_ns"] - launched, + "launch_to_first_response_ns": timings["first_response_at_ns"] - launched, + }, () + finally: + timer.cancel() + stop_process(process) + process.wait() + + +def startup( + python: Path, + output: Path, + directory: Path, + environment: Mapping[str, str], + log: TextIO, + samples: int, + timeout: float, +) -> dict[str, object]: + for name, code, arguments in ( + ("python_startup_exit", "pass", ()), + ("import_process_exit", f"exec({PROBE!r})", ("import_exit",)), + ): + command( + ( + sys.executable, + "-m", + "pyperf", + "command", + "--copy-env", + "--name", + name, + "--processes", + str(samples), + "--values", + "1", + "--loops", + "1", + "--warmups", + "1", + "--timeout", + str(int(timeout)), + "-o", + str(output / f"{name}.json"), + "--", + str(python), + "-I", + "-B", + "-c", + code, + *arguments, + ), + directory, + environment, + log, + timeout * (samples + 1), + ) + return { + name: summary(load_pyperf(output / f"{name}.json").get_values(), "seconds") + for name in ("python_startup_exit", "import_process_exit") + } + + +def load_pyperf(path: Path) -> BenchmarkValues: + return cast(BenchmarkValues, pyperf.Benchmark.load(str(path))) # pyright: ignore[reportUnknownMemberType] # Untyped upstream API diff --git a/scripts/test_bench_sdk.py b/scripts/test_bench_sdk.py new file mode 100644 index 00000000000..64e702a964e --- /dev/null +++ b/scripts/test_bench_sdk.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import http.client +import json +import subprocess +import sys +import tempfile +import unittest +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +from bench_sdk import file_bytes, lock_text, snapshot, wheel_info +from bench_sdk_runtime import PROBE, probe, provider, runtime_environment, summary + +FAKE_SDK: Final = """ +import sys +import time +if sys.argv[1] != "diagnostic": + assert "json" not in sys.modules + assert "typing" not in sys.modules + assert "http.client" not in sys.modules + assert "psutil" not in sys.modules + assert "pyperf" not in sys.modules +time.sleep(0.02) + +def completion(**arguments): + import json + from types import SimpleNamespace + from urllib.request import Request, urlopen + request = Request( + arguments["api_base"] + "/chat/completions", + data=json.dumps({"model": "benchmark-model", "messages": arguments["messages"]}).encode(), + headers={"Authorization": "Bearer " + arguments["api_key"], "Content-Type": "application/json"}, + ) + with urlopen(request, timeout=5) as response: + payload = json.load(response) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace( + content=payload["choices"][0]["message"]["content"] + ))]) +""" + + +def make_wheel(directory: Path) -> Path: + wheel: Final = directory / "litellm-0.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("litellm/__init__.py", FAKE_SDK) + archive.writestr( + "litellm-0.0.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: litellm\nVersion: 0.0.0\nProvides-Extra: proxy\n", + ) + archive.writestr( + "litellm-0.0.0.dist-info/WHEEL", "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + ) + archive.writestr("litellm-0.0.0.dist-info/RECORD", "") + return wheel + + +def make_target(directory: Path, code: str = FAKE_SDK) -> Path: + target: Final = directory / "target" + subprocess.run((sys.executable, "-I", "-m", "venv", "--without-pip", str(target)), check=True) + site: Final = target / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" + (site / "litellm.py").write_text(code) + (directory / "litellm.py").write_text('raise RuntimeError("Imported from current directory")') + return target / "bin" / "python" + + +class BenchmarkTests(unittest.TestCase): + def test_parallel_runs_use_distinct_environments_and_outputs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + wheel: Final = make_wheel(root) + + def run(index: int) -> dict[str, object]: + result: Final = subprocess.run( + ( + sys.executable, + str(Path(__file__).with_name("bench_sdk.py")), + "--wheel", + str(wheel), + "--wheelhouse", + str(root), + "--output", + str(root / str(index)), + "--samples", + "1", + "--install-samples", + "1", + ), + capture_output=True, + text=True, + timeout=90, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + with ThreadPoolExecutor(max_workers=2) as pool: + first, second = tuple(pool.map(run, (0, 1))) + self.assertNotEqual( + first["environment"]["runtime_variables"]["HOME"], + second["environment"]["runtime_variables"]["HOME"], + ) + self.assertTrue((root / "0" / "result.json").exists()) + self.assertTrue((root / "1" / "result.json").exists()) + + def test_local_snapshot_preserves_working_changes_without_build_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + source: Final = root / "checkout" + source.mkdir() + subprocess.run(("git", "init", "-q", str(source)), check=True) + (source / ".gitignore").write_text("target/\n") + (source / "tracked.py").write_text("original") + (source / "deleted.py").write_text("deleted") + subprocess.run(("git", "-C", str(source), "add", "."), check=True) + (source / "tracked.py").write_text("working changes") + (source / "deleted.py").unlink() + (source / "untracked.py").write_text("new source") + (source / "linked.py").symlink_to("tracked.py") + (source / "target").mkdir() + (source / "target" / "old.so").write_text("old build") + copied: Final = root / "copied" + snapshot(source, copied) + self.assertEqual((copied / "tracked.py").read_text(), "working changes") + self.assertEqual((copied / "untracked.py").read_text(), "new source") + self.assertFalse((copied / "deleted.py").exists()) + self.assertFalse((copied / "target").exists()) + self.assertFalse((copied / ".git").exists()) + self.assertTrue((copied / "linked.py").is_symlink()) + self.assertEqual((copied / "linked.py").read_text(), "working changes") + (copied / "tracked.py").write_text("build mutated its copy") + self.assertEqual((copied / "linked.py").read_text(), "build mutated its copy") + self.assertEqual((source / "tracked.py").read_text(), "working changes") + + def test_cli_requires_one_explicit_source(self) -> None: + script: Final = str(Path(__file__).with_name("bench_sdk.py")) + for arguments in (("--output", "/unused"), ("--local", ".", "--package", "0.0.0", "--output", "/unused")): + result: Final = subprocess.run((sys.executable, script, *arguments), capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + + def test_summary_retains_samples_and_does_not_invent_single_sample_variance(self) -> None: + result: Final = summary((1.0, 2.0, 6.0), "seconds") + self.assertEqual(result["samples"], (1.0, 2.0, 6.0)) + self.assertEqual(result["median"], 2.0) + self.assertEqual(result["mean"], 3.0) + self.assertEqual(result["mad"], 1.0) + self.assertIsNone(summary((2.0,), "seconds")["stdev"]) + with self.assertRaises(SystemExit): + summary((), "seconds") + + def test_size_counts_bytecode_but_not_interpreter_symlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + (root / "code.py").write_bytes(b"123") + (root / "code.pyc").write_bytes(b"12345") + (root / "python").symlink_to(sys.executable) + self.assertEqual(file_bytes(root), 8) + + def test_wheel_lock_preserves_extras_version_and_artifact_hash(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + wheel: Final = wheel_info(make_wheel(root)) + self.assertEqual(wheel.tags, ("py3-none-any",)) + self.assertEqual(wheel.extras, ("proxy",)) + self.assertGreater(wheel.uncompressed_bytes, 0) + self.assertEqual(lock_text((wheel,), "proxy"), f"litellm[proxy]==0.0.0 --hash=sha256:{wheel.sha256}\n") + + def test_runtime_does_not_inherit_credentials_or_import_overrides(self) -> None: + environment: Final = runtime_environment(Path("/tmp/benchmark-home")) + for name in ("OPENAI_API_KEY", "AWS_ACCESS_KEY_ID", "PYTHONPATH", "HTTP_PROXY", "HTTPS_PROXY"): + self.assertNotIn(name, environment) + self.assertEqual(environment["LITELLM_LOCAL_MODEL_COST_MAP"], "True") + + def test_independent_providers_reject_unexpected_requests(self) -> None: + with provider() as (first, first_requests), provider() as (second, second_requests): + self.assertNotEqual(first, second) + connection: Final = http.client.HTTPConnection(first.removeprefix("http://").removesuffix("/v1")) + try: + connection.request("POST", "/wrong", body="{}", headers={"Content-Type": "application/json"}) + response: Final = connection.getresponse() + self.assertEqual(response.status, 400) + response.read() + finally: + connection.close() + self.assertFalse(first_requests.get_nowait()) + self.assertTrue(second_requests.empty()) + + def test_probe_uses_fresh_isolated_import_and_real_http_and_separate_diagnostics(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + python: Final = make_target(root) + with (root / "probe.log").open("w") as log, provider() as (url, requests): + timing, _ = probe(python, url, root, runtime_environment(root), log, requests) + self.assertGreater(timing["import_ns"], 15_000_000) + self.assertGreater(timing["launch_to_import_ns"], timing["import_ns"]) + self.assertGreater(timing["launch_to_first_response_ns"], timing["import_to_first_response_ns"]) + self.assertGreater(timing["first_request_ns"], 0) + self.assertGreater(timing["second_request_ns"], 0) + _, diagnostics = probe(python, url, root, runtime_environment(root), log, requests, diagnostic=True) + self.assertEqual( + tuple(stage["stage"] for stage in diagnostics), + ( + "after_import", + "after_configuration", + "after_first_response", + "after_second_response", + ), + ) + self.assertIn("litellm", diagnostics[0]["modules"]) + self.assertGreater(diagnostics[0]["rss_bytes"], 0) + + def test_probe_timeout_is_a_failure(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + python: Final = make_target(root, "import time\ntime.sleep(30)\n") + with (root / "probe.log").open("w") as log, provider() as (url, requests): + with self.assertRaisesRegex(SystemExit, "timed out"): + probe(python, url, root, runtime_environment(root), log, requests, timeout=0.1) + + def test_egress_guard_rejects_external_dns_before_network_access(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + python: Final = make_target(root, 'import socket\nsocket.getaddrinfo("example.invalid", 443)\n') + result: Final = subprocess.run( + (str(python), "-I", "-B", "-c", PROBE, "import_exit"), + cwd=root, + env=runtime_environment(root), + capture_output=True, + text=True, + timeout=10, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("BENCH_EGRESS_BLOCKED", result.stderr) + + def test_full_cli_offline_artifacts_and_refusal_to_overwrite(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root: Final = Path(temporary) + wheel: Final = make_wheel(root) + output: Final = root / "results" + command: Final = ( + sys.executable, + str(Path(__file__).with_name("bench_sdk.py")), + "--wheel", + str(wheel), + "--wheelhouse", + str(root), + "--output", + str(output), + "--samples", + "2", + "--install-samples", + "2", + "--extras", + "proxy", + ) + result: Final = subprocess.run(command, capture_output=True, text=True, timeout=90) + self.assertEqual(result.returncode, 0, result.stderr + (output / "run.log").read_text()) + payload: Final = json.loads(result.stdout) + self.assertEqual(payload, json.loads((output / "result.json").read_text())) + self.assertEqual(payload["timings"]["import"]["count"], 2) + self.assertEqual(payload["source"]["kind"], "wheel") + self.assertFalse(payload["source"]["built_from_source"]) + self.assertEqual(payload["timings"]["offline_install"]["count"], 2) + self.assertEqual(payload["sizes"]["resolved_wheelhouse_bytes"], wheel.stat().st_size) + self.assertGreater( + payload["sizes"]["installed_delta_bytes"][0], payload["sizes"]["root_uncompressed_bytes"] + ) + self.assertEqual( + tuple( + item["metadata"]["name"] + for item in json.loads((output / "installed.json").read_text())["installed"] + ), + ("litellm",), + ) + duplicate: Final = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertNotEqual(duplicate.returncode, 0) + self.assertIn("already exists", duplicate.stderr) + published: Final = subprocess.run( + ( + sys.executable, + str(Path(__file__).with_name("bench_sdk.py")), + "--package", + "0.0.0", + "--wheelhouse", + str(root), + "--output", + str(root / "published"), + "--samples", + "1", + "--install-samples", + "1", + ), + capture_output=True, + text=True, + timeout=90, + ) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual( + json.loads(published.stdout)["source"], + { + "kind": "package", + "requested": "0.0.0", + "built_from_source": False, + }, + ) + self.assertNotIn("'wheel'", (root / "published" / "run.log").read_text()) + + +if __name__ == "__main__": + unittest.main()