scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

Merge 2880824203 into 570f41daad

Browse source
This commit is contained in:
yujonglee 2026-11-04 00:41:37 +00:00 • committed by GitHub
commit 60f8896852
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1057 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.github/actions/pytest-report/action.yml vendored Normal file
View file

@ -0,0 +1,38 @@
name: "Report pytest outcomes"
description: >-
Append non-gating pytest outcomes and metrics to the job summary, and optionally
retain the raw report for later comparison.
inputs:
report:
description: "Path to the pytest report JSON"
required: true
title:
description: "Heading shown in the job summary"
required: true
artifact-name:
description: "Optional artifact name for retaining the raw report"
required: false
default: ""
runs:
using: composite
steps:
- name: Render pytest report
if: always()
continue-on-error: true
shell: bash
env:
REPORT: ${{ inputs.report }}
TITLE: ${{ inputs.title }}
run: python3 "$GITHUB_ACTION_PATH/../../scripts/render_pytest_report.py" --title "$TITLE" "$REPORT" >> "$GITHUB_STEP_SUMMARY"
- name: Upload pytest report
if: ${{ always() && inputs.artifact-name != '' }}
continue-on-error: true
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ inputs.artifact-name }}
path: ${{ inputs.report }}
retention-days: 7
if-no-files-found: ignore

206
.github/scripts/render_pytest_report.py vendored Normal file
View file

@ -0,0 +1,206 @@
from __future__ import annotations
import argparse
import html
import json
import sys
from collections import Counter
from dataclasses import dataclass
from itertools import accumulate, groupby
from os.path import commonpath
from pathlib import Path
from typing import Final, cast
TOTAL_ORDER: Final = ("failed", "passed", "skipped", "xfailed", "xpassed", "error")
COLUMNS: Final = ("passed", "failed", "error", "skipped", "xfailed", "xpassed")
@dataclass(frozen=True, slots=True)
class Outcome:
nodeid: str
category: str
reason: str
def _object(value: object) -> dict[str, object]:
if not isinstance(value, dict):
raise ValueError("expected an object")
return cast(dict[str, object], value)
def _array(value: object) -> tuple[dict[str, object], ...]:
if not isinstance(value, list):
raise ValueError("expected an array")
return tuple(_object(row) for row in cast(list[object], value))
def _string(value: object) -> str:
if not isinstance(value, str):
raise ValueError("expected a string")
return value
def _outcome(row: dict[str, object]) -> Outcome:
return Outcome(_string(row["nodeid"]), _string(row["category"]), _string(row["reason"]))
def _cell(value: str | float, limit: int = 240) -> str:
raw: Final = str(value)
clipped: Final = raw[:limit] + ("..." if len(raw) > limit else "")
return (
html.escape(" ".join(clipped.split()), quote=True)
.replace("|", "|")
.replace("`", "`")
.replace("[", "[")
.replace("]", "]")
.replace("*", "*")
.replace("_", "_")
.replace("\\", "\")
)
def _limited(rows: tuple[str, ...]) -> str:
candidates: Final = rows[:500]
sizes: Final = accumulate(len(row.encode("utf-8")) + 1 for row in candidates)
visible: Final = tuple(row for row, size in zip(candidates, sizes) if size <= 200_000)
remainder: Final = len(rows) - len(visible)
return "\n".join(visible + ((f"\nand {remainder} more",) if remainder else ()))
def _file_row(path: str, outcomes: tuple[Outcome, ...]) -> str:
counts: Final = Counter(outcome.category for outcome in outcomes)
return "| " + _cell(path) + " | " + " | ".join(str(counts[column]) for column in COLUMNS) + " |"
def _metric_row(row: dict[str, object]) -> str:
value: Final = row["value"]
if not isinstance(value, (str, int, float)):
raise ValueError("invalid metric value")
unit: Final = _string(row["unit"])
measured: Final = _cell(f"{value} {unit}".strip())
formatted: Final = f"<strong>{measured}</strong>" if isinstance(value, (int, float)) else measured
return f"| {_cell(_string(row['name']))} | {formatted} |"
def _metric_group(nodeid: str, rows: tuple[dict[str, object], ...]) -> str:
return "\n".join(
(
"| Metric | Value |",
"| --- | --- |",
_limited(tuple(_metric_row(row) for row in rows)),
"",
"<details>",
"<summary>Metric source</summary>",
"",
f"<code>{_cell(nodeid)}</code>",
"",
"</details>",
"",
)
)
def _expected_detail(outcome: Outcome) -> str:
path, _, test = outcome.nodeid.partition("::")
return (
f"- **{outcome.category}**: {_cell(test or path)}\n\n"
f" {_cell(outcome.reason, 1000)}\n\n"
f" <sub>{_cell(path)}</sub>\n"
)
def render(data: dict[str, object], title: str) -> str:
outcomes: Final = tuple(_outcome(row) for row in _array(data["outcomes"]))
metrics: Final = _array(data["metrics"])
counts: Final = Counter(outcome.category for outcome in outcomes)
extras: Final = tuple(sorted(set(counts) - set(TOTAL_ORDER)))
totals: Final = (
", ".join(
f"{counts[category]} {'errors' if category == 'error' and counts[category] != 1 else category}"
for category in TOTAL_ORDER + extras
if counts[category]
)
or "No test outcomes recorded"
)
files: Final = frozenset(outcome.nodeid.split("::")[0] for outcome in outcomes if outcome.category != "passed")
prefix: Final = (commonpath(tuple(str(Path(path).parent) for path in files)) or ".") if files else "."
grouped: Final = groupby(
sorted(outcomes, key=lambda outcome: outcome.nodeid), key=lambda outcome: outcome.nodeid.split("::")[0]
)
file_rows: Final = tuple(
_file_row(str(Path(path).relative_to(prefix)), tuple(group)) for path, group in grouped if path in files
)
table: Final = (
"\n".join(
(
"### Files with non-pass outcomes",
"",
f"Paths relative to <code>{_cell(prefix)}/</code>\n" if prefix != "." else "",
"| File | Passed | Failed | Errors | Skipped | Xfailed | Xpassed |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
_limited(file_rows),
)
)
if file_rows
else ""
)
metric_groups: Final = groupby(
sorted(metrics, key=lambda row: _string(row["nodeid"])), key=lambda row: _string(row["nodeid"])
)
metric_table: Final = (
"\n".join(
(
"### Metrics",
"",
_limited(tuple(_metric_group(nodeid, tuple(rows)) for nodeid, rows in metric_groups)),
)
)
if metrics
else ""
)
expected: Final = tuple(
sorted(
(outcome for outcome in outcomes if outcome.category in ("xfailed", "xpassed")),
key=lambda outcome: (outcome.category != "xpassed", outcome.nodeid),
)
)
details: Final = (
"\n".join(
(
"",
"<details>",
f"<summary>Xfail and xpass reasons ({len(expected)})</summary>",
"",
_limited(tuple(_expected_detail(outcome) for outcome in expected)),
"",
"</details>",
)
)
if expected
else ""
)
return (
"\n\n".join(part for part in (f"## {_cell(title)}", f"**{totals}**", metric_table, table, details) if part)
+ "\n"
)
def main() -> None:
parser: Final = argparse.ArgumentParser()
_ = parser.add_argument("--title", required=True)
_ = parser.add_argument("report", type=Path)
arguments: Final = parser.parse_args()
path: Final = cast(Path, arguments.report)
title: Final = cast(str, arguments.title)
try:
data: Final = _object(cast(object, json.loads(path.read_text(encoding="utf-8"))))
if data.get("version") != 1:
sys.stdout.write("Unsupported report version; no report was produced\n")
return
sys.stdout.write(render(data, title))
except (OSError, ValueError, KeyError, TypeError, RecursionError):
sys.stdout.write("No report was produced (missing or unreadable report)\n")
if __name__ == "__main__":
main()

190
.github/scripts/verify_linux_native_wheel.py vendored
View file

@ -6,7 +6,8 @@ import re
import subprocess
import sys
import zipfile
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from itertools import product
from pathlib import Path, PurePosixPath
from types import MappingProxyType, ModuleType
@ -17,6 +18,36 @@ EXPECTED_ABI_TAG: Final = "abi3"
EXPECTED_PLATFORM_TAG: Final = "linux_x86_64"
@dataclass(frozen=True, slots=True)
class WheelInspectionError:
message: str
@dataclass(frozen=True, slots=True)
class WheelFacts:
python_tag: str
abi_tag: str
platform_tag: str
expected_dist_info_directory: str
dist_info_directories: frozenset[str]
dist_info_file_counts: Mapping[str, int]
wheel_metadata_tags: tuple[str, ...]
expanded_filename_tags: frozenset[str]
compressed_size: int
uncompressed_size: int
native_size: int
debug_sections: tuple[str, ...]
static_symbol_table_absent: bool
extension_entry_point_present: bool
native_module_loads: bool
panic_test_hook_absent: bool
duplicated_vocabularies: tuple[str, ...]
unexpected_members: tuple[str, ...]
commit_sha: str
rustc_version: str
maturin_version: str
class CommandRunner(Protocol):
def __call__(
self,
@ -65,31 +96,22 @@ def _load_native_module(native_path: Path) -> ModuleType | None:
return native_module
def main(
argv: Sequence[str] | None = None,
def inspect_wheel(
wheel: Path,
environment: Mapping[str, str] | None = None,
load_native_module: Callable[[Path], ModuleType | None] = _load_native_module,
run_command: CommandRunner = _run_command,
) -> int:
arguments: Final = tuple(sys.argv if argv is None else argv)
) -> WheelFacts | WheelInspectionError:
resolved_environment: Final = os.environ if environment is None else environment
if len(arguments) != 2:
sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n")
return 2
wheel: Final = Path(arguments[1])
wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3)
if len(wheel_tags) != 4:
sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n")
return 1
return WheelInspectionError(f"cannot parse wheel tags from {wheel.name}")
wheel_identity: Final = wheel_tags[0].split("-")
if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]:
sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n")
return 1
return WheelInspectionError(f"unexpected wheel identity: {wheel_tags[0]}")
expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info"
expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,))
python_tag: Final = wheel_tags[1]
abi_tag: Final = wheel_tags[2]
platform_tag: Final = wheel_tags[3]
@ -121,8 +143,7 @@ def main(
if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
)
if len(native_members) != 1:
sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
return 1
return WheelInspectionError(f"expected one native extension, found {len(native_members)}")
unexpected_members: Final = tuple(
member.filename
@ -145,10 +166,6 @@ def main(
and archive.read(member) in native_bytes
)
wheel_metadata_tags_match: Final = (
len(wheel_metadata_tags) == len(expanded_filename_tags)
and frozenset(wheel_metadata_tags) == expanded_filename_tags
)
commit_sha: Final = resolved_environment.get(
"RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown")
)
@ -161,39 +178,9 @@ def main(
pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text()
maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject)
if maturin_match is None:
sys.stderr.write("build-system does not pin an exact Maturin version\n")
return 1
return WheelInspectionError("build-system does not pin an exact Maturin version")
maturin_version: Final = maturin_match.group(1)
native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100
size_report: Final = "\n".join(
(
"## Native wheel build report",
"",
"| Build | Value |",
"| --- | --- |",
f"| Commit | `{commit_sha}` |",
f"| Platform | `{platform_tag}` |",
f"| Python ABI | `{python_tag}-{abi_tag}` |",
f"| Rust compiler | `{rustc_version}` |",
f"| Maturin | `{maturin_version}` |",
"| Cargo profile | `release` |",
"",
"| Artifact | Size |",
"| --- | ---: |",
f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |",
f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |",
f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |",
f"| Native share | {native_percentage:.2f}% |",
"",
)
)
summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY")
if summary_path is None:
sys.stdout.write(size_report)
else:
Path(summary_path).write_text(size_report)
sections: Final = run_command(
("readelf", "--sections", "--wide", str(native_path)),
check=True,
@ -201,7 +188,6 @@ def main(
text=True,
).stdout
debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections)
debug_sections_absent: Final = not debug_sections
static_symbol_table_absent: Final = ".symtab" not in sections
dynamic_symbols: Final = run_command(
@ -214,80 +200,26 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 48_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
(f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG),
(f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG),
("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories),
(
"Required dist-info files are present exactly once",
all(count == 1 for count in dist_info_file_counts.values()),
),
("Wheel metadata tags match the filename", wheel_metadata_tags_match),
("Debug sections are absent", debug_sections_absent),
("Static symbol table is absent", static_symbol_table_absent),
("Python extension entry point is present", extension_entry_point_present),
("Native module loads", native_module_loads),
("Production module omits the panic test hook", panic_test_hook_absent),
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
("Wheel contents are valid", not unexpected_members),
return WheelFacts(
python_tag=python_tag,
abi_tag=abi_tag,
platform_tag=platform_tag,
expected_dist_info_directory=expected_dist_info_directory,
dist_info_directories=dist_info_directories,
dist_info_file_counts=dist_info_file_counts,
wheel_metadata_tags=wheel_metadata_tags,
expanded_filename_tags=expanded_filename_tags,
compressed_size=wheel.stat().st_size,
uncompressed_size=uncompressed_wheel_size,
native_size=native_member.file_size,
debug_sections=debug_sections,
static_symbol_table_absent=static_symbol_table_absent,
extension_entry_point_present=extension_entry_point_present,
native_module_loads=native_module_loads,
panic_test_hook_absent=panic_test_hook_absent,
duplicated_vocabularies=duplicated_vocabularies,
unexpected_members=unexpected_members,
commit_sha=commit_sha,
rustc_version=rustc_version,
maturin_version=maturin_version,
)
verified_report: Final = size_report + "\n".join(
("", "| Validation | Expected | Result |", "| --- | --- | :---: |")
+ tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations)
+ ("",)
)
if summary_path is not None:
Path(summary_path).write_text(verified_report)
invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values())
validation_errors: Final = tuple(
message
for failed, message in (
(bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"),
(not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"),
(not extension_entry_point_present, "native extension does not export PyInit__native"),
(
python_tag != EXPECTED_PYTHON_TAG,
f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}",
),
(abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"),
(
platform_tag != EXPECTED_PLATFORM_TAG,
f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
),
(
dist_info_directories != expected_dist_info_directories,
f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
f"found {', '.join(sorted(dist_info_directories))}",
),
(invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"),
(
not invalid_dist_info_files and not wheel_metadata_tags_match,
f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
f"found {', '.join(sorted(wheel_metadata_tags))}",
),
(
native_module is not None and not panic_test_hook_absent,
"production native module exposes _panic_for_test",
),
(
not native_size_within_limit,
f"native extension exceeds {native_size_limit / 1_000_000:.0f} MB: "
f"{native_member.file_size / 1_000_000:.2f} MB",
),
(bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
)
if failed
)
sys.stderr.write("".join(f"{message}\n" for message in validation_errors))
return 0 if all(passed for _, passed in validations) else 1
if __name__ == "__main__":
sys.exit(main())

3
.github/workflows/test-linting.yml vendored
View file

@ -554,6 +554,9 @@ jobs:
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
- name: test_pytest_ci_report
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_pytest_ci_report.py
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py

32
.github/workflows/test-rust.yml vendored
View file

@ -31,6 +31,10 @@ on:
- ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
- ".github/actions/pytest-report/**"
- ".github/scripts/render_pytest_report.py"
- "scripts/pytest_ci_report.py"
- "tests/wheel/**"
- "tests/unit/rust_bridge/native_route_wheel_test.py"
- ".github/workflows/test-rust.yml"
pull_request:
@ -66,6 +70,10 @@ on:
- ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
- ".github/actions/pytest-report/**"
- ".github/scripts/render_pytest_report.py"
- "scripts/pytest_ci_report.py"
- "tests/wheel/**"
- "tests/unit/rust_bridge/native_route_wheel_test.py"
- ".github/workflows/test-rust.yml"
@ -211,14 +219,34 @@ jobs:
- run: uv build --wheel --out-dir dist
- run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
- name: Check native wheel
run: |
set -- dist/*.whl
[ "$#" -eq 1 ]
uv run --no-project --with pytest==9.0.3 python -m pytest tests/wheel --wheel "$1" -p pytest_ci_report --ci-report=wheel-report.json
env:
RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Report native wheel outcomes
if: always()
uses: ./.github/actions/pytest-report
with:
report: wheel-report.json
title: Native wheel
artifact-name: wheel-report-${{ github.run_id }}-${{ github.run_attempt }}
- run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
- name: Run pytest tests/test_litellm_rust with the compiled extension
run: make test-rust-extension
run: make test-rust-extension PYTEST_REPORT_ARGS="-p pytest_ci_report --ci-report=rust-extension-report.json"
- name: Report Rust extension outcomes
if: always()
uses: ./.github/actions/pytest-report
with:
report: rust-extension-report.json
title: Rust extension
artifact-name: rust-extension-report-${{ github.run_id }}-${{ github.run_attempt }}
- run: >-
uv build --wheel --out-dir panic-dist

10
.github/workflows/test-unit.yml vendored
View file

@ -1085,7 +1085,7 @@ jobs:
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
fi
set +e
uv run --no-sync pytest "${pytest_args[@]}" \
uv run --no-sync pytest -p pytest_ci_report --ci-report=pytest-report.json "${pytest_args[@]}" \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
"${xdist_args[@]}" \
@ -1115,6 +1115,14 @@ jobs:
name: coverage-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
- name: Report test outcomes
if: always() && steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/pytest-report
with:
report: pytest-report.json
title: ${{ matrix.shard }}
artifact-name: pytest-report-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
ui-unit:
name: ui-unit
permissions:

4
Makefile
View file

@ -279,6 +279,8 @@ pre-commit:
@$(MAKE) check
# Testing targets
PYTEST_REPORT_ARGS ?=
test-rust-extension:
@temporary=$$(mktemp -d) && \
trap 'rm -rf "$$temporary"' EXIT HUP INT TERM && \
@ -291,7 +293,7 @@ test-rust-extension:
--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
litellm.rust_bridge._native && \
LITELLM_LOCAL_MODEL_COST_MAP=True \
"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension $(PYTEST_REPORT_ARGS) tests/test_litellm_rust
rust-sqlx-prepare:
cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare

134
scripts/pytest_ci_report.py Normal file
View file

@ -0,0 +1,134 @@
from __future__ import annotations
import json
import sys
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Final, Protocol, cast
import pytest
class CIMetric(Protocol):
def __call__(self, name: str, value: str | float, unit: str = "") -> None: ...
@dataclass(frozen=True, slots=True)
class Outcome:
nodeid: str
when: str
category: str
reason: str
@dataclass(frozen=True, slots=True)
class Metric:
nodeid: str
name: str
value: str | float
unit: str
@dataclass(frozen=True, slots=True)
class Report:
outcomes: tuple[Outcome, ...] = ()
metrics: tuple[Metric, ...] = ()
version: int = 1
REPORT: Final = pytest.StashKey[Report]()
def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption("--ci-report", default=None, help="Write non-gating CI outcomes and metrics as JSON")
def pytest_configure(config: pytest.Config) -> None:
path: Final[object] = config.getoption("ci_report")
if not isinstance(path, str) or hasattr(config, "workerinput"):
return
config.stash[REPORT] = Report()
config.pluginmanager.register(CIReporter(config, Path(path)), "ci-report-writer")
def _reason(report: pytest.TestReport | pytest.CollectReport) -> str:
xfail: Final[object] = getattr(report, "wasxfail", None)
if isinstance(xfail, str):
return xfail
if report.skipped and isinstance(report.longrepr, tuple):
return str(report.longrepr[2])
if isinstance(report, pytest.CollectReport) and report.failed:
return str(report.longrepr)
return ""
def _metric(nodeid: str, prop: object) -> Metric | None:
if not isinstance(prop, (tuple, list)) or len(prop) != 2 or prop[0] != "ci_metric":
return None
value: Final[object] = prop[1]
if not isinstance(value, (tuple, list)) or len(value) != 3:
return None
name, amount, unit = cast(tuple[object, object, object], value)
if isinstance(name, str) and isinstance(amount, (str, int, float)) and isinstance(unit, str):
return Metric(nodeid, name, amount, unit)
return None
@dataclass(frozen=True, slots=True)
class CIReporter:
config: pytest.Config
path: Path
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
status: Final = cast(
tuple[str, str, object],
self.config.hook.pytest_report_teststatus(
report=report,
config=self.config,
),
)
category: Final = status[0]
if category not in ("error", "skipped"):
return
previous: Final = self.config.stash[REPORT]
self.config.stash[REPORT] = Report(
previous.outcomes + (Outcome(report.nodeid, "collect", category, _reason(report)),),
previous.metrics,
)
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
status: Final = cast(
tuple[str, str, object],
self.config.hook.pytest_report_teststatus(
report=report,
config=self.config,
),
)
category: Final = status[0]
previous: Final = self.config.stash[REPORT]
outcomes: Final = previous.outcomes + (
(Outcome(report.nodeid, report.when, category, _reason(report)),) if category not in ("", "rerun") else ()
)
metrics: Final = (
tuple(metric for metric in previous.metrics if metric.nodeid != report.nodeid)
+ tuple(metric for prop in report.user_properties if (metric := _metric(report.nodeid, prop)) is not None)
if report.when == "teardown"
else previous.metrics
)
self.config.stash[REPORT] = Report(outcomes, metrics)
def pytest_sessionfinish(self) -> None:
try:
self.path.parent.mkdir(parents=True, exist_ok=True)
_ = self.path.write_text(json.dumps(asdict(self.config.stash[REPORT])), encoding="utf-8")
except (OSError, ValueError) as error:
sys.stderr.write(f"Could not write CI report: {error}\n")
@pytest.fixture
def ci_metric(request: pytest.FixtureRequest) -> CIMetric:
def record(name: str, value: str | float, unit: str = "") -> None:
if request.config.getoption("ci_report") is not None:
request.node.user_properties.append(("ci_metric", (name, value, unit)))
return record

329
tests/code_coverage_tests/test_pytest_ci_report.py Normal file
View file

@ -0,0 +1,329 @@
from __future__ import annotations
import html
import json
import os
import re
import subprocess
import sys
from pathlib import Path
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, TypeAdapter
ROOT: Final = Path(__file__).resolve().parents[2]
RENDERER: Final = ROOT / ".github/scripts/render_pytest_report.py"
SUITE: Final = """
from pathlib import Path
import pytest
def test_pass(ci_metric):
ci_metric("Size", 41.2, "MB")
def test_fail():
assert False, "permanent failure"
def test_skip():
pytest.skip("not available")
@pytest.mark.xfail(reason="known bug")
def test_xfail():
assert False
@pytest.mark.xfail(reason="fixed bug")
def test_xpass():
pass
def test_retry():
marker = Path(__file__).with_name("attempted")
first = not marker.exists()
marker.touch()
assert not first, "retry me"
@pytest.fixture
def teardown_error():
yield
raise RuntimeError("cleanup failed")
def test_teardown(teardown_error):
pass
"""
class ReportOutcome(BaseModel):
model_config = ConfigDict(frozen=True)
nodeid: str
when: str
category: str
reason: str
class ReportMetric(BaseModel):
model_config = ConfigDict(frozen=True)
nodeid: str
name: str
value: str | float
unit: str
class ReportData(BaseModel):
model_config = ConfigDict(frozen=True)
version: int
outcomes: tuple[ReportOutcome, ...]
metrics: tuple[ReportMetric, ...]
REPORT_DATA: Final = TypeAdapter(ReportData)
def run_suite(directory: Path, source: str = SUITE, *args: str) -> subprocess.CompletedProcess[str]:
directory.mkdir(exist_ok=True)
_ = (directory / "test_sample.py").write_text(source)
return subprocess.run(
(
sys.executable,
"-P",
"-m",
"pytest",
"-c",
"/dev/null",
f"--rootdir={directory}",
"--noconftest",
"-p",
"pytest_ci_report",
"-p",
"no:cacheprovider",
"--color=no",
"-rA",
*args,
"test_sample.py",
),
cwd=directory,
env={**os.environ, "PYTHONPATH": str(ROOT / "scripts"), "PYTEST_ADDOPTS": ""},
capture_output=True,
text=True,
check=False,
timeout=90,
)
def render(path: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
(sys.executable, "-P", str(RENDERER), "--title", "CI outcomes", str(path)),
capture_output=True,
text=True,
check=False,
timeout=10,
)
@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist"))
def test_outcomes_match_pytest_and_metrics_survive_workers(tmp_path: Path, workers: tuple[str, ...]) -> None:
report: Final = tmp_path / "report.json"
result: Final = run_suite(
tmp_path, SUITE, "--ci-report", str(report), "--reruns", "1", "--only-rerun", "retry me", *workers
)
assert result.returncode == 1, result.stdout + result.stderr
data: Final = REPORT_DATA.validate_json(report.read_text())
assert data.version == 1
assert sorted((row.nodeid.split("::")[-1], row.when, row.category, row.reason) for row in data.outcomes) == [
("test_fail", "call", "failed", ""),
("test_pass", "call", "passed", ""),
("test_retry", "call", "passed", ""),
("test_skip", "call", "skipped", "Skipped: not available"),
("test_teardown", "call", "passed", ""),
("test_teardown", "teardown", "error", ""),
("test_xfail", "call", "xfailed", "known bug"),
("test_xpass", "call", "xpassed", "fixed bug"),
]
assert data.metrics == (ReportMetric(nodeid="test_sample.py::test_pass", name="Size", value=41.2, unit="MB"),)
assert tuple(tmp_path.glob("*.json")) == (report,)
rendered: Final = render(report)
assert rendered.returncode == 0, rendered.stderr
terminal: Final = re.findall(
r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", result.stdout.splitlines()[-1]
)
assert ", ".join(terminal) in rendered.stdout
assert "| test_sample.py | 3 | 1 | 1 | 1 | 1 | 1 |" in html.unescape(rendered.stdout)
assert "Paths relative to" not in html.unescape(rendered.stdout)
assert rendered.stdout.index("fixed bug") < rendered.stdout.index("known bug")
@pytest.mark.parametrize("workers", ((), ("-n", "2")), ids=("serial", "xdist"))
def test_collection_errors_are_reported_and_match_terminal(tmp_path: Path, workers: tuple[str, ...]) -> None:
report: Final = tmp_path / "report.json"
result: Final = run_suite(
tmp_path,
"import missing_ci_test_module\n",
"--ci-report",
str(report),
*workers,
)
assert result.returncode != 0, result.stdout + result.stderr
data: Final = REPORT_DATA.validate_json(report.read_text())
assert len(data.outcomes) == 1
outcome: Final = data.outcomes[0]
assert outcome.nodeid == "test_sample.py"
assert outcome.when == "collect"
assert outcome.category == "error"
assert "missing_ci_test_module" in outcome.reason
terminal_summary: Final = next(
line for line in reversed(result.stdout.splitlines()) if re.search(r"\d+ errors? in ", line)
)
terminal_counts: Final = re.findall(r"\d+ (?:failed|passed|skipped|xfailed|xpassed|errors?)\b", terminal_summary)
assert terminal_counts == ["1 error"]
rendered: Final = render(report)
assert rendered.returncode == 0, rendered.stderr
assert ", ".join(terminal_counts) in rendered.stdout
def test_collection_skips_are_reported(tmp_path: Path) -> None:
report: Final = tmp_path / "report.json"
result: Final = run_suite(
tmp_path,
"import pytest\npytest.skip('not available', allow_module_level=True)\n",
"--ci-report",
str(report),
)
data: Final = REPORT_DATA.validate_json(report.read_text())
assert data.outcomes == (
ReportOutcome(nodeid="test_sample.py", when="collect", category="skipped", reason="Skipped: not available"),
)
assert "1 skipped" in result.stdout
rendered: Final = render(report)
assert rendered.returncode == 0, rendered.stderr
assert "1 skipped" in rendered.stdout
def test_plugin_loads_with_repository_config_without_pythonpath() -> None:
environment: Final = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"}
result: Final = subprocess.run(
(sys.executable, "-P", "-m", "pytest", "-p", "pytest_ci_report", "--help"),
cwd=ROOT,
env=environment,
capture_output=True,
text=True,
check=False,
timeout=90,
)
assert result.returncode == 0, result.stdout + result.stderr
assert "--ci-report" in result.stdout
def test_maxfail_still_writes_report(tmp_path: Path) -> None:
result: Final = run_suite(
tmp_path, "def test_failure():\n assert False\n", "--ci-report=report.json", "--maxfail=1"
)
assert result.returncode == 1
data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text())
assert data.outcomes == (
ReportOutcome(nodeid="test_sample.py::test_failure", when="call", category="failed", reason=""),
)
def test_report_is_opt_in(tmp_path: Path) -> None:
result: Final = run_suite(tmp_path, "def test_pass(ci_metric):\n ci_metric('Size', 1)\n")
assert result.returncode == 0, result.stdout + result.stderr
assert not tuple(tmp_path.glob("*.json"))
def test_unwritable_report_does_not_fail_tests(tmp_path: Path) -> None:
result: Final = run_suite(tmp_path, "def test_pass():\n pass\n", f"--ci-report={tmp_path}")
assert result.returncode == 0, result.stdout + result.stderr
@pytest.mark.parametrize("content", (None, "{", '{"version": 2}', '{"version": 1, "outcomes": null}'))
def test_unavailable_or_unknown_reports_are_non_gating(tmp_path: Path, content: str | None) -> None:
report: Final = tmp_path / "report.json"
if content is not None:
_ = report.write_text(content)
result: Final = render(report)
assert result.returncode == 0, result.stderr
assert "No report was produced" in result.stdout or "Unsupported report version" in result.stdout
@pytest.mark.parametrize("reason", ("known bug", "<tag>|\n" * 10000), ids=("entry-limit", "byte-limit"))
def test_details_and_large_cells_fit_summary_limit(tmp_path: Path, reason: str) -> None:
report: Final = tmp_path / "report.json"
_ = report.write_text(
json.dumps(
{
"version": 1,
"metrics": [],
"outcomes": [
{"nodeid": f"test_sample.py::test_{index}", "when": "call", "category": "xfailed", "reason": reason}
for index in range(600)
],
}
)
)
result: Final = render(report)
assert result.returncode == 0
visible: Final = result.stdout.count("- **xfailed**:")
assert visible <= 500
assert f"and {600 - visible} more" in result.stdout
assert len(result.stdout.encode()) < 1_000_000
assert "<tag>" not in result.stdout
def test_passing_metrics_are_compact_and_preserve_sources(tmp_path: Path) -> None:
report: Final = tmp_path / "report.json"
_ = report.write_text(
json.dumps(
{
"version": 1,
"outcomes": [{"nodeid": "tests/wheel/test_build.py::test_sizes", "category": "passed", "reason": ""}],
"metrics": [
{"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Wheel", "value": 20.5, "unit": "MB"},
{"nodeid": "tests/wheel/test_build.py::test_sizes", "name": "Native", "value": 30, "unit": "MB"},
],
}
)
)
rendered: Final = render(report)
assert rendered.returncode == 0, rendered.stderr
readable: Final = html.unescape(rendered.stdout)
assert "**1 passed**" in readable
assert "| Wheel | <strong>20.5 MB</strong> |" in readable
assert "| Native | <strong>30 MB</strong> |" in readable
assert readable.count("tests/wheel/test_build.py::test_sizes") == 1
assert "Files with non-pass outcomes" not in readable
def test_file_rows_shorten_shared_directory_without_losing_paths(tmp_path: Path) -> None:
report: Final = tmp_path / "report.json"
_ = report.write_text(
json.dumps(
{
"version": 1,
"metrics": [],
"outcomes": [
{"nodeid": "tests/unit/proxy/test_first.py::test_one", "category": "skipped", "reason": ""},
{"nodeid": "tests/unit/proxy/test_second.py::test_two", "category": "failed", "reason": ""},
],
}
)
)
rendered: Final = render(report)
readable: Final = html.unescape(rendered.stdout)
assert rendered.returncode == 0, rendered.stderr
assert "Paths relative to <code>tests/unit/proxy/</code>" in readable
assert "| test_first.py | 0 | 0 | 0 | 1 | 0 | 0 |" in readable
assert "| test_second.py | 0 | 1 | 0 | 0 | 0 | 0 |" in readable
def test_metrics_are_compatible_with_junit(tmp_path: Path) -> None:
result: Final = run_suite(
tmp_path,
"def test_pass(ci_metric):\n ci_metric('Size', 1, 'MB')\n",
"--ci-report=report.json",
"--junitxml=report.xml",
)
assert result.returncode == 0, result.stdout + result.stderr
assert 'name="ci_metric"' in (tmp_path / "report.xml").read_text()
data: Final = REPORT_DATA.validate_json((tmp_path / "report.json").read_text())
assert data.metrics[0].value == 1

166
tests/unit/rust_bridge/test_verify_linux_native_wheel.py
View file

@ -4,7 +4,7 @@ import importlib.util
import subprocess
import sys
import zipfile
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Mapping
from pathlib import Path
from types import MappingProxyType, ModuleType
from typing import Final, Protocol, cast
@ -23,16 +23,35 @@ class _CommandRunner(Protocol):
) -> subprocess.CompletedProcess[str]: ...
class _WheelFacts(Protocol):
python_tag: str
abi_tag: str
platform_tag: str
expected_dist_info_directory: str
dist_info_directories: frozenset[str]
dist_info_file_counts: Mapping[str, int]
wheel_metadata_tags: tuple[str, ...]
expanded_filename_tags: frozenset[str]
compressed_size: int
uncompressed_size: int
native_size: int
debug_sections: tuple[str, ...]
static_symbol_table_absent: bool
extension_entry_point_present: bool
native_module_loads: bool
panic_test_hook_absent: bool
duplicated_vocabularies: tuple[str, ...]
unexpected_members: tuple[str, ...]
class _VerifierModule(Protocol):
main: Callable[
[
Sequence[str] | None,
Mapping[str, str] | None,
Callable[[Path], ModuleType | None],
_CommandRunner,
],
int,
]
def inspect_wheel(
self,
wheel: Path,
environment: Mapping[str, str] | None = None,
load_native_module: Callable[[Path], ModuleType | None] = ...,
run_command: _CommandRunner = ...,
) -> _WheelFacts: ...
_REPO_ROOT: Final = Path(__file__).resolve().parents[3]
@ -80,21 +99,28 @@ def _write_wheel(
return wheel
def _fake_subprocess_run(
command: tuple[str, ...],
*,
check: bool,
capture_output: bool,
text: bool,
) -> subprocess.CompletedProcess[str]:
assert check and capture_output and text
if command == ("rustc", "--version"):
return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="")
if "--sections" in command:
return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="")
if "--dyn-syms" in command:
return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="")
raise AssertionError(f"unexpected subprocess command: {command}")
_STRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n"
_UNSTRIPPED_SECTIONS: Final = "[ 1] .text PROGBITS\n[ 2] .debug_info PROGBITS\n[ 3] .symtab SYMTAB\n"
def _fake_runner(sections: str = _STRIPPED_SECTIONS) -> _CommandRunner:
def run(
command: tuple[str, ...],
*,
check: bool,
capture_output: bool,
text: bool,
) -> subprocess.CompletedProcess[str]:
assert check and capture_output and text
if command == ("rustc", "--version"):
return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="")
if "--sections" in command:
return subprocess.CompletedProcess(command, 0, stdout=sections, stderr="")
if "--dyn-syms" in command:
return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="")
raise AssertionError(f"unexpected subprocess command: {command}")
return run
class _NativeModuleWithPanicHook(ModuleType):
@ -102,11 +128,12 @@ class _NativeModuleWithPanicHook(ModuleType):
return None
def _run_verifier(
def _inspect(
wheel: Path,
*,
exposes_panic: bool = False,
) -> int:
sections: str = _STRIPPED_SECTIONS,
) -> _WheelFacts:
native_module: Final = (
_NativeModuleWithPanicHook("litellm.rust_bridge._native")
if exposes_panic
@ -116,33 +143,61 @@ def _run_verifier(
def _fake_load_native_module(_: Path) -> ModuleType:
return native_module
environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")})
return verifier.main(
(str(_MODULE_PATH), str(wheel)),
environment: Final = MappingProxyType({"RELEASE_WHEEL_COMMIT_SHA": "0" * 40})
return verifier.inspect_wheel(
wheel,
environment,
_fake_load_native_module,
_fake_subprocess_run,
_fake_runner(sections),
)
def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None:
def test_reports_expected_release_wheel_tags(tmp_path: Path) -> None:
native_bytes: Final = b"synthetic native extension"
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, native_bytes=native_bytes)
facts: Final = _inspect(wheel)
assert (facts.python_tag, facts.abi_tag, facts.platform_tag) == ("cp310", "abi3", "linux_x86_64")
assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,))
assert facts.expected_dist_info_directory == _DIST_INFO
assert dict(facts.dist_info_file_counts) == {"METADATA": 1, "RECORD": 1, "WHEEL": 1}
assert frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags
assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags)
assert facts.native_size == len(native_bytes)
assert facts.compressed_size == wheel.stat().st_size
assert facts.uncompressed_size >= facts.native_size > 0
assert not facts.duplicated_vocabularies
assert not facts.unexpected_members
assert facts.extension_entry_point_present and facts.native_module_loads
assert facts.panic_test_hook_absent
def test_reports_debug_sections_and_static_symbol_table(tmp_path: Path) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
assert _run_verifier(wheel) == 0
facts: Final = _inspect(wheel, sections=_UNSTRIPPED_SECTIONS)
assert facts.debug_sections == (".debug_",)
assert not facts.static_symbol_table_absent
def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None:
def test_reports_cp312_version_specific_tags(tmp_path: Path) -> None:
tag: Final = "cp312-cp312-linux_x86_64"
wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,))
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert (facts.python_tag, facts.abi_tag) == ("cp312", "cp312")
def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None:
def test_reports_non_linux_platform_tag(tmp_path: Path) -> None:
tag: Final = "cp310-abi3-win_amd64"
wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,))
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert facts.platform_tag == "win_amd64"
@pytest.mark.parametrize(
@ -150,16 +205,18 @@ def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None:
(None, ("cp312-cp312-linux_x86_64",)),
ids=("missing", "mismatched"),
)
def test_rejects_missing_or_mismatched_wheel_metadata_tag(
def test_reports_missing_or_mismatched_wheel_metadata_tag(
tmp_path: Path,
metadata_tags: tuple[str, ...] | None,
) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags)
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert frozenset(facts.wheel_metadata_tags) != facts.expanded_filename_tags
def test_rejects_wheel_metadata_from_wrong_dist_info_directory(
def test_reports_wheel_metadata_from_wrong_dist_info_directory(
tmp_path: Path,
) -> None:
wheel: Final = _write_wheel(
@ -168,20 +225,27 @@ def test_rejects_wheel_metadata_from_wrong_dist_info_directory(
dist_info="decoy-1.0.0.dist-info",
)
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert facts.dist_info_directories == frozenset(("decoy-1.0.0.dist-info",))
assert facts.dist_info_directories != frozenset((facts.expected_dist_info_directory,))
assert dict(facts.dist_info_file_counts) == {"METADATA": 0, "RECORD": 0, "WHEEL": 0}
def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None:
def test_reports_duplicate_wheel_metadata_tags(tmp_path: Path) -> None:
wheel: Final = _write_wheel(
tmp_path,
filename_tag=_EXPECTED_TAG,
metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG),
)
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert facts.wheel_metadata_tags == (_EXPECTED_TAG, _EXPECTED_TAG)
assert len(facts.wheel_metadata_tags) != len(facts.expanded_filename_tags)
def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
def test_reports_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
with pytest.warns(UserWarning, match="Duplicate name"):
wheel: Final = _write_wheel(
tmp_path,
@ -189,13 +253,17 @@ def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None:
duplicate_wheel=True,
)
assert _run_verifier(wheel) == 1
facts: Final = _inspect(wheel)
assert facts.dist_info_file_counts["WHEEL"] == 2
def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None:
def test_reports_production_module_exposing_panic_hook(tmp_path: Path) -> None:
wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG)
assert _run_verifier(wheel, exposes_panic=True) == 1
facts: Final = _inspect(wheel, exposes_panic=True)
assert not facts.panic_test_hook_absent
@pytest.mark.parametrize("embedded", (False, True))
@ -209,4 +277,6 @@ def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None:
with zipfile.ZipFile(wheel, "a") as archive:
archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks)
assert _run_verifier(wheel) == (1 if embedded else 0)
facts: Final = _inspect(wheel)
assert bool(facts.duplicated_vocabularies) == embedded

5
tests/unit/test_unit_shard_missing_paths.py
View file

@ -1,3 +1,4 @@
import json
import os
import subprocess
import sys
@ -28,7 +29,9 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl
shim_dir.mkdir()
(shim_dir / "uv").write_text(_UV_SHIM)
(shim_dir / "uv").chmod(0o755)
(tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n")
(tmp_path / "pyproject.toml").write_text(
f"[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\npythonpath = [{json.dumps(str(_REPO_ROOT / 'scripts'))}]\n"
)
return subprocess.run(
("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()),
cwd=tmp_path,

7
tests/wheel/AGENTS.md Normal file
View file

@ -0,0 +1,7 @@
# Release wheel checks
- Run against a release-built wheel passed with `--wheel`
- Only run in the `rust-wheel` job, never select from a unit shard
- May invoke `readelf` and load the compiled native extension
- Keep facts in `.github/scripts/verify_linux_native_wheel.py` and assertions here
- Record build metrics with `ci_metric`, never write a separate summary

26
tests/wheel/conftest.py Normal file
View file

@ -0,0 +1,26 @@
from __future__ import annotations
import sys
from pathlib import Path
from typing import Final
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".github" / "scripts")) # test-quality-ok: the wheel inspector lives in .github/scripts, outside any importable package
from verify_linux_native_wheel import WheelFacts, WheelInspectionError, inspect_wheel # noqa: E402 # CI helpers live outside Python packages
def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption("--wheel", help="Release-built native wheel to inspect")
@pytest.fixture(scope="session")
def facts(request: pytest.FixtureRequest) -> WheelFacts:
wheel: Final[object] = request.config.getoption("wheel")
if not isinstance(wheel, str):
pytest.fail("--wheel is required for release wheel checks")
inspected: Final = inspect_wheel(Path(wheel))
if isinstance(inspected, WheelInspectionError):
pytest.fail(inspected.message)
return inspected

89
tests/wheel/test_linux_native_wheel.py Normal file
View file

@ -0,0 +1,89 @@
from __future__ import annotations
from pytest_ci_report import CIMetric
from verify_linux_native_wheel import (
EXPECTED_ABI_TAG,
EXPECTED_PLATFORM_TAG,
EXPECTED_PYTHON_TAG,
WheelFacts,
)
def test_python_tag_is_cp310(facts: WheelFacts) -> None:
assert facts.python_tag == EXPECTED_PYTHON_TAG, f"unexpected Python tag: {facts.python_tag}"
def test_abi_tag_is_abi3(facts: WheelFacts) -> None:
assert facts.abi_tag == EXPECTED_ABI_TAG, f"unexpected ABI tag: {facts.abi_tag}"
def test_platform_tag_is_linux_x86_64(facts: WheelFacts) -> None:
assert facts.platform_tag == EXPECTED_PLATFORM_TAG, f"unexpected platform tag: {facts.platform_tag}"
def test_dist_info_directory_matches_filename(facts: WheelFacts) -> None:
assert facts.dist_info_directories == frozenset((facts.expected_dist_info_directory,)), (
f"unexpected dist-info directories: {facts.dist_info_directories}"
)
def test_required_dist_info_files_are_present_once(facts: WheelFacts) -> None:
assert all(count == 1 for count in facts.dist_info_file_counts.values()), (
f"required dist-info file counts are invalid: {facts.dist_info_file_counts}"
)
def test_metadata_tags_match_filename(facts: WheelFacts) -> None:
assert len(facts.wheel_metadata_tags) == len(facts.expanded_filename_tags) and (
frozenset(facts.wheel_metadata_tags) == facts.expanded_filename_tags
), f"WHEEL tags do not match filename: {facts.wheel_metadata_tags}"
def test_debug_sections_are_absent(facts: WheelFacts) -> None:
assert not facts.debug_sections, f"native extension contains debug sections: {facts.debug_sections}"
def test_static_symbol_table_is_absent(facts: WheelFacts) -> None:
assert facts.static_symbol_table_absent, "native extension contains a static symbol table"
def test_python_extension_entry_point_is_present(facts: WheelFacts) -> None:
assert facts.extension_entry_point_present, "native extension does not export PyInit__native"
def test_native_module_loads(facts: WheelFacts) -> None:
assert facts.native_module_loads, "native extension cannot be loaded"
def test_production_module_omits_panic_hook(facts: WheelFacts) -> None:
assert facts.panic_test_hook_absent, "production native extension exposes _panic_for_test"
def test_native_extension_does_not_exceed_48_mb(facts: WheelFacts) -> None:
assert facts.native_size <= 48_000_000, (
f"native extension exceeds 48 MB: {facts.native_size / 1_000_000:.2f} MB"
)
def test_tokenizer_vocabularies_are_not_duplicated(facts: WheelFacts) -> None:
assert not facts.duplicated_vocabularies, (
f"tokenizer vocabularies are duplicated in the native extension: {facts.duplicated_vocabularies}"
)
def test_wheel_contents_are_valid(facts: WheelFacts) -> None:
assert not facts.unexpected_members, f"wheel contains unexpected build artifacts: {facts.unexpected_members}"
def test_build_report(facts: WheelFacts, ci_metric: CIMetric) -> None:
assert 0 < facts.native_size <= facts.uncompressed_size
ci_metric("Compressed wheel", round(facts.compressed_size / 1_000_000, 2), "MB")
ci_metric("Uncompressed wheel", round(facts.uncompressed_size / 1_000_000, 2), "MB")
ci_metric("Native extension", round(facts.native_size / 1_000_000, 2), "MB")
ci_metric("Native share", round(facts.native_size / facts.uncompressed_size * 100, 2), "%")
ci_metric("Commit", facts.commit_sha)
ci_metric("Platform", facts.platform_tag)
ci_metric("Python ABI", f"{facts.python_tag}-{facts.abi_tag}")
ci_metric("Rust compiler", facts.rustc_version)
ci_metric("Maturin", facts.maturin_version)
ci_metric("Cargo profile", "release")