mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
ci: cut CircleCI wall time without loosening test isolation (#43347)
* ci: cut CircleCI wall time without loosening test isolation * fix(ci): parse integration split files that follow --results The CircleCI machine image ships Python 3.12.2, whose argparse leaves the files positional empty when it follows an option and another positional, so every extensions node exited with 'unrecognized arguments'. Reproduced on 3.12.2; parse_intermixed_args selects the files on 3.12.2, 3.12.13 and 3.13 * test(ci): resolve command references in the Rust toolchain guard The Windows rustup install moved into the install_windows_toolchain command, which the guard only recognized for install_rust. It now accepts any command that installs a pinned rustup and reads the Windows toolchain pin from it * ci: cache the Windows release cargo build from main windows_release_wheel rebuilt every dependency with fat LTO on each run. It now restores the release target and cargo registry saved by main's scheduled run, drops the workspace crates' fingerprints so they always rebuild from the checked-out source, and still runs the full LTO link * ci: run the Windows release wheel build on windows.xlarge The fat-LTO release build is the slowest job in the pipeline; more cores speed up the dependency compile ahead of the final link * ci: skip the Windows fingerprint cleanup when the cargo cache missed On a cold cache the release fingerprint directory does not exist, and the CircleCI PowerShell wrapper failed the step on the suppressed not-found error
This commit is contained in:
yuneng-jiang
GitHub
parent
3afcd176b3
commit
635a718ba1
12 changed files with 263 additions and 69 deletions
|
|
@ -141,7 +141,7 @@ commands:
|
|||
node --version
|
||||
npm --version
|
||||
install_rust:
|
||||
description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
|
||||
description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
|
||||
steps:
|
||||
- run:
|
||||
name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
|
||||
|
|
@ -167,9 +167,29 @@ commands:
|
|||
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
|
||||
rm -f /tmp/rustup-init
|
||||
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
|
||||
echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
rustc --version
|
||||
cargo --version
|
||||
{ rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
||||
- v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
|
||||
- run:
|
||||
name: Force a rebuild of the workspace crates restored from the cargo cache
|
||||
command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
|
||||
save_cargo_target:
|
||||
steps:
|
||||
- when:
|
||||
condition:
|
||||
equal: [main, << pipeline.git.branch >>]
|
||||
steps:
|
||||
- save_cache:
|
||||
key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
||||
paths:
|
||||
- ~/.cargo/registry
|
||||
- ~/project/litellm-rust/target/debug
|
||||
start_postgres:
|
||||
description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
|
||||
parameters:
|
||||
|
|
@ -281,51 +301,11 @@ commands:
|
|||
# `uv sync --package litellm-enterprise` here — that overwrites the
|
||||
# shared .venv and strips out dev/test deps (pytest, prisma, etc.).
|
||||
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
|
||||
setup_litellm_test_deps:
|
||||
install_windows_toolchain:
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
uv sync --frozen --all-groups --all-extras --python 3.12
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/.cache/uv
|
||||
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
|
||||
jobs:
|
||||
# Add Windows testing job
|
||||
using_litellm_on_windows:
|
||||
executor:
|
||||
name: win/default
|
||||
shell: powershell.exe
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
UV_PYTHON: "3.11"
|
||||
CARGO_HTTP_MULTIPLEXING: "false"
|
||||
CARGO_NET_RETRY: "5"
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install Python
|
||||
command: |
|
||||
choco install python --version=3.11.0 -y --no-progress --force
|
||||
refreshenv
|
||||
python --version
|
||||
environment:
|
||||
CHOCOLATEY_CONFIRM_ALL: "true"
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
name: Install Rust and uv
|
||||
no_output_timeout: 30m
|
||||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
$rustupInit = Join-Path $env:TEMP "rustup-init.exe"
|
||||
$rustupVersion = "1.28.2"
|
||||
|
|
@ -365,6 +345,55 @@ jobs:
|
|||
if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
|
||||
Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
|
||||
}
|
||||
setup_litellm_test_deps:
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
uv sync --frozen --all-groups --all-extras --python 3.12
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/.cache/uv
|
||||
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
- save_cargo_target
|
||||
|
||||
jobs:
|
||||
# Add Windows testing job
|
||||
using_litellm_on_windows:
|
||||
executor:
|
||||
name: win/default
|
||||
shell: powershell.exe
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
UV_PYTHON: "3.11"
|
||||
CARGO_HTTP_MULTIPLEXING: "false"
|
||||
CARGO_NET_RETRY: "5"
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install Python
|
||||
command: |
|
||||
choco install python --version=3.11.0 -y --no-progress --force
|
||||
refreshenv
|
||||
python --version
|
||||
environment:
|
||||
CHOCOLATEY_CONFIRM_ALL: "true"
|
||||
- install_windows_toolchain
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
no_output_timeout: 30m
|
||||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
$env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
|
||||
for ($attempt = 1; $attempt -le 5; $attempt++) {
|
||||
Write-Host "uv sync attempt $attempt/5"
|
||||
uv sync --frozen --group dev --python 3.11
|
||||
|
|
@ -380,17 +409,68 @@ jobs:
|
|||
name: Run Windows-specific test
|
||||
command: |
|
||||
uv run --no-sync python -m pytest tests/windows_tests/ -v
|
||||
|
||||
windows_release_wheel:
|
||||
executor:
|
||||
name: win/default
|
||||
shell: powershell.exe
|
||||
size: xlarge
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
UV_PYTHON: "3.11"
|
||||
CARGO_HTTP_MULTIPLEXING: "false"
|
||||
CARGO_NET_RETRY: "5"
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Guard against MAX_PATH-busting packaged wheel paths
|
||||
name: Skip job when no windows-release-relevant files changed
|
||||
shell: bash.exe
|
||||
command: bash .circleci/scripts/path_filter.sh windows-release
|
||||
- run:
|
||||
name: Install Python
|
||||
command: |
|
||||
choco install python --version=3.11.0 -y --no-progress --force
|
||||
refreshenv
|
||||
python --version
|
||||
environment:
|
||||
CHOCOLATEY_CONFIRM_ALL: "true"
|
||||
- install_windows_toolchain
|
||||
- run:
|
||||
name: Record the Rust build environment for the release cargo cache key
|
||||
command: |
|
||||
& "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
||||
- v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
|
||||
- run:
|
||||
name: Force a rebuild of the workspace crates restored from the cargo cache
|
||||
command: |
|
||||
$fingerprints = "litellm-rust/target/release/.fingerprint"
|
||||
if (Test-Path $fingerprints) {
|
||||
Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
|
||||
}
|
||||
- run:
|
||||
name: Build the release wheel and install it under a worst-case MAX_PATH prefix
|
||||
no_output_timeout: 30m
|
||||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
$env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
|
||||
cargo --version
|
||||
Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force
|
||||
uv build --wheel --out-dir dist
|
||||
uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
python tests/windows_tests/check_windows_wheel_install.py
|
||||
- when:
|
||||
condition:
|
||||
equal: [main, << pipeline.git.branch >>]
|
||||
steps:
|
||||
- save_cache:
|
||||
key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
||||
paths:
|
||||
- ~/.cargo/registry
|
||||
- ~/project/litellm-rust/target/release
|
||||
|
||||
base_sdk_install:
|
||||
docker:
|
||||
|
|
@ -418,6 +498,10 @@ jobs:
|
|||
uv venv /tmp/base-sdk --python 3.12
|
||||
VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
|
||||
/tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
|
||||
- run:
|
||||
name: Guard against MAX_PATH-busting packaged wheel paths
|
||||
command: |
|
||||
python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
|
||||
|
||||
local_testing_part1:
|
||||
docker:
|
||||
|
|
@ -446,6 +530,7 @@ jobs:
|
|||
paths:
|
||||
- ~/.cache/uv
|
||||
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
||||
- save_cargo_target
|
||||
- run:
|
||||
name: Run prisma ./docker/entrypoint.sh
|
||||
command: |
|
||||
|
|
@ -3120,10 +3205,14 @@ jobs:
|
|||
type: enum
|
||||
enum: [standard, replica]
|
||||
default: standard
|
||||
parallelism:
|
||||
type: integer
|
||||
default: 1
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
parallelism: << parameters.parallelism >>
|
||||
steps:
|
||||
- setup_litellm_test_deps
|
||||
- when:
|
||||
|
|
@ -3249,6 +3338,7 @@ jobs:
|
|||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
steps:
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
|
|
@ -3258,10 +3348,11 @@ jobs:
|
|||
name: Run unit tests
|
||||
command: |
|
||||
mkdir -p test-results/unit
|
||||
mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
|
||||
if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
|
||||
shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
|
||||
if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
|
||||
mapfile -t files < <(printf '%s\n' "${shard}")
|
||||
set +e
|
||||
LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
|
||||
LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
|
||||
|
|
@ -3328,7 +3419,11 @@ workflows:
|
|||
name: integration-<< matrix.suite >>
|
||||
matrix:
|
||||
parameters:
|
||||
suite: [management, accounting, database, providers, extensions, mcp, sdk, cost, browser]
|
||||
suite: [management, accounting, database, providers, mcp, sdk, cost, browser]
|
||||
- integration_contracts:
|
||||
name: integration-extensions
|
||||
suite: extensions
|
||||
parallelism: 4
|
||||
- integration_contracts:
|
||||
name: integration-<< matrix.suite >>-replica
|
||||
matrix:
|
||||
|
|
@ -3343,6 +3438,7 @@ workflows:
|
|||
equal: ["", << pipeline.parameters.routing_parity_base >>]
|
||||
jobs:
|
||||
- using_litellm_on_windows
|
||||
- windows_release_wheel
|
||||
- unit
|
||||
- provider_replay_harness
|
||||
- base_sdk_install
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies>}"
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
|
||||
|
||||
has_client=false
|
||||
has_backend=false
|
||||
|
|
@ -9,6 +9,7 @@ has_ci=false
|
|||
has_provider_harness=false
|
||||
has_cost_map=false
|
||||
has_mcp_dependencies=false
|
||||
has_windows_release=false
|
||||
outside_cost_map_set=false
|
||||
while IFS= read -r file || [ -n "$file" ]; do
|
||||
[ -n "$file" ] || continue
|
||||
|
|
@ -22,6 +23,10 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
|
||||
has_provider_harness=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
|
||||
has_windows_release=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
ui/* | tests/e2e/ui/*) has_client=true ;;
|
||||
docs/* | *.md | *.mdx) : ;;
|
||||
|
|
@ -46,6 +51,9 @@ case "$category" in
|
|||
provider-harness)
|
||||
[ "$has_provider_harness" = true ] && echo run || echo skip
|
||||
;;
|
||||
windows-release)
|
||||
[ "$has_windows_release" = true ] && echo run || echo skip
|
||||
;;
|
||||
backend)
|
||||
[ "$has_backend" = true ] && echo run || echo skip
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -212,6 +212,15 @@ if [ "$suite" = browser ]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
node_files=()
|
||||
if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
|
||||
split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
|
||||
| circleci tests split --split-by=timings --timings-type=filename)"
|
||||
read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
|
||||
test "${#node_files[@]}" -gt 0
|
||||
printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
|
||||
fi
|
||||
|
||||
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
||||
INTEGRATION_RUN_ID="$integration_identity" \
|
||||
DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
|
|
@ -225,7 +234,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|||
INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
|
||||
INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
|
||||
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
|
||||
.venv/bin/python tests/integration/run.py "$suite" --results "$results"
|
||||
.venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
|
||||
|
||||
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
||||
for covered_pid in "$proxy_pid" "$peer_pid"; do
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Use `tests/integration/run.py management`, `accounting`, `database`, `providers`
|
|||
|
||||
Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload
|
||||
|
||||
The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. Use `--seed` and `--order-seed` to reproduce a run. Actual installed Hypothesis version, settings, seeds and collected order are written beside the execution manifest
|
||||
The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles the file order and the test order inside each file but keeps each file's tests together, so module fixtures are built once per file. Use `--seed` and `--order-seed` to reproduce a run. Actual installed Hypothesis version, settings, seeds and collected order are written beside the execution manifest
|
||||
|
||||
Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ Streaming checks send real HTTP transfer chunks, including one-byte partitions,
|
|||
|
||||
The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy, database or Redis; a case that reaches the gateway belongs in one of the other shards
|
||||
|
||||
The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions
|
||||
The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on parallel nodes, and each node starts its own database, Redis, upstream and proxy and runs its share of the group's files serially, split by recorded timings with `circleci tests split`. Tests keep the isolation of a serial run; they still must not assume a particular set of sibling files. `run.py <group> --list` prints a group's files and `run.py <group> <file>...` runs a subset of them
|
||||
|
||||
The mcp shard runs the MCP gateway against SDK peers owned by each test (`_support/mcp.py`): streamable HTTP, SSE and stdio peers, an OpenAPI-spec app, and an OAuth 2.1 authorization-server double. Every peer records the requests it receives so a test can assert what reached the peer, not only what the proxy answered. The shard runs with `INTEGRATION_WORKERS` set and with `INTEGRATION_COVERAGE=1`, which starts the proxy under `coverage run --parallel-mode` limited to the MCP modules and stores `coverage.txt` plus an HTML report with the job artifacts. A test that fails because the product is wrong is skipped with `pytest.skip("BUG: <symptom>")` so the skip list in `execution.json` is the open MCP bug list
|
||||
|
||||
|
|
|
|||
|
|
@ -51,10 +51,18 @@ def _owned(nodeid: str) -> bool:
|
|||
return parts[:2] == ("tests", "integration") and len(parts) > 3 and parts[2] in OWNED_DIRECTORIES
|
||||
|
||||
|
||||
def _digest(seed: int, identity: str) -> bytes:
|
||||
return hashlib.sha256(f"{seed}:{identity}".encode()).digest()
|
||||
|
||||
|
||||
def _order_key(seed: int, nodeid: str) -> tuple[bytes, bytes]:
|
||||
return _digest(seed, nodeid.split("::", 1)[0]), _digest(seed, nodeid)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
order_seed: Final = config.getoption("integration_order_seed")
|
||||
if order_seed:
|
||||
items.sort(key=lambda item: hashlib.sha256(f"{order_seed}:{item.nodeid}".encode()).digest())
|
||||
items.sort(key=lambda item: _order_key(order_seed, item.nodeid))
|
||||
root: Final = Path(__file__).parent
|
||||
owned: Final = tuple(
|
||||
item
|
||||
|
|
|
|||
|
|
@ -30,13 +30,22 @@ def main() -> int:
|
|||
parser.add_argument("--seed", type=int, default=int(os.environ.get("INTEGRATION_SEED", "4106601")))
|
||||
parser.add_argument("--order-seed", type=int, default=int(os.environ.get("INTEGRATION_ORDER_SEED", "0")))
|
||||
parser.add_argument("--workers", type=int, default=int(os.environ.get("INTEGRATION_WORKERS", "1")))
|
||||
options: Final = parser.parse_args()
|
||||
parser.add_argument("--list", action="store_true", help="print the group's test files and exit")
|
||||
parser.add_argument("files", nargs="*", help="run only these files of the group")
|
||||
options: Final = parser.parse_intermixed_args()
|
||||
root: Final = Path(__file__).resolve().parents[2]
|
||||
selected: Final = tuple(
|
||||
group_files: Final = tuple(
|
||||
str(path.relative_to(root))
|
||||
for folder in GROUPS[options.group]
|
||||
for path in sorted((root / "tests/integration" / folder).glob("test_*.py"))
|
||||
)
|
||||
if options.list:
|
||||
print("\n".join(group_files))
|
||||
return 0
|
||||
foreign: Final = sorted(set(options.files) - set(group_files))
|
||||
if foreign:
|
||||
parser.error(f"Not in the {options.group} group: {', '.join(foreign)}")
|
||||
selected: Final = tuple(options.files) or group_files
|
||||
if not selected:
|
||||
parser.error(f"No integration test files selected for {options.group}")
|
||||
output: Final = options.results.resolve()
|
||||
|
|
@ -65,6 +74,8 @@ def main() -> int:
|
|||
f"--hypothesis-seed={options.seed}",
|
||||
f"--integration-order-seed={options.order_seed}",
|
||||
f"--junitxml={output / 'junit.xml'}",
|
||||
"-o",
|
||||
"junit_family=xunit1",
|
||||
*(("-n", str(options.workers)) if options.workers > 1 else ()),
|
||||
],
|
||||
cwd=root,
|
||||
|
|
|
|||
|
|
@ -647,6 +647,7 @@ async def test_negation_with_positive_tag():
|
|||
@pytest.mark.asyncio()
|
||||
async def test_negation_all_excluded_raises():
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
|
|
@ -907,6 +908,7 @@ async def test_positive_tags_unchanged_by_negation():
|
|||
@pytest.mark.asyncio()
|
||||
async def test_negation_skips_banned_group_and_uses_fallback():
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -943,6 +945,7 @@ async def test_negation_skips_banned_group_and_uses_fallback():
|
|||
@pytest.mark.asyncio()
|
||||
async def test_negation_exhausts_entire_fallback_chain():
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -1696,6 +1699,7 @@ async def test_required_and_single_tag_matches_trivially():
|
|||
async def test_required_and_unmatched_raises_by_default():
|
||||
# allow_fail_open unset -> unmatched required-AND raises, same as today's "!" behavior.
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
|
|
@ -1728,6 +1732,7 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default()
|
|||
# &A eliminates every candidate before the positive-tag preference even runs;
|
||||
# this must be gated by allow_fail_open too, not just the required-AND-only path.
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
|
|
@ -1858,6 +1863,7 @@ async def test_allow_fail_open_per_hop_across_fallback_chain():
|
|||
# required-AND fail-open must be re-evaluated fresh on every hop, the same
|
||||
# per-hop guarantee the negation feature already established.
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -1950,6 +1956,7 @@ async def test_allow_fail_open_resolves_locally_without_triggering_external_fall
|
|||
@pytest.mark.asyncio()
|
||||
async def test_negation_combined_with_positive_unmatched_raises_by_default():
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
|
|
@ -2287,6 +2294,7 @@ async def test_required_and_exhausts_primary_group_falls_through_to_fallback_gro
|
|||
# where the tag is satisfiable. No allow_fail_open involved; this is the plain
|
||||
# fallback-chain mechanics already established for "!" extended to "&".
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -2332,6 +2340,7 @@ async def test_required_and_negation_and_allow_fail_open_combine_across_three_mo
|
|||
# carrier is legitimately excluded, not hidden behind an invented tag, so the
|
||||
# opted-in allow_fail_open falls back to the group's own default deployment.
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -2393,6 +2402,7 @@ async def test_unknown_tag_denial_is_scoped_per_hop_not_leaked_across_fallback_g
|
|||
# discover what its own group knows; a deny decision from a prior hop's group
|
||||
# must not leak forward and block a later hop that has no relevant knowledge.
|
||||
router = litellm.Router(
|
||||
num_retries=0,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
|
|
@ -2868,6 +2878,7 @@ def _tagged_marker_router(tier_tags=None):
|
|||
},
|
||||
],
|
||||
enable_tag_filtering=True,
|
||||
num_retries=0,
|
||||
)
|
||||
router.auto_routers = {
|
||||
"gpt4o": [TaggedPreRoutingStrategy(tags=("route",), strategy=_RewriteToTierStrategy("gemini-flash"))]
|
||||
|
|
|
|||
|
|
@ -73,6 +73,17 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"]
|
|||
("provider-harness", ["tests/e2e/quota_management/test_quota.py"], "skip"),
|
||||
("provider-harness", ["litellm/main.py"], "skip"),
|
||||
("provider-harness", ["ui/litellm-dashboard/src/App.tsx"], "skip"),
|
||||
("windows-release", ["litellm-rust/crates/core/src/lib.rs"], "run"),
|
||||
("windows-release", ["litellm/rust_bridge/dispatch.py"], "run"),
|
||||
("windows-release", ["rust-toolchain.toml"], "run"),
|
||||
("windows-release", ["pyproject.toml"], "run"),
|
||||
("windows-release", ["uv.lock"], "run"),
|
||||
("windows-release", ["tests/windows_tests/check_windows_wheel_install.py"], "run"),
|
||||
("windows-release", [".circleci/config.yml"], "run"),
|
||||
("windows-release", ["litellm/main.py"], "skip"),
|
||||
("windows-release", ["tests/unit/test_utils.py"], "skip"),
|
||||
("windows-release", ["ui/litellm-dashboard/src/App.tsx"], "skip"),
|
||||
("windows-release", ["docs/my-website/docs/index.md"], "skip"),
|
||||
# docs-only: skip everything
|
||||
("backend", DOCS, "skip"),
|
||||
("client", DOCS, "skip"),
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ Two invariants are pinned here:
|
|||
|
||||
1. No step list (job or reusable command) reaches a `uv sync` / `uv build`
|
||||
without a Rust toolchain already provisioned ahead of it. That is the
|
||||
`install_rust` command on Linux and an inline pinned rustup install in the
|
||||
Windows job, so the check accepts either. A new job that syncs without one
|
||||
`install_rust` command on Linux and `install_windows_toolchain` on Windows,
|
||||
so the check accepts any command or step that installs a pinned rustup. A new job that syncs without one
|
||||
falls back to the unpinned path, which is exactly the regression a static
|
||||
check catches at PR time and a green CI run does not.
|
||||
2. Both installers pin what they download: an explicit rustup version, a
|
||||
|
|
@ -66,13 +66,23 @@ def _without_comments(text: str) -> str:
|
|||
return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
|
||||
|
||||
|
||||
def _provisions_rust(step: object) -> bool:
|
||||
if step == "install_rust":
|
||||
return True
|
||||
def _installs_pinned_rustup(step: object) -> bool:
|
||||
text = _step_text(step)
|
||||
return "rustup-init" in text and ("sha256sum" in text or "SHA256" in text)
|
||||
|
||||
|
||||
def _provisioning_commands() -> frozenset[str]:
|
||||
return frozenset(
|
||||
name.removeprefix("command ")
|
||||
for name, steps in _step_lists().items()
|
||||
if name.startswith("command ") and any(_installs_pinned_rustup(step) for step in steps)
|
||||
)
|
||||
|
||||
|
||||
def _provisions_rust(step: object, provisioning_commands: frozenset[str]) -> bool:
|
||||
return (isinstance(step, str) and step in provisioning_commands) or _installs_pinned_rustup(step)
|
||||
|
||||
|
||||
def _step_lists() -> dict[str, list[object]]:
|
||||
config = _config()
|
||||
lists: dict[str, list[object]] = {}
|
||||
|
|
@ -87,11 +97,11 @@ def _step_lists() -> dict[str, list[object]]:
|
|||
return lists
|
||||
|
||||
|
||||
def _first_unprovisioned_build(steps: list[object]) -> str | None:
|
||||
def _first_unprovisioned_build(steps: list[object], provisioning_commands: frozenset[str]) -> str | None:
|
||||
"""Return the shell text of the first workspace build reached without Rust, if any."""
|
||||
rust_ready = False
|
||||
for step in steps:
|
||||
if _provisions_rust(step):
|
||||
if _provisions_rust(step, provisioning_commands):
|
||||
rust_ready = True
|
||||
text = _step_text(step)
|
||||
if BUILDS_WORKSPACE.search(_without_comments(text)) and not rust_ready:
|
||||
|
|
@ -111,8 +121,12 @@ def test_step_lists_exist() -> None:
|
|||
|
||||
|
||||
def test_no_workspace_build_without_a_provisioned_rust_toolchain() -> None:
|
||||
provisioning_commands: Final = _provisioning_commands()
|
||||
assert {"install_rust", "install_windows_toolchain"} <= provisioning_commands
|
||||
offenders = {
|
||||
name: build for name, steps in _step_lists().items() if (build := _first_unprovisioned_build(steps)) is not None
|
||||
name: build
|
||||
for name, steps in _step_lists().items()
|
||||
if (build := _first_unprovisioned_build(steps, provisioning_commands)) is not None
|
||||
}
|
||||
assert not offenders, (
|
||||
"these CircleCI step lists run `uv sync`/`uv build` with no Rust toolchain provisioned first, "
|
||||
|
|
@ -156,7 +170,7 @@ def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str)
|
|||
|
||||
|
||||
def test_windows_installer_matches_the_repo_toolchain() -> None:
|
||||
windows_steps: Final = _step_lists()["job using_litellm_on_windows"]
|
||||
windows_steps: Final = _step_lists()["command install_windows_toolchain"]
|
||||
windows_command: Final = "\n".join(_step_text(step) for step in windows_steps)
|
||||
match: Final = EXACT_TOOLCHAIN.search(windows_command)
|
||||
assert match is not None
|
||||
|
|
|
|||
|
|
@ -388,6 +388,7 @@ def test_interrupt_spares_the_invoking_process(tmp_path: Path) -> None:
|
|||
)
|
||||
try:
|
||||
assert _wait_until((hang_dir / "make.started").exists, 10)
|
||||
assert _wait_until((hang_dir / "eslint_report.started").exists, 10)
|
||||
os.killpg(proc.pid, signal.SIGINT)
|
||||
assert proc.wait(timeout=10) == 0
|
||||
assert _wait_until(marker.exists, 5)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ def _run(cmd):
|
|||
return subprocess.call(cmd)
|
||||
|
||||
|
||||
def main():
|
||||
def main(argv):
|
||||
wheels = glob.glob(os.path.join("dist", "*.whl"))
|
||||
if not wheels:
|
||||
print("::error::no wheel in dist/; run `uv build --wheel --out-dir dist` first")
|
||||
|
|
@ -51,6 +51,9 @@ def main():
|
|||
for n in offenders[:15]:
|
||||
print(f" on-disk {WORST_CASE_PREFIX + len(n):4} {n}")
|
||||
return 1
|
||||
if "--lengths-only" in argv:
|
||||
print(f"ok: every path in {os.path.basename(wheel)} fits MAX_PATH at a {WORST_CASE_PREFIX}-char prefix")
|
||||
return 0
|
||||
|
||||
venv = _deep_venv_dir()
|
||||
os.makedirs(os.path.dirname(venv), exist_ok=True)
|
||||
|
|
@ -73,4 +76,4 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import zipfile
|
|||
from check_windows_wheel_install import (
|
||||
MAX_PATH,
|
||||
WORST_CASE_PREFIX,
|
||||
main,
|
||||
overlong_install_paths,
|
||||
)
|
||||
|
||||
|
|
@ -34,3 +35,24 @@ def test_orders_offenders_longest_first(tmp_path):
|
|||
longer,
|
||||
shorter,
|
||||
]
|
||||
|
||||
|
||||
def _dist_with(tmp_path, *entry_names):
|
||||
dist = tmp_path / "dist"
|
||||
dist.mkdir()
|
||||
with zipfile.ZipFile(dist / "litellm-0-py3-none-any.whl", "w") as zf:
|
||||
for name in entry_names:
|
||||
zf.writestr(name, "{}")
|
||||
|
||||
|
||||
def test_lengths_only_passes_without_installing(tmp_path, monkeypatch):
|
||||
_dist_with(tmp_path, "litellm/__init__.py")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("PATH", "")
|
||||
assert main(["--lengths-only"]) == 0
|
||||
|
||||
|
||||
def test_lengths_only_fails_on_an_overlong_path(tmp_path, monkeypatch):
|
||||
_dist_with(tmp_path, "a" * (MAX_PATH - WORST_CASE_PREFIX + 1))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
assert main(["--lengths-only"]) == 1
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue