mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
9b53ee878f
* perf(ci): distribute unit cases and reuse security sweep connections * revert(ci): reject first speed trial after new failures * perf(ci): avoid repeated test discovery and parsing * perf(ci): isolate parallel security workers * revert(ci): restore measured security execution isolation * perf(tests): reuse source scans and deduplicate cache cleanup * perf(ci): restore parallel security with safe log batching * test(ci): isolate fixture state and schema comparisons * test(ci): make schema and scheduler fixtures deterministic * test(ci): drain queued logging before fixture teardown * test(ci): preserve logging delivery without idle worker restarts * test(ci): keep guardrail sinks available during parallel bursts * test(ci): isolate full proxy lifecycle from gateway imports * test(ci): preserve first failure responses and sweep evidence * test(ci): restore documentation URL settings between cases * test(ci): isolate open MCP tools in private gateways * test(ci): group private MCP cases and handle exited workers * test(ci): retain proxy exits and MCP failure details * test(ci): repair merge interactions with main Key the schema shard's full clone on artifact-name, since main removed the unit-flag input. Give each owned Redis its own log file again. Hand the realtime transcription tests main added a logging object whose dispatch handlers are awaitable, so the per-test logging drain no longer waits on a MagicMock task forever * perf(ci): shard owned integration cases and close lifecycle leaks * fix(ci): qualify shard execution and harden interrupted cleanup * fix(ci): bound worker supervision and own fault traffic * perf(ci): reuse unparametrized fixtures across bounded case groups * fix(ci): preserve boot fault allowances and complete exit records * test(ci): tolerate startup retries and retain readiness details * perf(ci): balance integration shards by recorded duration * fix(ci): load migration database helpers only for workers
113 lines
4.2 KiB
Python
113 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from collections import Counter
|
|
from itertools import chain
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|
|
|
from tests.integration._support.ordering import TIMINGS, CaseTiming
|
|
|
|
|
|
class ShardingOptions(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
directory: Path
|
|
shards: int = Field(gt=0)
|
|
timings_output: Path | None = None
|
|
|
|
|
|
class ShardExecution(BaseModel):
|
|
model_config = ConfigDict(frozen=True, strict=True)
|
|
|
|
inventory: tuple[str, ...]
|
|
collected: tuple[str, ...]
|
|
passed: tuple[str, ...]
|
|
skipped: tuple[str, ...]
|
|
shard_count: int
|
|
shard_index: int
|
|
complete: bool
|
|
exitstatus: int
|
|
timings: tuple[CaseTiming, ...] = ()
|
|
|
|
|
|
def merged_timings(executions: tuple[ShardExecution, ...]) -> tuple[CaseTiming, ...]:
|
|
timings: Final = tuple(chain.from_iterable(execution.timings for execution in executions))
|
|
measured: Final = Counter(timing.nodeid for timing in timings)
|
|
selected: Final = Counter(chain.from_iterable(execution.collected for execution in executions))
|
|
if measured != selected or any(amount != 1 for amount in measured.values()):
|
|
return ()
|
|
return tuple(sorted(timings, key=lambda timing: timing.nodeid))
|
|
|
|
|
|
def shard_errors(executions: tuple[ShardExecution, ...], count: int) -> tuple[str, ...]:
|
|
if not executions:
|
|
return ("No integration shard execution records",)
|
|
inventory: Final = frozenset(executions[0].inventory)
|
|
selected: Final = Counter(chain.from_iterable(execution.collected for execution in executions))
|
|
indices: Final = tuple(execution.shard_index for execution in executions)
|
|
return (
|
|
*(("Missing or duplicate integration shards",) if sorted(indices) != list(range(count)) else ()),
|
|
*(
|
|
("Integration shard counts disagree",)
|
|
if any(execution.shard_count != count for execution in executions)
|
|
else ()
|
|
),
|
|
*(
|
|
("Integration collection inventories disagree",)
|
|
if any(frozenset(execution.inventory) != inventory for execution in executions)
|
|
else ()
|
|
),
|
|
*(
|
|
("Integration inventory is empty or contains duplicates",)
|
|
if not inventory or any(len(execution.inventory) != len(inventory) for execution in executions)
|
|
else ()
|
|
),
|
|
*(
|
|
("Integration selection omitted, duplicated, or added test cases",)
|
|
if frozenset(selected) != inventory or any(value != 1 for value in selected.values())
|
|
else ()
|
|
),
|
|
*(
|
|
("An integration shard failed or did not execute its complete selection",)
|
|
if any(
|
|
not execution.complete
|
|
or execution.exitstatus != 0
|
|
or Counter(execution.collected) != Counter(execution.passed + execution.skipped)
|
|
for execution in executions
|
|
)
|
|
else ()
|
|
),
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
parser: Final = argparse.ArgumentParser()
|
|
parser.add_argument("directory", type=Path)
|
|
parser.add_argument("--shards", required=True, type=int)
|
|
parser.add_argument("--timings-output", type=Path)
|
|
options: Final = ShardingOptions.model_validate(vars(parser.parse_args()))
|
|
paths: Final = tuple(sorted(options.directory.glob("*/execution.json")))
|
|
try:
|
|
executions: Final = tuple(ShardExecution.model_validate_json(path.read_text()) for path in paths)
|
|
except (OSError, ValidationError) as error:
|
|
sys.stderr.write(f"Invalid integration shard execution record: {error}\n")
|
|
return 1
|
|
errors: Final = shard_errors(executions, options.shards)
|
|
if errors:
|
|
sys.stderr.write("\n".join(errors) + "\n")
|
|
return 1
|
|
if options.timings_output is not None:
|
|
options.timings_output.parent.mkdir(parents=True, exist_ok=True)
|
|
options.timings_output.write_bytes(TIMINGS.dump_json(merged_timings(executions)))
|
|
sys.stdout.write(
|
|
f"Verified {len(executions[0].inventory)} integration cases across {options.shards} isolated shards\n"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|