mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
9b53ee878f
* perf(ci): distribute unit cases and reuse security sweep connections * revert(ci): reject first speed trial after new failures * perf(ci): avoid repeated test discovery and parsing * perf(ci): isolate parallel security workers * revert(ci): restore measured security execution isolation * perf(tests): reuse source scans and deduplicate cache cleanup * perf(ci): restore parallel security with safe log batching * test(ci): isolate fixture state and schema comparisons * test(ci): make schema and scheduler fixtures deterministic * test(ci): drain queued logging before fixture teardown * test(ci): preserve logging delivery without idle worker restarts * test(ci): keep guardrail sinks available during parallel bursts * test(ci): isolate full proxy lifecycle from gateway imports * test(ci): preserve first failure responses and sweep evidence * test(ci): restore documentation URL settings between cases * test(ci): isolate open MCP tools in private gateways * test(ci): group private MCP cases and handle exited workers * test(ci): retain proxy exits and MCP failure details * test(ci): repair merge interactions with main Key the schema shard's full clone on artifact-name, since main removed the unit-flag input. Give each owned Redis its own log file again. Hand the realtime transcription tests main added a logging object whose dispatch handlers are awaitable, so the per-test logging drain no longer waits on a MagicMock task forever * perf(ci): shard owned integration cases and close lifecycle leaks * fix(ci): qualify shard execution and harden interrupted cleanup * fix(ci): bound worker supervision and own fault traffic * perf(ci): reuse unparametrized fixtures across bounded case groups * fix(ci): preserve boot fault allowances and complete exit records * test(ci): tolerate startup retries and retain readiness details * perf(ci): balance integration shards by recorded duration * fix(ci): load migration database helpers only for workers
117 lines
4.7 KiB
Python
117 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
from types import MappingProxyType
|
|
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from tests.integration._support.ordering import CaseTiming
|
|
from tests.integration._support.sharding import ShardExecution, merged_timings, shard_errors
|
|
from tests.integration.conftest import COLLECTED, IntegrationReportPlugin
|
|
from tests.integration.conftest import INVENTORY as WORKER_INVENTORY
|
|
|
|
INVENTORY: Final = ("test_owned.py::test_first", "test_owned.py::test_skipped", "test_owned.py::test_last")
|
|
|
|
|
|
def _execution(
|
|
index: int,
|
|
collected: tuple[str, ...],
|
|
*,
|
|
inventory: tuple[str, ...] = INVENTORY,
|
|
passed: tuple[str, ...] | None = None,
|
|
skipped: tuple[str, ...] = (),
|
|
shard_count: int = 2,
|
|
complete: bool = True,
|
|
exitstatus: int = 0,
|
|
) -> ShardExecution:
|
|
return ShardExecution(
|
|
inventory=inventory,
|
|
collected=collected,
|
|
passed=tuple(node for node in collected if node not in skipped) if passed is None else passed,
|
|
skipped=skipped,
|
|
shard_count=shard_count,
|
|
shard_index=index,
|
|
complete=complete,
|
|
exitstatus=exitstatus,
|
|
)
|
|
|
|
|
|
def test_shard_qualification_retains_passes_and_skip_identities() -> None:
|
|
executions: Final = (
|
|
_execution(0, INVENTORY[:2], skipped=INVENTORY[1:2]),
|
|
_execution(1, INVENTORY[2:]),
|
|
)
|
|
assert shard_errors(executions, 2) == ()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"executions",
|
|
(
|
|
(),
|
|
(_execution(0, INVENTORY),),
|
|
(_execution(0, INVENTORY[:2]), _execution(0, INVENTORY[2:])),
|
|
(_execution(0, INVENTORY[:1]), _execution(1, INVENTORY[2:])),
|
|
(_execution(0, INVENTORY), _execution(1, INVENTORY[2:])),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], inventory=INVENTORY[:2])),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], complete=False)),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], exitstatus=1)),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], shard_count=3)),
|
|
(_execution(0, INVENTORY[:2], inventory=(*INVENTORY, INVENTORY[0])), _execution(1, INVENTORY[2:])),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], passed=())),
|
|
(_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], skipped=INVENTORY[2:] * 2)),
|
|
(_execution(0, (*INVENTORY[:2], "test_foreign.py::test_other")), _execution(1, INVENTORY[2:])),
|
|
),
|
|
)
|
|
def test_shard_qualification_rejects_lost_duplicate_foreign_or_failed_cases(
|
|
executions: tuple[ShardExecution, ...],
|
|
) -> None:
|
|
assert shard_errors(executions, 2)
|
|
|
|
|
|
def test_timing_snapshot_includes_every_executed_pass_and_skip_once() -> None:
|
|
first: Final = CaseTiming(nodeid=INVENTORY[0], seconds=40.0)
|
|
skipped: Final = CaseTiming(nodeid=INVENTORY[1], seconds=0.0)
|
|
last: Final = CaseTiming(nodeid=INVENTORY[2], seconds=1.0)
|
|
executions: Final = (
|
|
_execution(0, INVENTORY[:2], skipped=INVENTORY[1:2]).model_copy(update={"timings": (first, skipped)}),
|
|
_execution(1, INVENTORY[2:]).model_copy(update={"timings": (last,)}),
|
|
)
|
|
assert shard_errors(executions, 2) == ()
|
|
assert merged_timings(executions) == tuple(sorted((first, skipped, last), key=lambda timing: timing.nodeid))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"timings",
|
|
(
|
|
(),
|
|
(CaseTiming(nodeid=INVENTORY[0], seconds=1.0),),
|
|
(CaseTiming(nodeid=INVENTORY[0], seconds=1.0),) * 2,
|
|
(CaseTiming(nodeid="foreign", seconds=1.0), CaseTiming(nodeid=INVENTORY[1], seconds=0.0)),
|
|
),
|
|
)
|
|
def test_incomplete_duplicate_or_foreign_measurements_cannot_seed_the_timing_cache(
|
|
timings: tuple[CaseTiming, ...],
|
|
) -> None:
|
|
execution: Final = _execution(0, INVENTORY[:2]).model_copy(update={"timings": timings})
|
|
assert merged_timings((execution,)) == ()
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class WorkerResult:
|
|
workeroutput: Mapping[str, object]
|
|
|
|
|
|
def test_worker_results_retain_the_full_inventory_and_reject_disagreement() -> None:
|
|
config: Final = pytest.Config(pytest.PytestPluginManager())
|
|
config.stash[COLLECTED] = INVENTORY[:1]
|
|
plugin: Final = IntegrationReportPlugin(config)
|
|
complete: Final = WorkerResult(MappingProxyType({"integration_inventory": INVENTORY}))
|
|
plugin.pytest_testnodedown(complete, None)
|
|
plugin.pytest_testnodedown(complete, None)
|
|
assert config.stash[WORKER_INVENTORY] == INVENTORY
|
|
assert config.stash[COLLECTED] == INVENTORY[:1]
|
|
incomplete: Final = WorkerResult(MappingProxyType({"integration_inventory": INVENTORY[:1]}))
|
|
with pytest.raises(pytest.UsageError, match="inventories disagree"):
|
|
plugin.pytest_testnodedown(incomplete, None)
|