mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
e16d456291
* test(integration): isolate every multi-worker integration suite by default Per-worker database, Redis, upstream and proxy were opt-in for management, accounting, providers and extensions. mcp, security and cost ran 4 to 8 xdist workers against one shared proxy, database and Redis, and those suites hold most of the integration flakes CircleCI records. Every run with more than one worker now builds a private stack per worker, run_integration.sh no longer starts or exports the shared proxy, upstream, database or Redis in that mode, and each worker's scripted upstream runs in its own process instead of a thread in the pytest controller. Worker gateways get what the shared proxy used to provide: a peer for management and mcp, the scripted cost map for cost, stdio MCP servers, and coverage for mcp. mcp coverage now measures lines through sys.monitoring, because branch measurement on CPython 3.12 uses the trace function and that made a covered proxy take about 4x as long to become ready, which is why the shared mcp proxy kept missing its 90 s readiness wait. Routing parity snapshots now also read the worker databases, since its cost and mcp runs use several workers. The security per-worker fixture is redundant and removed * test(integration): give the nested probe root the files a worker gateway needs Two worker probes now run isolated, so the nested controller boots a real per-worker upstream and proxy from the probe root. That upstream imports the fake OpenAI server and the cost map modules, and the proxy reads proxy_config.yaml, none of which the probe copied. The nested run also drops INTEGRATION_COVERAGE_FILE so it never writes into an outer coverage file * test(integration): budget nested multi-worker runs for their worker stack boots A nested run.py database run with more than one worker now boots a private database, Redis, upstream and proxy per worker, one after another, before any case runs. The fixed 60 s subprocess timeout left no room for that, so the 2 worker probes sat at 57-60 s on CircleCI. The subprocess budget is now the old 60 s plus 30 s per worker stack, and the outer pytest timeout keeps its previous 30 s of slack on top. 30 s is the slowest worker proxy readiness recorded in startup.json on the integration-cost job of workflow e09b108b (21.6-30.3 s on 4 CPUs, 2026-10-12) --------- Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
48 lines
1.7 KiB
Python
48 lines
1.7 KiB
Python
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from tests.integration.run import GROUPS, select, uncollected, worker_settings
|
|
|
|
_GROUP: Final = (
|
|
"tests/integration/cost_calculation/test_cost_tracking.py",
|
|
"tests/integration/cost_calculation/test_rollups.py",
|
|
)
|
|
_CELL: Final = (
|
|
"tests/integration/cost_calculation/test_cost_tracking.py"
|
|
"::test_case_bills_expected_cost[perplexity/pplx-decider-v1-27b-decisions]"
|
|
)
|
|
|
|
|
|
def test_a_node_id_inside_a_group_file_is_selected_as_written() -> None:
|
|
selection: Final = select((_CELL,), _GROUP)
|
|
assert selection.nodes == (_CELL,)
|
|
assert selection.foreign == ()
|
|
|
|
|
|
def test_a_node_id_outside_the_group_is_foreign_by_its_file() -> None:
|
|
foreign: Final = "tests/integration/providers/test_decisions_wire.py::test_key_checks_match_chat"
|
|
assert select((foreign, _CELL), _GROUP).foreign == (foreign,)
|
|
|
|
|
|
def test_no_request_selects_every_group_file() -> None:
|
|
assert select((), _GROUP).nodes == _GROUP
|
|
|
|
|
|
def test_a_node_id_whose_file_collected_tests_is_not_empty() -> None:
|
|
collected: Final = frozenset({_CELL, "tests/integration/cost_calculation/test_cost_tracking.py::test_other"})
|
|
assert uncollected((_CELL,), collected) == ()
|
|
|
|
|
|
def test_a_selected_file_that_collected_nothing_is_reported() -> None:
|
|
assert uncollected(_GROUP, frozenset({_CELL})) == ("tests/integration/cost_calculation/test_rollups.py",)
|
|
|
|
|
|
@pytest.mark.parametrize("group", tuple(GROUPS))
|
|
def test_every_group_run_by_several_workers_gives_each_worker_its_own_stack(group: str) -> None:
|
|
assert worker_settings(group, 2)["INTEGRATION_WORKER_ISOLATION"] == "1"
|
|
|
|
|
|
@pytest.mark.parametrize("group", tuple(GROUPS))
|
|
def test_a_single_worker_run_keeps_the_supplied_deployment(group: str) -> None:
|
|
assert worker_settings(group, 1) == {}
|