mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
* fix(ci): measure PR gates from the target branch's current tip The lint gate base, the ui lint and ui unit test scopes, and the spend logs guard compared the event's pull_request.base.sha with the head to find the merge base. GitHub can leave base.sha weeks behind the base branch, and when that old commit is an ancestor of a head that merged newer main, compare returns the old commit itself, so the strict gates blamed main's own code on the PR. Compare base.ref with the head instead, passed through env, so GitHub resolves the branch tip when the job runs. This is the same base make check uses locally. The ui unit test push guard now keys on BASE_REF, which is empty on push and workflow_dispatch. The regression test runs every workflow step that reads merge_base_commit against a fake gh backed by a temp repo with a stale base.sha, for main and a litellm_/ base with a slash. * fix(ci): check out the base branch's current guard code in the PR guards * fix(ci): read the PR's current base branch so re-runs survive a deleted base Every merge-base step now asks the pulls API for the PR's base branch instead of reading the event's base.ref, which a re-run keeps after a stacked base is merged and deleted. The guard checkouts go back to base.sha, which stays reachable. * test(ci): mark the step script path Final * fix(ci): keep the spend logs guard on main and run both guard scripts from the base branch tip * fix(ci): measure from the event's base branch unless it was deleted, and test what each step hands on A re-run after a retarget off main kept following the PR's current base, so lint, ui-unit and the cost map guard could pass without measuring against main. They now read the current base only when the event's base branch no longer exists. The tests run each merge base step whole and check the value its consumer gets * fix(ci): follow a renamed base branch to its new name before the compare --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
593 lines
22 KiB
Python
593 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import fnmatch
|
|
import itertools
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
from collections.abc import Callable, Iterator, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from types import MappingProxyType
|
|
from typing import Final, TypeAlias
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml"
|
|
WORKFLOWS_DIR: Final = WORKFLOW.parent
|
|
REPO_ROOT: Final = WORKFLOW.parents[2]
|
|
DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|')
|
|
GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text()))
|
|
PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)"))
|
|
REPOSITORY: Final = "BerriAI/litellm"
|
|
UI_DASHBOARD: Final = "ui/litellm-dashboard"
|
|
STACKED_BASE_REF: Final = "litellm_/stacked"
|
|
RELEASE_BASE_REF: Final = "stable/next"
|
|
BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF)
|
|
RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun"
|
|
GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}")
|
|
FAKE_GH: Final = r"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
endpoint=${2%%\?*}
|
|
endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"}
|
|
commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; }
|
|
renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; }
|
|
case "$1 $endpoint ${3-} ${4-}" in
|
|
"api compare/"*"..."*" --jq .merge_base_commit.sha")
|
|
basehead=${endpoint#compare/}
|
|
git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;;
|
|
"api pulls/1 --jq .base.ref")
|
|
printf '%s\n' "$LIVE_BASE_REF" ;;
|
|
"api branches/"*)
|
|
branch=$(renamed "${endpoint#branches/}")
|
|
sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; }
|
|
case "${3-} ${4-}" in
|
|
"--silent ") ;;
|
|
"--jq .name") printf '%s\n' "$branch" ;;
|
|
"--jq .commit.sha") printf '%s\n' "$sha" ;;
|
|
*) echo "fake gh cannot answer: $*" >&2; exit 64 ;;
|
|
esac ;;
|
|
"api pulls/1/reviews --paginate --slurp")
|
|
printf '[[]]\n' ;;
|
|
*)
|
|
echo "fake gh cannot answer: $*" >&2
|
|
exit 64 ;;
|
|
esac
|
|
"""
|
|
FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash
|
|
printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv"
|
|
"""
|
|
|
|
|
|
class WorkflowStep(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
id: str = ""
|
|
name: str = ""
|
|
uses: str = ""
|
|
with_: Mapping[str, str] = Field(default_factory=dict, alias="with")
|
|
env: Mapping[str, str] = Field(default_factory=dict)
|
|
run: str = ""
|
|
|
|
|
|
class RunDefaults(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
working_directory: str = Field(default=".", alias="working-directory")
|
|
|
|
|
|
class JobDefaults(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
run: RunDefaults = Field(default_factory=RunDefaults)
|
|
|
|
|
|
class WorkflowJob(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
defaults: JobDefaults = Field(default_factory=JobDefaults)
|
|
steps: tuple[WorkflowStep, ...] = ()
|
|
|
|
|
|
class PullRequestTrigger(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
branches: tuple[str, ...] = ("**",)
|
|
|
|
|
|
class WorkflowTriggers(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger)
|
|
|
|
|
|
class Workflow(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
on: WorkflowTriggers
|
|
jobs: Mapping[str, WorkflowJob]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class MergeBaseStep:
|
|
workflow: str
|
|
job_id: str
|
|
job: WorkflowJob
|
|
step: WorkflowStep
|
|
base_branches: tuple[str, ...]
|
|
|
|
@property
|
|
def job_key(self) -> str:
|
|
return f"{self.workflow}:{self.job_id}"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class StaleBaseUpstream:
|
|
path: Path
|
|
stale_base: str
|
|
head: str
|
|
tip_by_ref: Mapping[str, str]
|
|
merge_base_by_ref: Mapping[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Runner:
|
|
root: Path
|
|
upstream: StaleBaseUpstream
|
|
live_base_ref: str
|
|
context: Mapping[str, str]
|
|
|
|
@property
|
|
def checkout(self) -> Path:
|
|
return self.root / "checkout"
|
|
|
|
@property
|
|
def tools(self) -> Path:
|
|
return self.root / "bin"
|
|
|
|
@property
|
|
def temp(self) -> Path:
|
|
return self.root / "runner_temp"
|
|
|
|
@property
|
|
def github_output(self) -> Path:
|
|
return self.root / "github_output"
|
|
|
|
@property
|
|
def github_env(self) -> Path:
|
|
return self.root / "github_env"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class StepRun:
|
|
step: MergeBaseStep
|
|
runner: Runner
|
|
|
|
|
|
Observation: TypeAlias = str | frozenset[str] | tuple[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Consumer:
|
|
observe: Callable[[StepRun], Observation]
|
|
expect: Callable[[StaleBaseUpstream, str], Observation]
|
|
|
|
|
|
def _workflow(path: Path) -> Workflow:
|
|
return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader))
|
|
|
|
|
|
def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]:
|
|
workflow: Final = _workflow(path)
|
|
for job_id, job in workflow.jobs.items():
|
|
yield from (
|
|
MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches)
|
|
for step in job.steps
|
|
if "merge_base_commit" in step.run
|
|
)
|
|
|
|
|
|
MERGE_BASE_STEPS: Final = tuple(
|
|
itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml"))))
|
|
)
|
|
|
|
|
|
def _git(cwd: Path, *args: str) -> str:
|
|
return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def _scoped_root(pathspec: str) -> str:
|
|
return re.sub(r"^:\([^)]*\)", "", pathspec).split("*", 1)[0]
|
|
|
|
|
|
def _gate_rooted_at(root: str) -> tuple[str, ...]:
|
|
return next(gate for gate in GATES if _scoped_root(gate[0]) == root)
|
|
|
|
|
|
def _changed_files_selected_by(tmp_path: Path, pathspecs: tuple[str, ...], files: tuple[str, ...]) -> frozenset[str]:
|
|
_git(tmp_path, "init", "-q", "-b", "main")
|
|
_git(tmp_path, "config", "user.email", "t@t")
|
|
_git(tmp_path, "config", "user.name", "t")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base")
|
|
for name in files:
|
|
target = tmp_path / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text("x = 1\n")
|
|
_git(tmp_path, "add", "-A")
|
|
_git(tmp_path, "commit", "-qm", "change")
|
|
return frozenset(
|
|
_git(tmp_path, "diff", "--name-only", "--diff-filter=ACMRD", "HEAD~1", "HEAD", "--", *pathspecs).split()
|
|
)
|
|
|
|
|
|
def test_workflow_still_carries_the_ruff_format_e2e_basedpyright_and_e2e_harness_diff_gates() -> None:
|
|
assert len(GATES) == 3
|
|
assert frozenset(gate[0] for gate in GATES) == frozenset(
|
|
{":(glob)litellm/**/*.py", ":(glob)tests/e2e/**/*.py", "tests/e2e"}
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("pathspecs", PYTHON_GATES, ids=" ".join)
|
|
def test_diff_gate_selects_top_level_and_nested_python_files_only(tmp_path: Path, pathspecs: tuple[str, ...]) -> None:
|
|
root = _scoped_root(pathspecs[0])
|
|
top_level = f"{root}top_level_module.py"
|
|
nested = f"{root}pkg/sub/nested_module.py"
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
pathspecs,
|
|
(top_level, nested, f"{root}notes.md", "elsewhere/top_level_module.py", "elsewhere/pkg/nested_module.py"),
|
|
)
|
|
assert selected == frozenset({top_level, nested})
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"trigger",
|
|
(
|
|
"tests/e2e_harness/top_level_module.py",
|
|
"tests/e2e_harness/pkg/sub/nested_module.py",
|
|
"pyrightconfig.json",
|
|
),
|
|
)
|
|
def test_e2e_basedpyright_gate_also_fires_on_harness_python_and_pyrightconfig(tmp_path: Path, trigger: str) -> None:
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
_gate_rooted_at("tests/e2e/"),
|
|
(trigger, "tests/e2e_harness/notes.md", "elsewhere/pyrightconfig.json", "tests/e2e_harnessish/module.py"),
|
|
)
|
|
assert selected == frozenset({trigger})
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"trigger",
|
|
(
|
|
"tests/e2e/claude_code/cron_vm/install_claude_code.sh",
|
|
"tests/e2e/notes.md",
|
|
"tests/e2e/pkg/sub/nested_module.py",
|
|
"tests/e2e_harness/claude_code/test_driver.py",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
".github/workflows/test-linting.yml",
|
|
),
|
|
)
|
|
def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manifests_and_workflow(
|
|
tmp_path: Path, trigger: str
|
|
) -> None:
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
_gate_rooted_at("tests/e2e"),
|
|
(trigger, "tests/e2e/ui/spec.ts", "tests/e2e/ui/pkg/page.py", "elsewhere/pyproject.toml", "tests/e2e_other/module.py"),
|
|
)
|
|
assert selected == frozenset({trigger})
|
|
|
|
|
|
def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]:
|
|
return MappingProxyType(
|
|
{
|
|
"github.repository": REPOSITORY,
|
|
"github.token": "fake-token",
|
|
**{f"github.event.pull_request.{field}": value for field, value in pull_request.items()},
|
|
}
|
|
)
|
|
|
|
|
|
def _context_value(context: Mapping[str, str], expression: str) -> str:
|
|
if expression not in context:
|
|
pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}")
|
|
return context[expression]
|
|
|
|
|
|
def _resolved(text: str, context: Mapping[str, str]) -> str:
|
|
return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text)
|
|
|
|
|
|
def _install_tool(path: Path, script: str) -> None:
|
|
path.write_text(script)
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner:
|
|
runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request))
|
|
runner.tools.mkdir()
|
|
runner.temp.mkdir()
|
|
for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)):
|
|
_install_tool(runner.tools / name, script)
|
|
return runner
|
|
|
|
|
|
def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None:
|
|
checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@"))
|
|
ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context)
|
|
_git(runner.root, "init", "-q", str(runner.checkout))
|
|
_git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}")
|
|
_git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref)
|
|
_git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD")
|
|
|
|
|
|
def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None:
|
|
result: Final = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)],
|
|
cwd=runner.checkout / job.defaults.run.working_directory,
|
|
env={
|
|
**{name: _resolved(value, context) for name, value in step.env.items()},
|
|
"PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}",
|
|
"HOME": str(runner.root),
|
|
"GITHUB_REPOSITORY": REPOSITORY,
|
|
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
|
"GITHUB_OUTPUT": str(runner.github_output),
|
|
"GITHUB_ENV": str(runner.github_env),
|
|
"RUNNER_TEMP": str(runner.temp),
|
|
"UPSTREAM": str(runner.upstream.path),
|
|
"LIVE_BASE_REF": runner.live_base_ref,
|
|
},
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def _key_values(path: Path) -> Mapping[str, str]:
|
|
return MappingProxyType(
|
|
{key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())}
|
|
)
|
|
|
|
|
|
def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]:
|
|
return tuple((runner.temp / f"{tool}.argv").read_text().splitlines())
|
|
|
|
|
|
def _gate_base_exported(run: StepRun) -> str:
|
|
return _key_values(run.runner.github_env)["GATE_BASE_SHA"]
|
|
|
|
|
|
def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]:
|
|
return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines())
|
|
|
|
|
|
def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]:
|
|
argv: Final = _argv_recorded(run.runner, "npm")
|
|
return frozenset(argv[argv.index("related") + 1 : argv.index("--run")])
|
|
|
|
|
|
def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]:
|
|
guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard")
|
|
outputs: Final = {
|
|
f"steps.{run.step.step.id}.outputs.{name}": value
|
|
for name, value in _key_values(run.runner.github_output).items()
|
|
}
|
|
_run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner)
|
|
argv: Final = _argv_recorded(run.runner, "uv")
|
|
return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD")
|
|
|
|
|
|
def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str:
|
|
return upstream.merge_base_by_ref[ref]
|
|
|
|
|
|
def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]:
|
|
merge_base: Final = upstream.merge_base_by_ref[ref]
|
|
changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head)
|
|
return frozenset(changed.split())
|
|
|
|
|
|
def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]:
|
|
return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref]
|
|
|
|
|
|
CONSUMERS: Final = MappingProxyType(
|
|
{
|
|
"test-linting.yml:python": Consumer(_gate_base_exported, _merge_base),
|
|
"test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base),
|
|
"test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base),
|
|
"spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
|
"cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
|
}
|
|
)
|
|
|
|
|
|
def _sha(repo: Path, revision: str) -> str:
|
|
return _git(repo, "rev-parse", revision).strip()
|
|
|
|
|
|
def _commit_ui_file(repo: Path, name: str) -> None:
|
|
target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts"
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(f"export const {name} = 1;\n")
|
|
_git(repo, "add", "-A")
|
|
_git(repo, "commit", "-qm", name)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream:
|
|
path: Final = tmp_path_factory.mktemp("upstream")
|
|
_git(path, "init", "-q", "-b", "main")
|
|
_git(path, "config", "user.email", "t@t")
|
|
_git(path, "config", "user.name", "t")
|
|
_commit_ui_file(path, "main_when_the_pull_request_was_opened")
|
|
_git(path, "branch", "feature")
|
|
_git(path, "branch", RELEASE_BASE_REF)
|
|
_commit_ui_file(path, "main_since")
|
|
_git(path, "checkout", "-q", "-b", STACKED_BASE_REF)
|
|
_commit_ui_file(path, "stacked_base_since")
|
|
_git(path, "checkout", "-q", "feature")
|
|
_commit_ui_file(path, "pull_request_work")
|
|
_git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF)
|
|
_git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF)
|
|
return StaleBaseUpstream(
|
|
path=path,
|
|
stale_base=_sha(path, "main~1"),
|
|
head=_sha(path, "feature"),
|
|
tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}),
|
|
merge_base_by_ref=MappingProxyType(
|
|
{ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS}
|
|
),
|
|
)
|
|
|
|
|
|
def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None:
|
|
assert MERGE_BASE_STEPS
|
|
assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset(
|
|
path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text()
|
|
)
|
|
|
|
|
|
def _runs_for(step: MergeBaseStep, base_ref: str) -> bool:
|
|
return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches)
|
|
|
|
|
|
BASE_REF_CASES: Final = (
|
|
("main", "main", "main"),
|
|
("stacked", STACKED_BASE_REF, STACKED_BASE_REF),
|
|
("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"),
|
|
("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF),
|
|
("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF),
|
|
("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF),
|
|
)
|
|
MERGE_BASE_CASES: Final = tuple(
|
|
pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}")
|
|
for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES)
|
|
if _runs_for(step, event_base_ref)
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES)
|
|
def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone(
|
|
upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str
|
|
) -> None:
|
|
pull_request: Final = MappingProxyType(
|
|
{
|
|
"number": "1",
|
|
"base.ref": event_base_ref,
|
|
"base.sha": upstream.stale_base,
|
|
"head.sha": upstream.head,
|
|
"head.ref": "feature",
|
|
"user.type": "User",
|
|
}
|
|
)
|
|
runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref)
|
|
_check_out_like_the_job(step.job, runner)
|
|
_run_step(step.step, step.job, runner.context, runner)
|
|
consumer: Final = CONSUMERS[step.job_key]
|
|
measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref
|
|
assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref)
|
|
|
|
|
|
UI_API_TYPES_DETECT: Final[str] = next(
|
|
step["run"]
|
|
for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"]
|
|
if step.get("id") == "changes"
|
|
)
|
|
PRE_COMMIT_LINT: Final = WORKFLOW.parents[2] / "scripts" / "pre_commit_lint.sh"
|
|
TERRAFORM_PROVIDER_WORKFLOW: Final = WORKFLOW.with_name("test-terraform-provider.yml")
|
|
|
|
|
|
def _ui_api_types_relevant(tmp_path: Path, changed: str) -> bool:
|
|
_git(tmp_path, "init", "-q", "-b", "main")
|
|
_git(tmp_path, "config", "user.email", "t@t")
|
|
_git(tmp_path, "config", "user.name", "t")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base")
|
|
_git(tmp_path, "switch", "-q", "-c", "pr")
|
|
target: Final = tmp_path / changed
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text("x = 1\n")
|
|
_git(tmp_path, "add", "-A")
|
|
_git(tmp_path, "commit", "-qm", "change")
|
|
_git(tmp_path, "switch", "-q", "main")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "main moved")
|
|
_git(tmp_path, "merge", "-q", "--no-ff", "--no-edit", "pr")
|
|
output: Final = tmp_path / "github_output"
|
|
subprocess.run(
|
|
["bash", "-c", UI_API_TYPES_DETECT],
|
|
cwd=tmp_path,
|
|
check=True,
|
|
capture_output=True,
|
|
env={"PATH": "/usr/bin:/bin", "GITHUB_OUTPUT": str(output)},
|
|
)
|
|
return output.read_text().strip() == "relevant=true"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
(
|
|
"litellm/proxy/proxy_server.py",
|
|
"litellm/types/router.py",
|
|
"litellm/tracing/generated/models.py",
|
|
"litellm/router_strategy/complexity_router/config.py",
|
|
"litellm/models/credentials.py",
|
|
"enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
"ui/litellm-dashboard/scripts/gen-api-types.mjs",
|
|
"ui/litellm-dashboard/package-lock.json",
|
|
"ui/litellm-dashboard/src/lib/http/schema.d.ts",
|
|
),
|
|
)
|
|
def test_ui_api_types_check_runs_on_a_pull_request_touching_any_spec_input(tmp_path: Path, changed: str) -> None:
|
|
assert _ui_api_types_relevant(tmp_path, changed)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
(
|
|
"litellm-rust/crates/traces/src/lib.rs",
|
|
"litellm-proxy-extras/litellm_proxy_extras/utils.py",
|
|
"tests/unit/test_router.py",
|
|
"ui/litellm-dashboard/src/app/page.tsx",
|
|
"docs/my-website/docs/proxy/config.md",
|
|
"litellm-dashboard/pyproject.toml",
|
|
),
|
|
)
|
|
def test_ui_api_types_check_skips_a_pull_request_that_cannot_change_the_spec(tmp_path: Path, changed: str) -> None:
|
|
assert not _ui_api_types_relevant(tmp_path, changed)
|
|
|
|
|
|
def _first_single_quoted_capture(pattern: str, text: str) -> str:
|
|
match: Final = re.search(pattern, text, re.MULTILINE)
|
|
assert match is not None, pattern
|
|
return match.group(1)
|
|
|
|
|
|
def _tracked_paths_matching(extended_regex: str) -> frozenset[str]:
|
|
tracked: Final = _git(WORKFLOW.parents[2], "ls-files")
|
|
selected: Final = subprocess.run(["grep", "-E", extended_regex], input=tracked, capture_output=True, text=True)
|
|
return frozenset(selected.stdout.splitlines())
|
|
|
|
|
|
def test_make_check_runs_the_api_type_sync_on_the_same_tracked_files_as_ci() -> None:
|
|
ci_trigger: Final = _first_single_quoted_capture(r"grep -Eq '([^']+)'", UI_API_TYPES_DETECT)
|
|
make_check_trigger: Final = _first_single_quoted_capture(r"^spec_pattern='([^']+)'$", PRE_COMMIT_LINT.read_text())
|
|
ci_selected: Final = _tracked_paths_matching(ci_trigger)
|
|
assert "pyproject.toml" in ci_selected
|
|
assert _tracked_paths_matching(make_check_trigger) == ci_selected - {".github/workflows/test-linting.yml"}
|
|
|
|
|
|
def test_a_pull_request_changing_any_route_file_runs_the_terraform_provider_endpoint_audit() -> None:
|
|
triggers: Final = yaml.safe_load(TERRAFORM_PROVIDER_WORKFLOW.read_text())[True]["pull_request"]["paths"]
|
|
route_files: Final = _git(
|
|
WORKFLOW.parents[2], "grep", "-l", "-E", r"APIRouter\(", "--", "*.py", ":(exclude)tests/**"
|
|
).splitlines()
|
|
assert route_files
|
|
assert [name for name in route_files if not any(fnmatch.fnmatch(name, pattern) for pattern in triggers)] == []
|