diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index fb86237e504..f26c649aad2 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -8,6 +8,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -21,16 +22,19 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - - name: Fetch the pull request head, its merge base, and the base branch tip + - name: Check out the base branch tip and fetch the pull request head and its merge base id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')" + base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)" + target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')" merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target" + git checkout -q --detach "$target" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" echo "target=$target" >> "$GITHUB_OUTPUT" - name: Set up uv diff --git a/.github/workflows/spend-logs-guard.yml b/.github/workflows/spend-logs-guard.yml index cc3bda9b880..bdba95f6a5d 100644 --- a/.github/workflows/spend-logs-guard.yml +++ b/.github/workflows/spend-logs-guard.yml @@ -19,17 +19,17 @@ jobs: steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ github.event.pull_request.base.ref }} persist-credentials: false - name: Fetch the pull request head, its merge base and its reviews id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | - merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 88c66f0011d..c3388430589 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -48,11 +48,13 @@ jobs: if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; } - MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$MERGE_BASE" retry git fetch --no-tags --depth=1 origin "$MERGE_BASE" echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" @@ -264,6 +266,7 @@ jobs: name: ui permissions: contents: read + pull-requests: read runs-on: ubuntu-latest timeout-minutes: 8 defaults: @@ -281,16 +284,12 @@ jobs: id: changed env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - # base.sha is the base branch tip from when the PR was opened, while - # actions/checkout leaves HEAD on a merge of the PR into the *current* - # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit - # landed since, so a PR that touches no UI file still gets linted - # against hundreds of other people's files. Diff the PR head against its - # own merge base instead, which is exactly what this PR changed. - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" : > "$RUNNER_TEMP/prettier_files.txt" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a5013155eae..9ebb9bab018 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1150,18 +1150,20 @@ jobs: env: CI: "true" GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | full_suite() { npm run test -- --run --pool forks --maxWorkers=14; } - if [ -z "$BASE_SHA" ]; then + if [ -z "$PR_NUMBER" ]; then echo "Push to $GITHUB_REF_NAME: running the full suite" full_suite exit 0 fi - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" changed_files=() diff --git a/tests/unit/test_lint_workflow_diff_gates.py b/tests/unit/test_lint_workflow_diff_gates.py index 5d3c0d80aa6..816ef6ce707 100644 --- a/tests/unit/test_lint_workflow_diff_gates.py +++ b/tests/unit/test_lint_workflow_diff_gates.py @@ -1,19 +1,198 @@ from __future__ import annotations import fnmatch +import itertools +import os import re import shlex import subprocess +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, TypeAlias import pytest import yaml +from pydantic import BaseModel, ConfigDict, Field WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml" +WORKFLOWS_DIR: Final = WORKFLOW.parent +REPO_ROOT: Final = WORKFLOW.parents[2] DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|') GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text())) PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)")) +REPOSITORY: Final = "BerriAI/litellm" +UI_DASHBOARD: Final = "ui/litellm-dashboard" +STACKED_BASE_REF: Final = "litellm_/stacked" +RELEASE_BASE_REF: Final = "stable/next" +BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF) +RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun" +GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +FAKE_GH: Final = r"""#!/usr/bin/env bash +set -euo pipefail +endpoint=${2%%\?*} +endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"} +commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; } +renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; } +case "$1 $endpoint ${3-} ${4-}" in + "api compare/"*"..."*" --jq .merge_base_commit.sha") + basehead=${endpoint#compare/} + git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;; + "api pulls/1 --jq .base.ref") + printf '%s\n' "$LIVE_BASE_REF" ;; + "api branches/"*) + branch=$(renamed "${endpoint#branches/}") + sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; } + case "${3-} ${4-}" in + "--silent ") ;; + "--jq .name") printf '%s\n' "$branch" ;; + "--jq .commit.sha") printf '%s\n' "$sha" ;; + *) echo "fake gh cannot answer: $*" >&2; exit 64 ;; + esac ;; + "api pulls/1/reviews --paginate --slurp") + printf '[[]]\n' ;; + *) + echo "fake gh cannot answer: $*" >&2 + exit 64 ;; +esac +""" +FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash +printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv" +""" + + +class WorkflowStep(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = "" + name: str = "" + uses: str = "" + with_: Mapping[str, str] = Field(default_factory=dict, alias="with") + env: Mapping[str, str] = Field(default_factory=dict) + run: str = "" + + +class RunDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + working_directory: str = Field(default=".", alias="working-directory") + + +class JobDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + run: RunDefaults = Field(default_factory=RunDefaults) + + +class WorkflowJob(BaseModel): + model_config = ConfigDict(frozen=True) + + defaults: JobDefaults = Field(default_factory=JobDefaults) + steps: tuple[WorkflowStep, ...] = () + + +class PullRequestTrigger(BaseModel): + model_config = ConfigDict(frozen=True) + + branches: tuple[str, ...] = ("**",) + + +class WorkflowTriggers(BaseModel): + model_config = ConfigDict(frozen=True) + + pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger) + + +class Workflow(BaseModel): + model_config = ConfigDict(frozen=True) + + on: WorkflowTriggers + jobs: Mapping[str, WorkflowJob] + + +@dataclass(frozen=True, slots=True) +class MergeBaseStep: + workflow: str + job_id: str + job: WorkflowJob + step: WorkflowStep + base_branches: tuple[str, ...] + + @property + def job_key(self) -> str: + return f"{self.workflow}:{self.job_id}" + + +@dataclass(frozen=True, slots=True) +class StaleBaseUpstream: + path: Path + stale_base: str + head: str + tip_by_ref: Mapping[str, str] + merge_base_by_ref: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class Runner: + root: Path + upstream: StaleBaseUpstream + live_base_ref: str + context: Mapping[str, str] + + @property + def checkout(self) -> Path: + return self.root / "checkout" + + @property + def tools(self) -> Path: + return self.root / "bin" + + @property + def temp(self) -> Path: + return self.root / "runner_temp" + + @property + def github_output(self) -> Path: + return self.root / "github_output" + + @property + def github_env(self) -> Path: + return self.root / "github_env" + + +@dataclass(frozen=True, slots=True) +class StepRun: + step: MergeBaseStep + runner: Runner + + +Observation: TypeAlias = str | frozenset[str] | tuple[str, str] + + +@dataclass(frozen=True, slots=True) +class Consumer: + observe: Callable[[StepRun], Observation] + expect: Callable[[StaleBaseUpstream, str], Observation] + + +def _workflow(path: Path) -> Workflow: + return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader)) + + +def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]: + workflow: Final = _workflow(path) + for job_id, job in workflow.jobs.items(): + yield from ( + MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches) + for step in job.steps + if "merge_base_commit" in step.run + ) + + +MERGE_BASE_STEPS: Final = tuple( + itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml")))) +) def _git(cwd: Path, *args: str) -> str: @@ -104,6 +283,218 @@ def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manife assert selected == frozenset({trigger}) +def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType( + { + "github.repository": REPOSITORY, + "github.token": "fake-token", + **{f"github.event.pull_request.{field}": value for field, value in pull_request.items()}, + } + ) + + +def _context_value(context: Mapping[str, str], expression: str) -> str: + if expression not in context: + pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}") + return context[expression] + + +def _resolved(text: str, context: Mapping[str, str]) -> str: + return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text) + + +def _install_tool(path: Path, script: str) -> None: + path.write_text(script) + path.chmod(0o755) + + +def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner: + runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request)) + runner.tools.mkdir() + runner.temp.mkdir() + for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)): + _install_tool(runner.tools / name, script) + return runner + + +def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None: + checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@")) + ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context) + _git(runner.root, "init", "-q", str(runner.checkout)) + _git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}") + _git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref) + _git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD") + + +def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None: + result: Final = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)], + cwd=runner.checkout / job.defaults.run.working_directory, + env={ + **{name: _resolved(value, context) for name, value in step.env.items()}, + "PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}", + "HOME": str(runner.root), + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_OUTPUT": str(runner.github_output), + "GITHUB_ENV": str(runner.github_env), + "RUNNER_TEMP": str(runner.temp), + "UPSTREAM": str(runner.upstream.path), + "LIVE_BASE_REF": runner.live_base_ref, + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def _key_values(path: Path) -> Mapping[str, str]: + return MappingProxyType( + {key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())} + ) + + +def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]: + return tuple((runner.temp / f"{tool}.argv").read_text().splitlines()) + + +def _gate_base_exported(run: StepRun) -> str: + return _key_values(run.runner.github_env)["GATE_BASE_SHA"] + + +def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]: + return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines()) + + +def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]: + argv: Final = _argv_recorded(run.runner, "npm") + return frozenset(argv[argv.index("related") + 1 : argv.index("--run")]) + + +def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]: + guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard") + outputs: Final = { + f"steps.{run.step.step.id}.outputs.{name}": value + for name, value in _key_values(run.runner.github_output).items() + } + _run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner) + argv: Final = _argv_recorded(run.runner, "uv") + return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD") + + +def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str: + return upstream.merge_base_by_ref[ref] + + +def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]: + merge_base: Final = upstream.merge_base_by_ref[ref] + changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head) + return frozenset(changed.split()) + + +def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]: + return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref] + + +CONSUMERS: Final = MappingProxyType( + { + "test-linting.yml:python": Consumer(_gate_base_exported, _merge_base), + "test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base), + "test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base), + "spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + "cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + } +) + + +def _sha(repo: Path, revision: str) -> str: + return _git(repo, "rev-parse", revision).strip() + + +def _commit_ui_file(repo: Path, name: str) -> None: + target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"export const {name} = 1;\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", name) + + +@pytest.fixture(scope="module") +def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream: + path: Final = tmp_path_factory.mktemp("upstream") + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.email", "t@t") + _git(path, "config", "user.name", "t") + _commit_ui_file(path, "main_when_the_pull_request_was_opened") + _git(path, "branch", "feature") + _git(path, "branch", RELEASE_BASE_REF) + _commit_ui_file(path, "main_since") + _git(path, "checkout", "-q", "-b", STACKED_BASE_REF) + _commit_ui_file(path, "stacked_base_since") + _git(path, "checkout", "-q", "feature") + _commit_ui_file(path, "pull_request_work") + _git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF) + _git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF) + return StaleBaseUpstream( + path=path, + stale_base=_sha(path, "main~1"), + head=_sha(path, "feature"), + tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}), + merge_base_by_ref=MappingProxyType( + {ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS} + ), + ) + + +def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None: + assert MERGE_BASE_STEPS + assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset( + path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text() + ) + + +def _runs_for(step: MergeBaseStep, base_ref: str) -> bool: + return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches) + + +BASE_REF_CASES: Final = ( + ("main", "main", "main"), + ("stacked", STACKED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"), + ("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF), +) +MERGE_BASE_CASES: Final = tuple( + pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}") + for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES) + if _runs_for(step, event_base_ref) +) + + +@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES) +def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone( + upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str +) -> None: + pull_request: Final = MappingProxyType( + { + "number": "1", + "base.ref": event_base_ref, + "base.sha": upstream.stale_base, + "head.sha": upstream.head, + "head.ref": "feature", + "user.type": "User", + } + ) + runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref) + _check_out_like_the_job(step.job, runner) + _run_step(step.step, step.job, runner.context, runner) + consumer: Final = CONSUMERS[step.job_key] + measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref + assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref) + + UI_API_TYPES_DETECT: Final[str] = next( step["run"] for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"] diff --git a/tests/unit/test_select_ui_test_scope.py b/tests/unit/test_select_ui_test_scope.py index 4605aab956c..8f5a4b464bd 100644 --- a/tests/unit/test_select_ui_test_scope.py +++ b/tests/unit/test_select_ui_test_scope.py @@ -18,8 +18,12 @@ that consumes it, running the step's real shell against stubbed `gh`, `git` and from __future__ import annotations import os +import re import subprocess +from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType +from typing import Final import pytest import yaml @@ -28,6 +32,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SCOPE_SCRIPT = REPO_ROOT / ".github" / "scripts" / "select_ui_test_scope.sh" WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test-unit.yml" STEP_NAME = "Run UI unit tests (Vitest)" +EXPRESSION = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +PULL_REQUEST_EVENT: Final = MappingProxyType( + {"number": "1", "base.ref": "main", "base.sha": "basesha", "head.sha": "headsha"} +) FULL_SUITE_ARGV = ["run", "test", "--", "--run", "--pool", "forks", "--maxWorkers=14"] @@ -73,13 +81,26 @@ def test_an_empty_change_set_fails_open_to_the_full_suite() -> None: assert scope([]) == "full" -def _step_script() -> str: - workflow = yaml.safe_load(WORKFLOW.read_text()) - steps = workflow["jobs"]["ui-unit"]["steps"] - script = next(step["run"] for step in steps if step.get("name") == STEP_NAME) - resolved = script.replace("${{ github.repository }}", "BerriAI/litellm") - assert "${{" not in resolved, "the step uses an Actions expression this harness does not resolve" - return resolved +def _resolve(text: str, pull_request: Mapping[str, str] | None) -> str: + def value(match: re.Match[str]) -> str: + expression: Final = match.group(1) + if expression == "github.repository": + return "BerriAI/litellm" + if expression == "github.token": + return "token" + field: Final = expression.removeprefix("github.event.pull_request.") + assert field != expression, f"the step uses ${{{{ {expression} }}}}, which this harness does not resolve" + return "" if pull_request is None else pull_request[field] + + return EXPRESSION.sub(value, text) + + +def _step_script_and_env(pull_request: Mapping[str, str] | None) -> tuple[str, dict[str, str]]: + workflow: Final = yaml.safe_load(WORKFLOW.read_text()) + steps: Final = workflow["jobs"]["ui-unit"]["steps"] + step: Final = next(step for step in steps if step.get("name") == STEP_NAME) + env: Final = {name: _resolve(str(value), pull_request) for name, value in step.get("env", {}).items()} + return _resolve(step["run"], pull_request), env def _stub(bin_dir: Path, name: str, body: str) -> None: @@ -88,7 +109,9 @@ def _stub(bin_dir: Path, name: str, body: str) -> None: stub.chmod(0o755) -def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> tuple[int, str, list[list[str]]]: +def _run_step( + tmp_path: Path, changed: list[str], pull_request: Mapping[str, str] | None = PULL_REQUEST_EVENT +) -> tuple[int, str, list[list[str]]]: """Run the workflow step's real shell; return its status, stdout and every npm argv.""" bin_dir = tmp_path / "bin" bin_dir.mkdir() @@ -100,20 +123,22 @@ def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> _stub(bin_dir, "git", 'if [ "$1" = diff ]; then cat "$CHANGED_FILES"; fi') _stub(bin_dir, "npm", 'printf "%s\\n" "$@" >>"$NPM_LOG"; printf "\\0" >>"$NPM_LOG"') - step = tmp_path / "step.sh" - step.write_text(_step_script()) + step_script, step_env = _step_script_and_env(pull_request) + script: Final = tmp_path / "step.sh" + script.write_text(step_script) - env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env["BASE_SHA"] = base_sha - env["HEAD_SHA"] = "headsha" - env["GITHUB_WORKSPACE"] = str(REPO_ROOT) - env["GITHUB_REF_NAME"] = "release_branch" - env["CHANGED_FILES"] = str(changed_file) - env["NPM_LOG"] = str(npm_log) + env: Final = { + **os.environ, + **step_env, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_REF_NAME": "release_branch", + "CHANGED_FILES": str(changed_file), + "NPM_LOG": str(npm_log), + } result = subprocess.run( - ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(step)], + ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], cwd=tmp_path, capture_output=True, text=True, @@ -155,7 +180,7 @@ def test_step_runs_the_full_suite_when_a_changed_file_is_outside_src(tmp_path: P def test_step_runs_the_full_suite_on_a_push(tmp_path: Path) -> None: - returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], base_sha="") + returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], pull_request=None) assert returncode == 0, output assert invocations == [FULL_SUITE_ARGV]