mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(ci): measure PR gates from the target branch's current tip (#49078)
* fix(ci): measure PR gates from the target branch's current tip The lint gate base, the ui lint and ui unit test scopes, and the spend logs guard compared the event's pull_request.base.sha with the head to find the merge base. GitHub can leave base.sha weeks behind the base branch, and when that old commit is an ancestor of a head that merged newer main, compare returns the old commit itself, so the strict gates blamed main's own code on the PR. Compare base.ref with the head instead, passed through env, so GitHub resolves the branch tip when the job runs. This is the same base make check uses locally. The ui unit test push guard now keys on BASE_REF, which is empty on push and workflow_dispatch. The regression test runs every workflow step that reads merge_base_commit against a fake gh backed by a temp repo with a stale base.sha, for main and a litellm_/ base with a slash. * fix(ci): check out the base branch's current guard code in the PR guards * fix(ci): read the PR's current base branch so re-runs survive a deleted base Every merge-base step now asks the pulls API for the PR's base branch instead of reading the event's base.ref, which a re-run keeps after a stacked base is merged and deleted. The guard checkouts go back to base.sha, which stays reachable. * test(ci): mark the step script path Final * fix(ci): keep the spend logs guard on main and run both guard scripts from the base branch tip * fix(ci): measure from the event's base branch unless it was deleted, and test what each step hands on A re-run after a retarget off main kept following the PR's current base, so lint, ui-unit and the cost map guard could pass without measuring against main. They now read the current base only when the event's base branch no longer exists. The tests run each merge base step whole and check the value its consumer gets * fix(ci): follow a renamed base branch to its new name before the compare --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tavren27
GitHub
parent
23ad8870b4
commit
89ca8d83f8
6 changed files with 461 additions and 40 deletions
10
.github/workflows/cost-map-guard.yml
vendored
10
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -8,6 +8,7 @@ on:
|
|||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
|
|
@ -21,16 +22,19 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
- name: Check out the base branch tip and fetch the pull request head and its merge base
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
git checkout -q --detach "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
|
|
|
|||
6
.github/workflows/spend-logs-guard.yml
vendored
6
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -19,17 +19,17 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
|
|
|
|||
19
.github/workflows/test-linting.yml
vendored
19
.github/workflows/test-linting.yml
vendored
|
|
@ -48,11 +48,13 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$MERGE_BASE"
|
||||
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
|
||||
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
|
||||
|
|
@ -264,6 +266,7 @@ jobs:
|
|||
name: ui
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
|
|
@ -281,16 +284,12 @@ jobs:
|
|||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
|
|
|
|||
8
.github/workflows/test-unit.yml
vendored
8
.github/workflows/test-unit.yml
vendored
|
|
@ -1150,18 +1150,20 @@ jobs:
|
|||
env:
|
||||
CI: "true"
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
|
||||
|
||||
if [ -z "$BASE_SHA" ]; then
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "Push to $GITHUB_REF_NAME: running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
changed_files=()
|
||||
|
|
|
|||
|
|
@ -1,19 +1,198 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml"
|
||||
WORKFLOWS_DIR: Final = WORKFLOW.parent
|
||||
REPO_ROOT: Final = WORKFLOW.parents[2]
|
||||
DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|')
|
||||
GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text()))
|
||||
PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)"))
|
||||
REPOSITORY: Final = "BerriAI/litellm"
|
||||
UI_DASHBOARD: Final = "ui/litellm-dashboard"
|
||||
STACKED_BASE_REF: Final = "litellm_/stacked"
|
||||
RELEASE_BASE_REF: Final = "stable/next"
|
||||
BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF)
|
||||
RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun"
|
||||
GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}")
|
||||
FAKE_GH: Final = r"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
endpoint=${2%%\?*}
|
||||
endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"}
|
||||
commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; }
|
||||
renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; }
|
||||
case "$1 $endpoint ${3-} ${4-}" in
|
||||
"api compare/"*"..."*" --jq .merge_base_commit.sha")
|
||||
basehead=${endpoint#compare/}
|
||||
git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;;
|
||||
"api pulls/1 --jq .base.ref")
|
||||
printf '%s\n' "$LIVE_BASE_REF" ;;
|
||||
"api branches/"*)
|
||||
branch=$(renamed "${endpoint#branches/}")
|
||||
sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; }
|
||||
case "${3-} ${4-}" in
|
||||
"--silent ") ;;
|
||||
"--jq .name") printf '%s\n' "$branch" ;;
|
||||
"--jq .commit.sha") printf '%s\n' "$sha" ;;
|
||||
*) echo "fake gh cannot answer: $*" >&2; exit 64 ;;
|
||||
esac ;;
|
||||
"api pulls/1/reviews --paginate --slurp")
|
||||
printf '[[]]\n' ;;
|
||||
*)
|
||||
echo "fake gh cannot answer: $*" >&2
|
||||
exit 64 ;;
|
||||
esac
|
||||
"""
|
||||
FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash
|
||||
printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv"
|
||||
"""
|
||||
|
||||
|
||||
class WorkflowStep(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str = ""
|
||||
name: str = ""
|
||||
uses: str = ""
|
||||
with_: Mapping[str, str] = Field(default_factory=dict, alias="with")
|
||||
env: Mapping[str, str] = Field(default_factory=dict)
|
||||
run: str = ""
|
||||
|
||||
|
||||
class RunDefaults(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
working_directory: str = Field(default=".", alias="working-directory")
|
||||
|
||||
|
||||
class JobDefaults(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
run: RunDefaults = Field(default_factory=RunDefaults)
|
||||
|
||||
|
||||
class WorkflowJob(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
defaults: JobDefaults = Field(default_factory=JobDefaults)
|
||||
steps: tuple[WorkflowStep, ...] = ()
|
||||
|
||||
|
||||
class PullRequestTrigger(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
branches: tuple[str, ...] = ("**",)
|
||||
|
||||
|
||||
class WorkflowTriggers(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger)
|
||||
|
||||
|
||||
class Workflow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
on: WorkflowTriggers
|
||||
jobs: Mapping[str, WorkflowJob]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MergeBaseStep:
|
||||
workflow: str
|
||||
job_id: str
|
||||
job: WorkflowJob
|
||||
step: WorkflowStep
|
||||
base_branches: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def job_key(self) -> str:
|
||||
return f"{self.workflow}:{self.job_id}"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StaleBaseUpstream:
|
||||
path: Path
|
||||
stale_base: str
|
||||
head: str
|
||||
tip_by_ref: Mapping[str, str]
|
||||
merge_base_by_ref: Mapping[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Runner:
|
||||
root: Path
|
||||
upstream: StaleBaseUpstream
|
||||
live_base_ref: str
|
||||
context: Mapping[str, str]
|
||||
|
||||
@property
|
||||
def checkout(self) -> Path:
|
||||
return self.root / "checkout"
|
||||
|
||||
@property
|
||||
def tools(self) -> Path:
|
||||
return self.root / "bin"
|
||||
|
||||
@property
|
||||
def temp(self) -> Path:
|
||||
return self.root / "runner_temp"
|
||||
|
||||
@property
|
||||
def github_output(self) -> Path:
|
||||
return self.root / "github_output"
|
||||
|
||||
@property
|
||||
def github_env(self) -> Path:
|
||||
return self.root / "github_env"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StepRun:
|
||||
step: MergeBaseStep
|
||||
runner: Runner
|
||||
|
||||
|
||||
Observation: TypeAlias = str | frozenset[str] | tuple[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Consumer:
|
||||
observe: Callable[[StepRun], Observation]
|
||||
expect: Callable[[StaleBaseUpstream, str], Observation]
|
||||
|
||||
|
||||
def _workflow(path: Path) -> Workflow:
|
||||
return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader))
|
||||
|
||||
|
||||
def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]:
|
||||
workflow: Final = _workflow(path)
|
||||
for job_id, job in workflow.jobs.items():
|
||||
yield from (
|
||||
MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches)
|
||||
for step in job.steps
|
||||
if "merge_base_commit" in step.run
|
||||
)
|
||||
|
||||
|
||||
MERGE_BASE_STEPS: Final = tuple(
|
||||
itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml"))))
|
||||
)
|
||||
|
||||
|
||||
def _git(cwd: Path, *args: str) -> str:
|
||||
|
|
@ -104,6 +283,218 @@ def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manife
|
|||
assert selected == frozenset({trigger})
|
||||
|
||||
|
||||
def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
"github.repository": REPOSITORY,
|
||||
"github.token": "fake-token",
|
||||
**{f"github.event.pull_request.{field}": value for field, value in pull_request.items()},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _context_value(context: Mapping[str, str], expression: str) -> str:
|
||||
if expression not in context:
|
||||
pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}")
|
||||
return context[expression]
|
||||
|
||||
|
||||
def _resolved(text: str, context: Mapping[str, str]) -> str:
|
||||
return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text)
|
||||
|
||||
|
||||
def _install_tool(path: Path, script: str) -> None:
|
||||
path.write_text(script)
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner:
|
||||
runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request))
|
||||
runner.tools.mkdir()
|
||||
runner.temp.mkdir()
|
||||
for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)):
|
||||
_install_tool(runner.tools / name, script)
|
||||
return runner
|
||||
|
||||
|
||||
def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None:
|
||||
checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@"))
|
||||
ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context)
|
||||
_git(runner.root, "init", "-q", str(runner.checkout))
|
||||
_git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}")
|
||||
_git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref)
|
||||
_git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD")
|
||||
|
||||
|
||||
def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None:
|
||||
result: Final = subprocess.run(
|
||||
["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)],
|
||||
cwd=runner.checkout / job.defaults.run.working_directory,
|
||||
env={
|
||||
**{name: _resolved(value, context) for name, value in step.env.items()},
|
||||
"PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}",
|
||||
"HOME": str(runner.root),
|
||||
"GITHUB_REPOSITORY": REPOSITORY,
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"GITHUB_OUTPUT": str(runner.github_output),
|
||||
"GITHUB_ENV": str(runner.github_env),
|
||||
"RUNNER_TEMP": str(runner.temp),
|
||||
"UPSTREAM": str(runner.upstream.path),
|
||||
"LIVE_BASE_REF": runner.live_base_ref,
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def _key_values(path: Path) -> Mapping[str, str]:
|
||||
return MappingProxyType(
|
||||
{key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())}
|
||||
)
|
||||
|
||||
|
||||
def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]:
|
||||
return tuple((runner.temp / f"{tool}.argv").read_text().splitlines())
|
||||
|
||||
|
||||
def _gate_base_exported(run: StepRun) -> str:
|
||||
return _key_values(run.runner.github_env)["GATE_BASE_SHA"]
|
||||
|
||||
|
||||
def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]:
|
||||
return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines())
|
||||
|
||||
|
||||
def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]:
|
||||
argv: Final = _argv_recorded(run.runner, "npm")
|
||||
return frozenset(argv[argv.index("related") + 1 : argv.index("--run")])
|
||||
|
||||
|
||||
def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]:
|
||||
guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard")
|
||||
outputs: Final = {
|
||||
f"steps.{run.step.step.id}.outputs.{name}": value
|
||||
for name, value in _key_values(run.runner.github_output).items()
|
||||
}
|
||||
_run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner)
|
||||
argv: Final = _argv_recorded(run.runner, "uv")
|
||||
return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD")
|
||||
|
||||
|
||||
def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str:
|
||||
return upstream.merge_base_by_ref[ref]
|
||||
|
||||
|
||||
def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]:
|
||||
merge_base: Final = upstream.merge_base_by_ref[ref]
|
||||
changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head)
|
||||
return frozenset(changed.split())
|
||||
|
||||
|
||||
def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]:
|
||||
return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref]
|
||||
|
||||
|
||||
CONSUMERS: Final = MappingProxyType(
|
||||
{
|
||||
"test-linting.yml:python": Consumer(_gate_base_exported, _merge_base),
|
||||
"test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base),
|
||||
"test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base),
|
||||
"spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
||||
"cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _sha(repo: Path, revision: str) -> str:
|
||||
return _git(repo, "rev-parse", revision).strip()
|
||||
|
||||
|
||||
def _commit_ui_file(repo: Path, name: str) -> None:
|
||||
target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(f"export const {name} = 1;\n")
|
||||
_git(repo, "add", "-A")
|
||||
_git(repo, "commit", "-qm", name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream:
|
||||
path: Final = tmp_path_factory.mktemp("upstream")
|
||||
_git(path, "init", "-q", "-b", "main")
|
||||
_git(path, "config", "user.email", "t@t")
|
||||
_git(path, "config", "user.name", "t")
|
||||
_commit_ui_file(path, "main_when_the_pull_request_was_opened")
|
||||
_git(path, "branch", "feature")
|
||||
_git(path, "branch", RELEASE_BASE_REF)
|
||||
_commit_ui_file(path, "main_since")
|
||||
_git(path, "checkout", "-q", "-b", STACKED_BASE_REF)
|
||||
_commit_ui_file(path, "stacked_base_since")
|
||||
_git(path, "checkout", "-q", "feature")
|
||||
_commit_ui_file(path, "pull_request_work")
|
||||
_git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF)
|
||||
_git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF)
|
||||
return StaleBaseUpstream(
|
||||
path=path,
|
||||
stale_base=_sha(path, "main~1"),
|
||||
head=_sha(path, "feature"),
|
||||
tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}),
|
||||
merge_base_by_ref=MappingProxyType(
|
||||
{ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None:
|
||||
assert MERGE_BASE_STEPS
|
||||
assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset(
|
||||
path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text()
|
||||
)
|
||||
|
||||
|
||||
def _runs_for(step: MergeBaseStep, base_ref: str) -> bool:
|
||||
return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches)
|
||||
|
||||
|
||||
BASE_REF_CASES: Final = (
|
||||
("main", "main", "main"),
|
||||
("stacked", STACKED_BASE_REF, STACKED_BASE_REF),
|
||||
("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"),
|
||||
("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF),
|
||||
("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF),
|
||||
("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF),
|
||||
)
|
||||
MERGE_BASE_CASES: Final = tuple(
|
||||
pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}")
|
||||
for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES)
|
||||
if _runs_for(step, event_base_ref)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES)
|
||||
def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone(
|
||||
upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str
|
||||
) -> None:
|
||||
pull_request: Final = MappingProxyType(
|
||||
{
|
||||
"number": "1",
|
||||
"base.ref": event_base_ref,
|
||||
"base.sha": upstream.stale_base,
|
||||
"head.sha": upstream.head,
|
||||
"head.ref": "feature",
|
||||
"user.type": "User",
|
||||
}
|
||||
)
|
||||
runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref)
|
||||
_check_out_like_the_job(step.job, runner)
|
||||
_run_step(step.step, step.job, runner.context, runner)
|
||||
consumer: Final = CONSUMERS[step.job_key]
|
||||
measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref
|
||||
assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref)
|
||||
|
||||
|
||||
UI_API_TYPES_DETECT: Final[str] = next(
|
||||
step["run"]
|
||||
for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"]
|
||||
|
|
|
|||
|
|
@ -18,8 +18,12 @@ that consumes it, running the step's real shell against stubbed `gh`, `git` and
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
|
@ -28,6 +32,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
|
|||
SCOPE_SCRIPT = REPO_ROOT / ".github" / "scripts" / "select_ui_test_scope.sh"
|
||||
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test-unit.yml"
|
||||
STEP_NAME = "Run UI unit tests (Vitest)"
|
||||
EXPRESSION = re.compile(r"\$\{\{\s*(.+?)\s*\}\}")
|
||||
PULL_REQUEST_EVENT: Final = MappingProxyType(
|
||||
{"number": "1", "base.ref": "main", "base.sha": "basesha", "head.sha": "headsha"}
|
||||
)
|
||||
|
||||
FULL_SUITE_ARGV = ["run", "test", "--", "--run", "--pool", "forks", "--maxWorkers=14"]
|
||||
|
||||
|
|
@ -73,13 +81,26 @@ def test_an_empty_change_set_fails_open_to_the_full_suite() -> None:
|
|||
assert scope([]) == "full"
|
||||
|
||||
|
||||
def _step_script() -> str:
|
||||
workflow = yaml.safe_load(WORKFLOW.read_text())
|
||||
steps = workflow["jobs"]["ui-unit"]["steps"]
|
||||
script = next(step["run"] for step in steps if step.get("name") == STEP_NAME)
|
||||
resolved = script.replace("${{ github.repository }}", "BerriAI/litellm")
|
||||
assert "${{" not in resolved, "the step uses an Actions expression this harness does not resolve"
|
||||
return resolved
|
||||
def _resolve(text: str, pull_request: Mapping[str, str] | None) -> str:
|
||||
def value(match: re.Match[str]) -> str:
|
||||
expression: Final = match.group(1)
|
||||
if expression == "github.repository":
|
||||
return "BerriAI/litellm"
|
||||
if expression == "github.token":
|
||||
return "token"
|
||||
field: Final = expression.removeprefix("github.event.pull_request.")
|
||||
assert field != expression, f"the step uses ${{{{ {expression} }}}}, which this harness does not resolve"
|
||||
return "" if pull_request is None else pull_request[field]
|
||||
|
||||
return EXPRESSION.sub(value, text)
|
||||
|
||||
|
||||
def _step_script_and_env(pull_request: Mapping[str, str] | None) -> tuple[str, dict[str, str]]:
|
||||
workflow: Final = yaml.safe_load(WORKFLOW.read_text())
|
||||
steps: Final = workflow["jobs"]["ui-unit"]["steps"]
|
||||
step: Final = next(step for step in steps if step.get("name") == STEP_NAME)
|
||||
env: Final = {name: _resolve(str(value), pull_request) for name, value in step.get("env", {}).items()}
|
||||
return _resolve(step["run"], pull_request), env
|
||||
|
||||
|
||||
def _stub(bin_dir: Path, name: str, body: str) -> None:
|
||||
|
|
@ -88,7 +109,9 @@ def _stub(bin_dir: Path, name: str, body: str) -> None:
|
|||
stub.chmod(0o755)
|
||||
|
||||
|
||||
def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> tuple[int, str, list[list[str]]]:
|
||||
def _run_step(
|
||||
tmp_path: Path, changed: list[str], pull_request: Mapping[str, str] | None = PULL_REQUEST_EVENT
|
||||
) -> tuple[int, str, list[list[str]]]:
|
||||
"""Run the workflow step's real shell; return its status, stdout and every npm argv."""
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
|
|
@ -100,20 +123,22 @@ def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") ->
|
|||
_stub(bin_dir, "git", 'if [ "$1" = diff ]; then cat "$CHANGED_FILES"; fi')
|
||||
_stub(bin_dir, "npm", 'printf "%s\\n" "$@" >>"$NPM_LOG"; printf "\\0" >>"$NPM_LOG"')
|
||||
|
||||
step = tmp_path / "step.sh"
|
||||
step.write_text(_step_script())
|
||||
step_script, step_env = _step_script_and_env(pull_request)
|
||||
script: Final = tmp_path / "step.sh"
|
||||
script.write_text(step_script)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
|
||||
env["BASE_SHA"] = base_sha
|
||||
env["HEAD_SHA"] = "headsha"
|
||||
env["GITHUB_WORKSPACE"] = str(REPO_ROOT)
|
||||
env["GITHUB_REF_NAME"] = "release_branch"
|
||||
env["CHANGED_FILES"] = str(changed_file)
|
||||
env["NPM_LOG"] = str(npm_log)
|
||||
env: Final = {
|
||||
**os.environ,
|
||||
**step_env,
|
||||
"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
||||
"GITHUB_REF_NAME": "release_branch",
|
||||
"CHANGED_FILES": str(changed_file),
|
||||
"NPM_LOG": str(npm_log),
|
||||
}
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", "--noprofile", "--norc", "-eo", "pipefail", str(step)],
|
||||
["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)],
|
||||
cwd=tmp_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
@ -155,7 +180,7 @@ def test_step_runs_the_full_suite_when_a_changed_file_is_outside_src(tmp_path: P
|
|||
|
||||
|
||||
def test_step_runs_the_full_suite_on_a_push(tmp_path: Path) -> None:
|
||||
returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], base_sha="")
|
||||
returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], pull_request=None)
|
||||
assert returncode == 0, output
|
||||
assert invocations == [FULL_SUITE_ARGV]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue