mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-08 08:08:37 +00:00
d29c515444
* ci(unit): fetch the merge base by sha for the db-and-spend schema check instead of cloning full history * ci(unit): read the schema check arguments through one typed mapping * ci(unit): give the schema check its own step with the ui-unit base fallback and register it with the merge-base step test * ci(unit): keep the setup ceiling at 35 minutes, the install step never took over 33 seconds --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
604 lines
22 KiB
Python
604 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import fnmatch
|
|
import itertools
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
from collections.abc import Callable, Iterator, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from types import MappingProxyType
|
|
from typing import Final, TypeAlias
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml"
|
|
WORKFLOWS_DIR: Final = WORKFLOW.parent
|
|
REPO_ROOT: Final = WORKFLOW.parents[2]
|
|
DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|')
|
|
GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text()))
|
|
PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)"))
|
|
REPOSITORY: Final = "BerriAI/litellm"
|
|
UI_DASHBOARD: Final = "ui/litellm-dashboard"
|
|
STACKED_BASE_REF: Final = "litellm_/stacked"
|
|
RELEASE_BASE_REF: Final = "stable/next"
|
|
BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF)
|
|
RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun"
|
|
GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}")
|
|
FAKE_GH: Final = r"""#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
endpoint=${2%%\?*}
|
|
endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"}
|
|
commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; }
|
|
renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; }
|
|
case "$1 $endpoint ${3-} ${4-}" in
|
|
"api compare/"*"..."*" --jq .merge_base_commit.sha")
|
|
basehead=${endpoint#compare/}
|
|
git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;;
|
|
"api pulls/1 --jq .base.ref")
|
|
printf '%s\n' "$LIVE_BASE_REF" ;;
|
|
"api branches/"*)
|
|
branch=$(renamed "${endpoint#branches/}")
|
|
sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; }
|
|
case "${3-} ${4-}" in
|
|
"--silent ") ;;
|
|
"--jq .name") printf '%s\n' "$branch" ;;
|
|
"--jq .commit.sha") printf '%s\n' "$sha" ;;
|
|
*) echo "fake gh cannot answer: $*" >&2; exit 64 ;;
|
|
esac ;;
|
|
"api pulls/1/reviews --paginate --slurp")
|
|
printf '[[]]\n' ;;
|
|
*)
|
|
echo "fake gh cannot answer: $*" >&2
|
|
exit 64 ;;
|
|
esac
|
|
"""
|
|
FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash
|
|
printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv"
|
|
"""
|
|
|
|
|
|
class WorkflowStep(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
id: str = ""
|
|
name: str = ""
|
|
uses: str = ""
|
|
with_: Mapping[str, str] = Field(default_factory=dict, alias="with")
|
|
env: Mapping[str, str] = Field(default_factory=dict)
|
|
run: str = ""
|
|
|
|
|
|
class RunDefaults(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
working_directory: str = Field(default=".", alias="working-directory")
|
|
|
|
|
|
class JobDefaults(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
run: RunDefaults = Field(default_factory=RunDefaults)
|
|
|
|
|
|
class WorkflowJob(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
defaults: JobDefaults = Field(default_factory=JobDefaults)
|
|
steps: tuple[WorkflowStep, ...] = ()
|
|
|
|
|
|
class PullRequestTrigger(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
branches: tuple[str, ...] = ("**",)
|
|
|
|
|
|
class WorkflowTriggers(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger)
|
|
|
|
|
|
class Workflow(BaseModel):
|
|
model_config = ConfigDict(frozen=True)
|
|
|
|
on: WorkflowTriggers
|
|
jobs: Mapping[str, WorkflowJob]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class MergeBaseStep:
|
|
workflow: str
|
|
job_id: str
|
|
job: WorkflowJob
|
|
step: WorkflowStep
|
|
base_branches: tuple[str, ...]
|
|
|
|
@property
|
|
def job_key(self) -> str:
|
|
return f"{self.workflow}:{self.job_id}"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class StaleBaseUpstream:
|
|
path: Path
|
|
stale_base: str
|
|
head: str
|
|
tip_by_ref: Mapping[str, str]
|
|
merge_base_by_ref: Mapping[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Runner:
|
|
root: Path
|
|
upstream: StaleBaseUpstream
|
|
live_base_ref: str
|
|
context: Mapping[str, str]
|
|
|
|
@property
|
|
def checkout(self) -> Path:
|
|
return self.root / "checkout"
|
|
|
|
@property
|
|
def tools(self) -> Path:
|
|
return self.root / "bin"
|
|
|
|
@property
|
|
def temp(self) -> Path:
|
|
return self.root / "runner_temp"
|
|
|
|
@property
|
|
def github_output(self) -> Path:
|
|
return self.root / "github_output"
|
|
|
|
@property
|
|
def github_env(self) -> Path:
|
|
return self.root / "github_env"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class StepRun:
|
|
step: MergeBaseStep
|
|
runner: Runner
|
|
|
|
|
|
Observation: TypeAlias = str | frozenset[str] | tuple[str, str]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Consumer:
|
|
observe: Callable[[StepRun], Observation]
|
|
expect: Callable[[StaleBaseUpstream, str], Observation]
|
|
|
|
|
|
def _workflow(path: Path) -> Workflow:
|
|
return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader))
|
|
|
|
|
|
def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]:
|
|
workflow: Final = _workflow(path)
|
|
for job_id, job in workflow.jobs.items():
|
|
yield from (
|
|
MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches)
|
|
for step in job.steps
|
|
if "merge_base_commit" in step.run
|
|
)
|
|
|
|
|
|
MERGE_BASE_STEPS: Final = tuple(
|
|
itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml"))))
|
|
)
|
|
|
|
|
|
def _git(cwd: Path, *args: str) -> str:
|
|
return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def _scoped_root(pathspec: str) -> str:
|
|
return re.sub(r"^:\([^)]*\)", "", pathspec).split("*", 1)[0]
|
|
|
|
|
|
def _gate_rooted_at(root: str) -> tuple[str, ...]:
|
|
return next(gate for gate in GATES if _scoped_root(gate[0]) == root)
|
|
|
|
|
|
def _changed_files_selected_by(tmp_path: Path, pathspecs: tuple[str, ...], files: tuple[str, ...]) -> frozenset[str]:
|
|
_git(tmp_path, "init", "-q", "-b", "main")
|
|
_git(tmp_path, "config", "user.email", "t@t")
|
|
_git(tmp_path, "config", "user.name", "t")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base")
|
|
for name in files:
|
|
target = tmp_path / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text("x = 1\n")
|
|
_git(tmp_path, "add", "-A")
|
|
_git(tmp_path, "commit", "-qm", "change")
|
|
return frozenset(
|
|
_git(tmp_path, "diff", "--name-only", "--diff-filter=ACMRD", "HEAD~1", "HEAD", "--", *pathspecs).split()
|
|
)
|
|
|
|
|
|
def test_workflow_still_carries_the_ruff_format_e2e_basedpyright_and_e2e_harness_diff_gates() -> None:
|
|
assert len(GATES) == 3
|
|
assert frozenset(gate[0] for gate in GATES) == frozenset(
|
|
{":(glob)litellm/**/*.py", ":(glob)tests/e2e/**/*.py", "tests/e2e"}
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("pathspecs", PYTHON_GATES, ids=" ".join)
|
|
def test_diff_gate_selects_top_level_and_nested_python_files_only(tmp_path: Path, pathspecs: tuple[str, ...]) -> None:
|
|
root = _scoped_root(pathspecs[0])
|
|
top_level = f"{root}top_level_module.py"
|
|
nested = f"{root}pkg/sub/nested_module.py"
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
pathspecs,
|
|
(top_level, nested, f"{root}notes.md", "elsewhere/top_level_module.py", "elsewhere/pkg/nested_module.py"),
|
|
)
|
|
assert selected == frozenset({top_level, nested})
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"trigger",
|
|
(
|
|
"tests/e2e_harness/top_level_module.py",
|
|
"tests/e2e_harness/pkg/sub/nested_module.py",
|
|
"pyrightconfig.json",
|
|
),
|
|
)
|
|
def test_e2e_basedpyright_gate_also_fires_on_harness_python_and_pyrightconfig(tmp_path: Path, trigger: str) -> None:
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
_gate_rooted_at("tests/e2e/"),
|
|
(trigger, "tests/e2e_harness/notes.md", "elsewhere/pyrightconfig.json", "tests/e2e_harnessish/module.py"),
|
|
)
|
|
assert selected == frozenset({trigger})
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"trigger",
|
|
(
|
|
"tests/e2e/claude_code/cron_vm/install_claude_code.sh",
|
|
"tests/e2e/notes.md",
|
|
"tests/e2e/pkg/sub/nested_module.py",
|
|
"tests/e2e_harness/claude_code/test_driver.py",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
".github/workflows/test-linting.yml",
|
|
),
|
|
)
|
|
def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manifests_and_workflow(
|
|
tmp_path: Path, trigger: str
|
|
) -> None:
|
|
selected = _changed_files_selected_by(
|
|
tmp_path,
|
|
_gate_rooted_at("tests/e2e"),
|
|
(trigger, "tests/e2e/ui/spec.ts", "tests/e2e/ui/pkg/page.py", "elsewhere/pyproject.toml", "tests/e2e_other/module.py"),
|
|
)
|
|
assert selected == frozenset({trigger})
|
|
|
|
|
|
def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]:
|
|
return MappingProxyType(
|
|
{
|
|
"github.repository": REPOSITORY,
|
|
"github.token": "fake-token",
|
|
**{f"github.event.pull_request.{field}": value for field, value in pull_request.items()},
|
|
}
|
|
)
|
|
|
|
|
|
def _context_value(context: Mapping[str, str], expression: str) -> str:
|
|
if expression not in context:
|
|
pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}")
|
|
return context[expression]
|
|
|
|
|
|
def _resolved(text: str, context: Mapping[str, str]) -> str:
|
|
return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text)
|
|
|
|
|
|
def _install_tool(path: Path, script: str) -> None:
|
|
path.write_text(script)
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner:
|
|
runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request))
|
|
runner.tools.mkdir()
|
|
runner.temp.mkdir()
|
|
for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)):
|
|
_install_tool(runner.tools / name, script)
|
|
return runner
|
|
|
|
|
|
def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None:
|
|
checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@"))
|
|
ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context)
|
|
_git(runner.root, "init", "-q", str(runner.checkout))
|
|
_git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}")
|
|
_git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref)
|
|
_git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD")
|
|
|
|
|
|
def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None:
|
|
result: Final = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)],
|
|
cwd=runner.checkout / job.defaults.run.working_directory,
|
|
env={
|
|
**{name: _resolved(value, context) for name, value in step.env.items()},
|
|
"PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}",
|
|
"HOME": str(runner.root),
|
|
"GITHUB_REPOSITORY": REPOSITORY,
|
|
"GITHUB_WORKSPACE": str(REPO_ROOT),
|
|
"GITHUB_OUTPUT": str(runner.github_output),
|
|
"GITHUB_ENV": str(runner.github_env),
|
|
"RUNNER_TEMP": str(runner.temp),
|
|
"UPSTREAM": str(runner.upstream.path),
|
|
"LIVE_BASE_REF": runner.live_base_ref,
|
|
},
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def _key_values(path: Path) -> Mapping[str, str]:
|
|
return MappingProxyType(
|
|
{key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())}
|
|
)
|
|
|
|
|
|
def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]:
|
|
return tuple((runner.temp / f"{tool}.argv").read_text().splitlines())
|
|
|
|
|
|
def _gate_base_exported(run: StepRun) -> str:
|
|
return _key_values(run.runner.github_env)["GATE_BASE_SHA"]
|
|
|
|
|
|
def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]:
|
|
return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines())
|
|
|
|
|
|
def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]:
|
|
argv: Final = _argv_recorded(run.runner, "npm")
|
|
return frozenset(argv[argv.index("related") + 1 : argv.index("--run")])
|
|
|
|
|
|
def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]:
|
|
guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard")
|
|
outputs: Final = {
|
|
f"steps.{run.step.step.id}.outputs.{name}": value
|
|
for name, value in _key_values(run.runner.github_output).items()
|
|
}
|
|
_run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner)
|
|
argv: Final = _argv_recorded(run.runner, "uv")
|
|
return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD")
|
|
|
|
|
|
def _schema_base_commit_and_its_type_in_the_checkout(run: StepRun) -> tuple[str, str]:
|
|
argv: Final = _argv_recorded(run.runner, "uv")
|
|
base_commit: Final = argv[argv.index("--base-commit") + 1]
|
|
return base_commit, _git(run.runner.checkout, "cat-file", "-t", base_commit).strip()
|
|
|
|
|
|
def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str:
|
|
return upstream.merge_base_by_ref[ref]
|
|
|
|
|
|
def _merge_base_as_a_commit(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]:
|
|
return upstream.merge_base_by_ref[ref], "commit"
|
|
|
|
|
|
def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]:
|
|
merge_base: Final = upstream.merge_base_by_ref[ref]
|
|
changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head)
|
|
return frozenset(changed.split())
|
|
|
|
|
|
def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]:
|
|
return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref]
|
|
|
|
|
|
CONSUMERS: Final = MappingProxyType(
|
|
{
|
|
"test-linting.yml:python": Consumer(_gate_base_exported, _merge_base),
|
|
"test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base),
|
|
"test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base),
|
|
"test-unit.yml:unit": Consumer(_schema_base_commit_and_its_type_in_the_checkout, _merge_base_as_a_commit),
|
|
"spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
|
"cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip),
|
|
}
|
|
)
|
|
|
|
|
|
def _sha(repo: Path, revision: str) -> str:
|
|
return _git(repo, "rev-parse", revision).strip()
|
|
|
|
|
|
def _commit_ui_file(repo: Path, name: str) -> None:
|
|
target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts"
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(f"export const {name} = 1;\n")
|
|
_git(repo, "add", "-A")
|
|
_git(repo, "commit", "-qm", name)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream:
|
|
path: Final = tmp_path_factory.mktemp("upstream")
|
|
_git(path, "init", "-q", "-b", "main")
|
|
_git(path, "config", "user.email", "t@t")
|
|
_git(path, "config", "user.name", "t")
|
|
_commit_ui_file(path, "main_when_the_pull_request_was_opened")
|
|
_git(path, "branch", "feature")
|
|
_git(path, "branch", RELEASE_BASE_REF)
|
|
_commit_ui_file(path, "main_since")
|
|
_git(path, "checkout", "-q", "-b", STACKED_BASE_REF)
|
|
_commit_ui_file(path, "stacked_base_since")
|
|
_git(path, "checkout", "-q", "feature")
|
|
_commit_ui_file(path, "pull_request_work")
|
|
_git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF)
|
|
_git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF)
|
|
return StaleBaseUpstream(
|
|
path=path,
|
|
stale_base=_sha(path, "main~1"),
|
|
head=_sha(path, "feature"),
|
|
tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}),
|
|
merge_base_by_ref=MappingProxyType(
|
|
{ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS}
|
|
),
|
|
)
|
|
|
|
|
|
def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None:
|
|
assert MERGE_BASE_STEPS
|
|
assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset(
|
|
path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text()
|
|
)
|
|
|
|
|
|
def _runs_for(step: MergeBaseStep, base_ref: str) -> bool:
|
|
return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches)
|
|
|
|
|
|
BASE_REF_CASES: Final = (
|
|
("main", "main", "main"),
|
|
("stacked", STACKED_BASE_REF, STACKED_BASE_REF),
|
|
("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"),
|
|
("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF),
|
|
("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF),
|
|
("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF),
|
|
)
|
|
MERGE_BASE_CASES: Final = tuple(
|
|
pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}")
|
|
for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES)
|
|
if _runs_for(step, event_base_ref)
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES)
|
|
def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone(
|
|
upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str
|
|
) -> None:
|
|
pull_request: Final = MappingProxyType(
|
|
{
|
|
"number": "1",
|
|
"base.ref": event_base_ref,
|
|
"base.sha": upstream.stale_base,
|
|
"head.sha": upstream.head,
|
|
"head.ref": "feature",
|
|
"user.type": "User",
|
|
}
|
|
)
|
|
runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref)
|
|
_check_out_like_the_job(step.job, runner)
|
|
_run_step(step.step, step.job, runner.context, runner)
|
|
consumer: Final = CONSUMERS[step.job_key]
|
|
measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref
|
|
assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref)
|
|
|
|
|
|
UI_API_TYPES_DETECT: Final[str] = next(
|
|
step["run"]
|
|
for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"]
|
|
if step.get("id") == "changes"
|
|
)
|
|
PRE_COMMIT_LINT: Final = WORKFLOW.parents[2] / "scripts" / "pre_commit_lint.sh"
|
|
TERRAFORM_PROVIDER_WORKFLOW: Final = WORKFLOW.with_name("test-terraform-provider.yml")
|
|
|
|
|
|
def _ui_api_types_relevant(tmp_path: Path, changed: str) -> bool:
|
|
_git(tmp_path, "init", "-q", "-b", "main")
|
|
_git(tmp_path, "config", "user.email", "t@t")
|
|
_git(tmp_path, "config", "user.name", "t")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "base")
|
|
_git(tmp_path, "switch", "-q", "-c", "pr")
|
|
target: Final = tmp_path / changed
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text("x = 1\n")
|
|
_git(tmp_path, "add", "-A")
|
|
_git(tmp_path, "commit", "-qm", "change")
|
|
_git(tmp_path, "switch", "-q", "main")
|
|
_git(tmp_path, "commit", "-q", "--allow-empty", "-m", "main moved")
|
|
_git(tmp_path, "merge", "-q", "--no-ff", "--no-edit", "pr")
|
|
output: Final = tmp_path / "github_output"
|
|
subprocess.run(
|
|
["bash", "-c", UI_API_TYPES_DETECT],
|
|
cwd=tmp_path,
|
|
check=True,
|
|
capture_output=True,
|
|
env={"PATH": "/usr/bin:/bin", "GITHUB_OUTPUT": str(output)},
|
|
)
|
|
return output.read_text().strip() == "relevant=true"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
(
|
|
"litellm/proxy/proxy_server.py",
|
|
"litellm/types/router.py",
|
|
"litellm/tracing/generated/models.py",
|
|
"litellm/router_strategy/complexity_router/config.py",
|
|
"litellm/models/credentials.py",
|
|
"enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
"ui/litellm-dashboard/scripts/gen-api-types.mjs",
|
|
"ui/litellm-dashboard/package-lock.json",
|
|
"ui/litellm-dashboard/src/lib/http/schema.d.ts",
|
|
),
|
|
)
|
|
def test_ui_api_types_check_runs_on_a_pull_request_touching_any_spec_input(tmp_path: Path, changed: str) -> None:
|
|
assert _ui_api_types_relevant(tmp_path, changed)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
(
|
|
"litellm-rust/crates/traces/src/lib.rs",
|
|
"litellm-proxy-extras/litellm_proxy_extras/utils.py",
|
|
"tests/unit/test_router.py",
|
|
"ui/litellm-dashboard/src/app/page.tsx",
|
|
"docs/my-website/docs/proxy/config.md",
|
|
"litellm-dashboard/pyproject.toml",
|
|
),
|
|
)
|
|
def test_ui_api_types_check_skips_a_pull_request_that_cannot_change_the_spec(tmp_path: Path, changed: str) -> None:
|
|
assert not _ui_api_types_relevant(tmp_path, changed)
|
|
|
|
|
|
def _first_single_quoted_capture(pattern: str, text: str) -> str:
|
|
match: Final = re.search(pattern, text, re.MULTILINE)
|
|
assert match is not None, pattern
|
|
return match.group(1)
|
|
|
|
|
|
def _tracked_paths_matching(extended_regex: str) -> frozenset[str]:
|
|
tracked: Final = _git(WORKFLOW.parents[2], "ls-files")
|
|
selected: Final = subprocess.run(["grep", "-E", extended_regex], input=tracked, capture_output=True, text=True)
|
|
return frozenset(selected.stdout.splitlines())
|
|
|
|
|
|
def test_make_check_runs_the_api_type_sync_on_the_same_tracked_files_as_ci() -> None:
|
|
ci_trigger: Final = _first_single_quoted_capture(r"grep -Eq '([^']+)'", UI_API_TYPES_DETECT)
|
|
make_check_trigger: Final = _first_single_quoted_capture(r"^spec_pattern='([^']+)'$", PRE_COMMIT_LINT.read_text())
|
|
ci_selected: Final = _tracked_paths_matching(ci_trigger)
|
|
assert "pyproject.toml" in ci_selected
|
|
assert _tracked_paths_matching(make_check_trigger) == ci_selected - {".github/workflows/test-linting.yml"}
|
|
|
|
|
|
def test_a_pull_request_changing_any_route_file_runs_the_terraform_provider_endpoint_audit() -> None:
|
|
triggers: Final = yaml.safe_load(TERRAFORM_PROVIDER_WORKFLOW.read_text())[True]["pull_request"]["paths"]
|
|
route_files: Final = _git(
|
|
WORKFLOW.parents[2], "grep", "-l", "-E", r"APIRouter\(", "--", "*.py", ":(exclude)tests/**"
|
|
).splitlines()
|
|
assert route_files
|
|
assert [name for name in route_files if not any(fnmatch.fnmatch(name, pattern) for pattern in triggers)] == []
|