diff --git a/.github/scripts/check_pr_body.py b/.github/scripts/check_pr_body.py index 70881e69bcf..3ac6cd1b28d 100644 --- a/.github/scripts/check_pr_body.py +++ b/.github/scripts/check_pr_body.py @@ -22,6 +22,7 @@ E2E_PREFIX: Final = "tests/e2e/" PLACEHOLDER_TOKENS: Final = ("",) NO_CAVEATS_PATTERN: Final = re.compile(r"^(none|n/?a)[.!]?$", re.IGNORECASE) HTML_COMMENT_PATTERN: Final = re.compile(r"", re.DOTALL) +FENCE_PATTERN: Final = re.compile(r"^\s{0,3}(?:```|~~~)") HEADING_PATTERN: Final = re.compile(r"^#{2,6}\s+(?P.+?)\s*$") BULLET_PATTERN: Final = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+(?P<text>.*)$") LABEL_PATTERN: Final = re.compile(r"^[^-*+].*:\s*$") @@ -46,6 +47,17 @@ def strip_html_comments(body: str) -> str: return HTML_COMMENT_PATTERN.sub("", body.replace("\r\n", "\n")) +def mask_fenced_blocks(body: str) -> str: + def step(acc: tuple[tuple[str, ...], bool], line: str) -> tuple[tuple[str, ...], bool]: + lines, in_fence = acc + if FENCE_PATTERN.match(line): + return (*lines, ""), not in_fence + return (*lines, "" if in_fence else line), in_fence + + masked, _ = reduce(step, body.split("\n"), ((), False)) + return "\n".join(masked) + + def split_sections(body: str) -> tuple[Section, ...]: lines: Final = tuple(body.split("\n")) headings: Final = tuple( @@ -119,7 +131,7 @@ def check_qa_runbook(sections: tuple[Section, ...], changed_files: tuple[str, .. def check_body(body: str, changed_files: tuple[str, ...]) -> tuple[Violation, ...]: - sections: Final = split_sections(strip_html_comments(body)) + sections: Final = split_sections(mask_fenced_blocks(strip_html_comments(body))) bullet_violations: Final = tuple( violation for section in sections diff --git a/tests/test_litellm/test_github_check_pr_body.py b/tests/test_litellm/test_github_check_pr_body.py index edf3587e42f..245668aeaca 100644 --- a/tests/test_litellm/test_github_check_pr_body.py +++ b/tests/test_litellm/test_github_check_pr_body.py @@ -104,6 +104,25 @@ def test_empty_body_passes(checker): assert checker.check_body("", ()) == () +def test_ellipsis_inside_code_fence_is_not_a_placeholder(checker): + body = "## Screenshots / Proof of Fix\n\n```\n$ curl http://localhost:4000/health\n...\n{\"status\": \"ok\"}\n```\n" + assert checker.check_body(body, ()) == () + + +def test_headings_inside_code_fence_do_not_split_sections(checker): + body = ( + "## Screenshots / Proof of Fix\n\nThe old body looked like this:\n\n```\n## TLDR\n\n- <blah>\n- ...\n\n" + "## Caveats (if any)\n\nA long prose paragraph that would fail the bullet rule if parsed.\n```\n" + ) + assert checker.check_body(body, ()) == () + + +def test_ellipsis_outside_fences_still_fails(checker): + violations = checker.check_body("## TLDR\n\nProblem this solves:\n\n- ...\n", ()) + assert len(violations) == 1 + assert "placeholder" in violations[0].detail + + def test_multiline_html_comment_spanning_section_is_stripped(checker): body = "## QA runbook\n\n<!-- Only needed when your PR edits tests/e2e; delete this section otherwise\n\nExample:\n\n- step one\n-->\n" violations = checker.check_body(body, ("litellm/main.py",))