scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102
litellm/tests/test_litellm/test_github_review_gate.py
mateo-berri 13f346232a
feat: staged rollout mechanics for Agent Shin 
Adds the pieces needed for the 3-stage public rollout:

- red 'not ready' label paired with 'ready for review'; the review gate
  reconciles the pair on every run and reconsider flips it on reopen
- lite mode (AGENT_SHIN_MODE=lite): the review gate never closes and
  instead posts a one-time 'closes start in 7 days' notice linking the
  policy blog post (AGENT_SHIN_POLICY_URL); labels still reconcile
- stricter reconsider for closed PRs: requires a Greptile confidence
  score of at least 4/5 (missing score fails with a comment pointing at
  @greptileai) and bypasses the linked-issue short-circuit so the QA
  evidence rubric always applies; a pass reopens and tags ready
- reconsider on an OPEN PR now runs the review gate so the label pair
  flips immediately instead of waiting for the daily sweep
- PR rubric tightened: bug fixes need before AND after evidence; command
  proof from a custom script must include the script source (collapsible
  section or linked gist)
- default triage model bumped to gpt-5.6-luna
2026-07-13 01:51:09 +00:00

722 lines
27 KiB
Python
Raw Permalink Blame History

"""Unit tests for the `ready for review` label lifecycle (Agent Shin review gate).
Exercises `triage_with_llm.review_gate`, the state machine that keeps the
`ready for review` label in sync with whether a PR clears both the LLM rubric
and Greptile's confidence score:
* pass (untagged) -> add label + "ready for review" comment
* pass (untagged, recovered) -> add label + "all clear again" comment
* pass (already tagged) -> noop
* regress (tagged) -> remove label + "what's missing" comment, stays open
* fail (untagged, within 24h)-> one-time "what's missing" notice
* fail (untagged, >24h) -> close + comment
* dry run (close=False) -> would-* previews, no side effects
"""
from __future__ import annotations
import datetime as dt
import importlib.util
import sys
from pathlib import Path
import pytest
SCRIPT_PATH = (
Path(__file__).resolve().parents[2] / ".github" / "scripts" / "triage_with_llm.py"
)
NOW = dt.datetime(2026, 5, 24, 12, 0, 0, tzinfo=dt.timezone.utc)
JUST_NOW = "2026-05-24T11:00:00Z" # 1h old -> within 24h grace
TWO_DAYS_AGO = "2026-05-22T11:00:00Z" # >24h old -> past grace
@pytest.fixture(scope="module")
def triage_module():
spec = importlib.util.spec_from_file_location("triage_with_llm", SCRIPT_PATH)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules["triage_with_llm"] = module
spec.loader.exec_module(module)
return module
class _Recorder:
"""Captures every gh mutation review_gate could fire, and fails loudly
on the ones a given scenario forbids."""
def __init__(self, triage_module, monkeypatch):
self.comments: list[str] = []
self.added: list[str] = []
self.removed: list[str] = []
self.closed: list[int] = []
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: self.comments.append(body),
)
monkeypatch.setattr(
triage_module,
"add_label",
lambda repo, n, label: self.added.append(label),
)
monkeypatch.setattr(
triage_module,
"remove_label",
lambda repo, n, label: self.removed.append(label),
)
monkeypatch.setattr(
triage_module,
"close_pr",
lambda repo, n: self.closed.append(n),
)
def _make_pr(**overrides):
base = {
"number": 7,
"title": "feat: do a thing",
"body": "some body without a linked issue or QA proof",
"state": "open",
"author_association": "NONE",
"user": {"login": "mateo-berri"},
"labels": [],
"created_at": JUST_NOW,
}
base.update(overrides)
return base
def _pass(prompt):
return '{"verdict": "pass", "missing": [], "explanation": "looks good"}'
def _fail(prompt):
return (
'{"verdict": "fail", "missing": ["QA proof", "expected vs. actual"],'
' "explanation": "thin description"}'
)
def _gate(triage_module, **kwargs):
"""Call review_gate with safe defaults for the injectable hooks."""
params = dict(
repo="o/r",
number=7,
close=True,
model="m",
judge=_pass,
greptile_score=None,
comments=[],
now=NOW,
)
params.update(kwargs)
return triage_module.review_gate(**params)
class TestReviewGatePass:
def test_pass_untagged_adds_label_and_ready_comment(
self, triage_module, monkeypatch
):
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
assert rec.removed == [] and rec.closed == []
assert len(rec.comments) == 1
assert "ready for review" in rec.comments[0].lower()
assert triage_module.READY_MARKER in rec.comments[0]
assert "5/5" in rec.comments[0]
def test_pass_already_tagged_is_noop(self, triage_module, monkeypatch):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "noop-passing"
assert rec.added == [] and rec.removed == [] and rec.comments == []
def test_pass_after_prior_regression_uses_all_clear_wording(
self, triage_module, monkeypatch
):
# A regression marker in history -> this is a recovery, not a first pass.
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
}
]
result = _gate(triage_module, judge=_pass, greptile_score=5, comments=prior)
assert result["action"] == "labeled-ready"
assert "all clear" in rec.comments[0].lower()
def test_linked_issue_passes_without_calling_judge(
self, triage_module, monkeypatch
):
pr = _make_pr(body="Fixes #4321\n\nbody")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("LLM must not be called for linked issue"),
greptile_score=5,
)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
class TestReviewGateRegression:
def test_regression_removes_label_and_keeps_pr_open(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=5)
assert result["action"] == "label-removed-regressed"
assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL]
assert rec.closed == [] # regression NEVER closes the PR
assert triage_module.REGRESSED_MARKER in rec.comments[0]
assert "QA proof" in rec.comments[0]
# The state machine closes a still-failing PR `grace_days` after this
# notice (default 24h); the comment must disclose that deadline rather
# than implying the PR stays open indefinitely.
assert "24 hours" in rec.comments[0]
assert "auto-closed" in rec.comments[0]
def test_regression_comment_discloses_grace_deadline(self, triage_module):
one_day = triage_module.format_regression_comment(
["QA proof"], "needs work", grace_days=1
)
assert "24 hours" in one_day
assert "auto-closed" in one_day
three_days = triage_module.format_regression_comment(
["QA proof"], "needs work", grace_days=3
)
assert "3 days" in three_days
assert "auto-closed" in three_days
def test_greptile_drop_alone_triggers_regression(self, triage_module, monkeypatch):
# Rubric still passes, but Greptile fell to 2/5 -> not passing.
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=2)
assert result["action"] == "label-removed-regressed"
assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL]
assert "2/5" in rec.comments[0]
def test_greptile_score_read_from_comments_when_not_injected(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
greptile = [
{
"user": {"login": "greptile-apps[bot]"},
"body": "Confidence Score: 2/5",
"created_at": "2026-05-24T10:00:00Z",
}
]
result = _gate(
triage_module,
judge=_pass,
greptile_score=triage_module._UNSET,
comments=greptile,
)
assert result["action"] == "label-removed-regressed"
assert "2/5" in rec.comments[0]
class TestReviewGateGraceAndClose:
def test_within_grace_posts_one_time_notice(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None)
assert result["action"] == "within-grace-notified"
assert rec.closed == [] and rec.removed == []
assert rec.added == [triage_module.NOT_READY_LABEL]
assert triage_module.WITHIN_GRACE_MARKER in rec.comments[0]
assert "QA proof" in rec.comments[0]
def test_within_grace_does_not_double_notify(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.WITHIN_GRACE_MARKER,
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "within-grace-already-notified"
assert rec.comments == []
def test_past_grace_closes_with_comment(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None)
assert result["action"] == "closed"
assert rec.closed == [7]
assert len(rec.comments) == 1
# The close comment must carry the reconsider provenance marker so
# `was_closed_by_agent_shin` can later recognize this as an Agent Shin
# close (and not some other workflow's `github-actions[bot]` close).
assert triage_module.AGENT_SHIN_CLOSE_MARKER in rec.comments[0]
def test_recent_regression_marker_blocks_close(self, triage_module, monkeypatch):
"""A failing PR with a fresh regression notice must NOT be closed —
the contributor needs a window to address the regression."""
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
# Posted just an hour before NOW -> well inside grace_days.
"created_at": "2026-05-24T11:00:00Z",
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "regressed-already-notified"
assert rec.closed == [] and rec.comments == []
def test_stale_regression_marker_allows_close(self, triage_module, monkeypatch):
"""Once grace_days have elapsed since the regression notice, the
review gate must let the close path fire — otherwise PRs that were
regressed and then abandoned stay open forever."""
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.REGRESSED_MARKER,
# Posted 30 days before NOW -> well past the default 1-day grace.
"created_at": "2026-04-24T11:00:00Z",
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "closed"
assert rec.closed == [7]
assert len(rec.comments) == 1
def test_linked_issue_with_greptile_fail_uses_greptile_explanation(
self, triage_module, monkeypatch
):
"""When the rubric short-circuits to pass (linked-issue regex) but
Greptile dragged the PR under the bar, the close comment's
explanation must describe the Greptile shortfall, not the
misleading "LLM was not called" rubric placeholder."""
pr = _make_pr(body="Fixes #4321\n\nbody", created_at=TWO_DAYS_AGO)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("LLM must not be called for linked issue"),
greptile_score=2,
)
assert result["action"] == "closed"
assert len(rec.comments) == 1
body = rec.comments[0]
assert "LLM was not called" not in body
assert "Greptile" in body and "2/5" in body
class TestReviewGateDryRun:
@pytest.mark.parametrize(
"scenario,labels,judge,score,created,expected",
[
("pass", [], _pass, 5, JUST_NOW, "would-label-ready"),
(
"regress",
[{"name": "ready for review"}],
_fail,
5,
JUST_NOW,
"would-remove-label",
),
("within-grace", [], _fail, None, JUST_NOW, "would-notify-within-grace"),
("past-grace", [], _fail, None, TWO_DAYS_AGO, "would-close"),
],
)
def test_dry_run_previews_without_side_effects(
self,
triage_module,
monkeypatch,
scenario,
labels,
judge,
score,
created,
expected,
):
pr = _make_pr(labels=labels, created_at=created)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, close=False, judge=judge, greptile_score=score)
assert result["action"] == expected
# Dry run touches nothing.
assert rec.added == [] and rec.removed == [] and rec.closed == []
assert rec.comments == []
assert "comment" in result # preview body still surfaced
class TestReviewGateGuards:
def test_skips_internal_author(self, triage_module, monkeypatch):
pr = _make_pr(author_association="MEMBER", user={"login": "krrish"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("no LLM for internal"),
allowlist=frozenset(),
)
assert result["action"] == "skip-internal-author"
def test_skips_closed_pr(self, triage_module, monkeypatch):
pr = _make_pr(state="closed")
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(triage_module, judge=lambda p: pytest.fail("no LLM for closed"))
assert result["action"] == "skip-not-open"
def test_llm_error_is_non_destructive(self, triage_module, monkeypatch):
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
def boom(prompt):
raise RuntimeError("api down")
result = _gate(triage_module, judge=boom, greptile_score=None)
assert result["action"] == "skip-llm-error"
assert rec.closed == [] and rec.added == [] and rec.removed == []
def test_full_recovery_cycle(self, triage_module, monkeypatch):
"""pass -> regress -> recover, threading labels/comments like GitHub would."""
state = {"labels": [], "comments": []}
def fake_fetch(repo, n):
return _make_pr(labels=list(state["labels"]), created_at=JUST_NOW)
monkeypatch.setattr(triage_module, "fetch_pr", fake_fetch)
monkeypatch.setattr(
triage_module,
"post_comment",
lambda repo, n, body: state["comments"].append(
{"user": {"login": "github-actions[bot]"}, "body": body}
),
)
monkeypatch.setattr(
triage_module,
"add_label",
lambda repo, n, label: state["labels"].append({"name": label}),
)
monkeypatch.setattr(
triage_module,
"remove_label",
lambda repo, n, label: state["labels"].clear(),
)
monkeypatch.setattr(
triage_module, "close_pr", lambda repo, n: pytest.fail("must not close")
)
# 1) passes -> tagged
r1 = _gate(
triage_module, judge=_pass, greptile_score=5, comments=state["comments"]
)
assert r1["action"] == "labeled-ready"
assert any(lbl["name"] == "ready for review" for lbl in state["labels"])
# 2) regresses -> tag removed, comment posted, PR still open
r2 = _gate(
triage_module, judge=_fail, greptile_score=2, comments=state["comments"]
)
assert r2["action"] == "label-removed-regressed"
assert state["labels"] == []
# 3) fixed again -> "all clear" + tag back
r3 = _gate(
triage_module, judge=_pass, greptile_score=5, comments=state["comments"]
)
assert r3["action"] == "labeled-ready"
assert any(lbl["name"] == "ready for review" for lbl in state["labels"])
assert "all clear" in state["comments"][-1]["body"].lower()
class TestReviewGateNotReadyLabel:
"""The red `not ready` label mirrors `ready for review`: added whenever
the PR is not passing, removed whenever it passes, so the pair always
shows the current verdict."""
def test_fail_untagged_adds_not_ready_label(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None)
assert result["action"] == "within-grace-notified"
assert rec.added == [triage_module.NOT_READY_LABEL]
assert rec.closed == []
def test_regression_swaps_ready_for_not_ready(self, triage_module, monkeypatch):
pr = _make_pr(labels=[{"name": "ready for review"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=5)
assert result["action"] == "label-removed-regressed"
assert rec.added == [triage_module.NOT_READY_LABEL]
assert rec.removed == [triage_module.READY_FOR_REVIEW_LABEL]
def test_pass_removes_not_ready_label(self, triage_module, monkeypatch):
pr = _make_pr(labels=[{"name": "not ready"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
assert rec.removed == [triage_module.NOT_READY_LABEL]
def test_pass_already_tagged_still_clears_stale_not_ready(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}, {"name": "not ready"}])
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "noop-passing"
assert rec.removed == [triage_module.NOT_READY_LABEL]
assert rec.added == [] and rec.comments == []
def test_labels_already_in_sync_are_untouched(self, triage_module, monkeypatch):
pr = _make_pr(labels=[{"name": "not ready"}], created_at=JUST_NOW)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.WITHIN_GRACE_MARKER,
}
]
result = _gate(triage_module, judge=_fail, greptile_score=None, comments=prior)
assert result["action"] == "within-grace-already-notified"
assert rec.added == [] and rec.removed == []
def test_dry_run_never_touches_labels(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module, "fetch_pr", lambda repo, n: _make_pr(created_at=JUST_NOW)
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, close=False, judge=_fail, greptile_score=None)
assert result["action"] == "would-notify-within-grace"
assert rec.added == [] and rec.removed == []
class TestReviewGateLiteMode:
"""Lite mode (notice_only=True): the launch-week posture. Never closes;
failing untagged PRs get a one-time "closes start in 7 days" notice
linking the policy post. Labels still reconcile."""
def test_lite_posts_one_time_notice_instead_of_closing(
self, triage_module, monkeypatch
):
# TWO_DAYS_AGO is past the full-mode grace window, so this PR would
# be CLOSED in full mode — lite must notice instead.
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=None, notice_only=True)
assert result["action"] == "lite-notified"
assert rec.closed == []
assert len(rec.comments) == 1
assert triage_module.LITE_NOTICE_MARKER in rec.comments[0]
assert "7 days" in rec.comments[0]
assert "@agent-shin reconsider" in rec.comments[0]
assert rec.added == [triage_module.NOT_READY_LABEL]
def test_lite_notice_links_policy_url(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
url = "https://docs.litellm.ai/blog/agent-shin"
_gate(
triage_module,
judge=_fail,
greptile_score=None,
notice_only=True,
policy_url=url,
)
assert url in rec.comments[0]
def test_lite_does_not_double_notify(self, triage_module, monkeypatch):
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
prior = [
{
"user": {"login": "github-actions[bot]"},
"body": triage_module.LITE_NOTICE_MARKER,
}
]
result = _gate(
triage_module,
judge=_fail,
greptile_score=None,
notice_only=True,
comments=prior,
)
assert result["action"] == "lite-already-notified"
assert rec.comments == [] and rec.closed == []
def test_lite_dry_run_previews_without_side_effects(
self, triage_module, monkeypatch
):
monkeypatch.setattr(
triage_module,
"fetch_pr",
lambda repo, n: _make_pr(created_at=TWO_DAYS_AGO),
)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module,
close=False,
judge=_fail,
greptile_score=None,
notice_only=True,
)
assert result["action"] == "would-notify-lite"
assert rec.comments == [] and rec.closed == [] and rec.added == []
assert "comment" in result
def test_lite_regression_quotes_lite_window_and_never_closes(
self, triage_module, monkeypatch
):
pr = _make_pr(labels=[{"name": "ready for review"}], created_at=TWO_DAYS_AGO)
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_fail, greptile_score=2, notice_only=True)
assert result["action"] == "label-removed-regressed"
assert rec.closed == []
assert "7 days" in rec.comments[0]
assert "24 hours" not in rec.comments[0]
def test_lite_pass_still_labels_ready(self, triage_module, monkeypatch):
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: _make_pr())
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5, notice_only=True)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
class TestReviewGateAllowlist:
"""While the dogfood allowlist is active it is the sole author gate:
only the named accounts pass, and for them the internal-author exemption
is bypassed. Emptying it restores the normal internal-author skip."""
def test_should_skip_author_not_on_allowlist(self, triage_module, monkeypatch):
pr = _make_pr(user={"login": "random-oss-dev"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(
triage_module, judge=lambda p: pytest.fail("no LLM for non-allowlisted")
)
assert result["action"] == "skip-not-allowlisted"
assert rec.added == [] and rec.comments == [] and rec.closed == []
def test_should_act_on_allowlisted_internal_author(
self, triage_module, monkeypatch
):
pr = _make_pr(author_association="MEMBER", user={"login": "mateo-berri"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
rec = _Recorder(triage_module, monkeypatch)
result = _gate(triage_module, judge=_pass, greptile_score=5)
assert result["action"] == "labeled-ready"
assert rec.added == [triage_module.READY_FOR_REVIEW_LABEL]
def test_empty_allowlist_restores_internal_skip(self, triage_module, monkeypatch):
pr = _make_pr(author_association="MEMBER", user={"login": "krrish"})
monkeypatch.setattr(triage_module, "fetch_pr", lambda repo, n: pr)
result = _gate(
triage_module,
judge=lambda p: pytest.fail("no LLM for internal"),
allowlist=frozenset(),
)
assert result["action"] == "skip-internal-author"
Reference in a new issue View git blame Copy permalink