diff --git a/.github/scripts/triage/embedding_utils.py b/.github/scripts/triage/embedding_utils.py new file mode 100644 index 000000000..95e872de1 --- /dev/null +++ b/.github/scripts/triage/embedding_utils.py @@ -0,0 +1,178 @@ +"""Pure math utilities for triage sweep embedding analysis. + +All functions are stateless and perform no I/O (except model loading by FastEmbed). +Each function operates on numpy arrays and returns numpy arrays or plain Python types. +""" +from __future__ import annotations + +import numpy as np +from numpy.typing import NDArray +from fastembed import TextEmbedding +from sklearn.decomposition import PCA +from sklearn.covariance import EllipticEnvelope +from sklearn.metrics.pairwise import cosine_similarity + +# FastEmbed model — BAAI/bge-small-en-v1.5 produces 384-dimensional embeddings. +# ~46MB quantized ONNX, runs on CPU in ~0.5s per batch of 32. +EMBEDDING_MODEL: str = "BAAI/bge-small-en-v1.5" + +# Embedding dimensionality (determined by model choice). +EMBEDDING_DIM: int = 384 + +# Batch size for FastEmbed. 32 balances memory and throughput on +# a 2-vCPU GitHub Actions runner with ~7GB RAM. +EMBEDDING_BATCH_SIZE: int = 32 + + +def embed_texts(texts: list[str]) -> NDArray[np.float32]: + """Embed a list of texts into dense vectors using FastEmbed. + + Returns an array of shape (len(texts), 384) with dtype float32. + Empty input returns a (0, 384) array. + """ + if not texts: + return np.empty((0, EMBEDDING_DIM), dtype=np.float32) + + model = TextEmbedding(model_name=EMBEDDING_MODEL) + vectors = list(model.embed(texts, batch_size=EMBEDDING_BATCH_SIZE)) + return np.vstack(vectors).astype(np.float32) + + +def normalize_rows(matrix: NDArray[np.float32]) -> NDArray[np.float32]: + """L2-normalize each row to unit length. + + Zero-norm rows (e.g. from empty text) remain zero vectors. + Uses eps=1e-10 in the denominator to avoid division by zero. + """ + if matrix.shape[0] == 0: + return matrix + + norms = np.linalg.norm(matrix, axis=1, keepdims=True) + return matrix / (norms + 1e-10) + + +def reduce_dimensions( + matrix: NDArray[np.float32], + variance_ratio: float, + max_components: int, +) -> NDArray[np.float32]: + """Reduce dimensionality via PCA. + + Computes n_components = min(max_components, n-1, d). If n_components < 1, + returns the matrix unchanged. Logs explained variance for observability. + The variance_ratio parameter documents intent but is not strictly enforced; + the actual retained variance depends on the data and component cap. + """ + n, d = matrix.shape + if n <= 1: + return matrix + + n_components = min(max_components, n - 1, d) + if n_components < 1: + return matrix + + pca = PCA(n_components=n_components) + reduced = pca.fit_transform(matrix) + explained = pca.explained_variance_ratio_.sum() + print(f"PCA: {d}d -> {n_components}d, explained variance: {explained:.3f}") + return reduced.astype(np.float32) + + +def detect_outliers( + matrix: NDArray[np.float32], + threshold: float, +) -> list[int]: + """Flag items whose Mahalanobis distance exceeds the threshold. + + Uses EllipticEnvelope (robust covariance via MCD) to estimate the + multivariate Gaussian, then computes sqrt(squared Mahalanobis distance) + for each sample. Returns indices of outliers sorted ascending. + """ + n = matrix.shape[0] + if n < 2: + return [] + + envelope = EllipticEnvelope(contamination=0.1, random_state=42) + envelope.fit(matrix) + + # .mahalanobis() returns squared Mahalanobis distances + distances = np.sqrt(envelope.mahalanobis(matrix)) + outlier_mask = distances > threshold + return list(np.where(outlier_mask)[0]) + + +def find_duplicate_pairs( + matrix: NDArray[np.float32], + threshold: float, +) -> list[tuple[int, int, float]]: + """Find pairs of items with cosine similarity above threshold. + + Returns (i, j, similarity) tuples where i < j. The input should be + L2-normalized embeddings (full dimensionality, not PCA-reduced) so + cosine similarity equals the dot product. + """ + n = matrix.shape[0] + if n <= 1: + return [] + + sim_matrix = cosine_similarity(matrix) + # Upper triangle indices (i < j), excluding diagonal + rows, cols = np.triu_indices(n, k=1) + similarities = sim_matrix[rows, cols] + + mask = similarities > threshold + pairs: list[tuple[int, int, float]] = [] + for idx in np.where(mask)[0]: + pairs.append((int(rows[idx]), int(cols[idx]), float(similarities[idx]))) + + return pairs + + +# ── Label suggestion via embedding similarity ──────────────────────── + +# Minimum similarity between an item and a label to suggest it. +# 0.4 is intentionally permissive — the report is for human review. +LABEL_SIMILARITY_THRESHOLD: float = 0.4 + +# Maximum number of labels to suggest per item. +MAX_LABELS_PER_ITEM: int = 3 + + +def suggest_labels( + item_embeddings: NDArray[np.float32], + label_embeddings: NDArray[np.float32], + label_names: list[str], + threshold: float = LABEL_SIMILARITY_THRESHOLD, + max_per_item: int = MAX_LABELS_PER_ITEM, +) -> list[list[tuple[str, float]]]: + """Suggest labels for each item based on embedding similarity. + + Computes cosine similarity between item embeddings (n, 384) and + label embeddings (m, 384). For each item, returns the top-k labels + whose similarity exceeds the threshold, sorted by similarity descending. + + Returns a list of length n, where each element is a list of + (label_name, similarity) tuples. Empty list if no label exceeds threshold. + """ + n = item_embeddings.shape[0] + m = label_embeddings.shape[0] + if n == 0 or m == 0: + return [[] for _ in range(n)] + + # (n, m) similarity matrix: each row is one item vs all labels + sim_matrix = cosine_similarity(item_embeddings, label_embeddings) + + suggestions: list[list[tuple[str, float]]] = [] + for i in range(n): + row = sim_matrix[i] + # Indices sorted by similarity descending + ranked = np.argsort(row)[::-1] + item_labels: list[tuple[str, float]] = [] + for idx in ranked[:max_per_item]: + score = float(row[idx]) + if score < threshold: + break + item_labels.append((label_names[idx], score)) + suggestions.append(item_labels) + + return suggestions diff --git a/.github/scripts/triage/requirements.txt b/.github/scripts/triage/requirements.txt new file mode 100644 index 000000000..82531ae4b --- /dev/null +++ b/.github/scripts/triage/requirements.txt @@ -0,0 +1,3 @@ +fastembed>=0.5.0 +numpy>=1.26.0 +scikit-learn>=1.4.0 diff --git a/.github/scripts/triage/sweep.py b/.github/scripts/triage/sweep.py new file mode 100644 index 000000000..f6cebc7f1 --- /dev/null +++ b/.github/scripts/triage/sweep.py @@ -0,0 +1,442 @@ +"""Triage sweep: fetch open issues/PRs, detect outliers and duplicates, generate a report. + +Entrypoint script for the triage-sweep workflow. Fetches all open items via +the GitHub REST API, delegates embedding and analysis to embedding_utils, +generates a markdown report, and optionally creates a report issue. +""" +from __future__ import annotations + +import json +import os +import sys +import urllib.request +import urllib.parse +from typing import TypedDict +from datetime import datetime, timezone + +from embedding_utils import ( + embed_texts, + normalize_rows, + reduce_dimensions, + detect_outliers, + find_duplicate_pairs, + suggest_labels, +) + +# ── Thresholds (overridable via workflow_dispatch inputs) ────────────── + +# Mahalanobis distance beyond which an item is flagged as an outlier. +# Default 3.0 ~ 99.7% of a Gaussian distribution (3-sigma rule). +MAHALANOBIS_THRESHOLD: float = float(os.environ.get("INPUT_MAHALANOBIS_THRESHOLD", "3.0")) + +# Cosine similarity above which two items are flagged as duplicates. +# 0.92 catches near-identical issues while tolerating paraphrasing. +COSINE_THRESHOLD: float = float(os.environ.get("INPUT_COSINE_THRESHOLD", "0.92")) + +# Hard cap on items to process. Prevents runaway costs on very large repos. +MAX_ITEMS: int = int(os.environ.get("INPUT_MAX_ITEMS", "500")) + +# When true, print report to stdout/file but do not create a GitHub issue. +DRY_RUN: bool = os.environ.get("INPUT_DRY_RUN", "false").lower() == "true" + +# ── Fixed constants (not user-configurable) ─────────────────────────── + +# Minimum number of samples required for EllipticEnvelope to fit +# a Gaussian. Below this, outlier detection is skipped because +# covariance estimation is unreliable. +MIN_SAMPLES_FOR_OUTLIER_DETECTION: int = 10 + +# PCA: retain components explaining this fraction of variance. +# 0.95 keeps 95% of information while reducing dimensionality enough +# for EllipticEnvelope to be numerically stable. +PCA_VARIANCE_RATIO: float = 0.95 + +# PCA: maximum number of components regardless of variance ratio. +# Caps dimensionality for EllipticEnvelope's n_samples > n_features^2 rule. +PCA_MAX_COMPONENTS: int = 50 + +# GitHub REST API page size (max allowed is 100). +API_PAGE_SIZE: int = 100 + +# Report issue label. +REPORT_LABEL: str = "triage-report" + +# Report file path (written for the summary step to pick up). +REPORT_FILE: str = "/tmp/triage-report.md" + + +class TriageItem(TypedDict): + """One open issue or PR, with only the fields we need.""" + number: int + title: str + html_url: str + is_pr: bool + labels: list[str] + created_at: str + # title + body concatenated, used as embedding input + text: str + + +def github_api_get(path: str) -> list[dict]: + """Make a single authenticated GET request to the GitHub REST API. + + Reads GITHUB_TOKEN and GITHUB_REPOSITORY from env. Raises SystemExit + with the HTTP status and response body on any non-2xx response. + """ + token = os.environ["GITHUB_TOKEN"] + repo = os.environ["GITHUB_REPOSITORY"] + url = f"https://api.github.com/repos/{repo}{path}" + + req = urllib.request.Request(url) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("Authorization", f"Bearer {token}") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + print(f"::error::GitHub API {e.code}: {body}") + sys.exit(1) + + +def fetch_all_open_items() -> list[TriageItem]: + """Paginate through all open issues and PRs. + + Returns up to MAX_ITEMS TriageItem dicts. Items with a pull_request + key are marked is_pr=True. The text field is title + body concatenated. + """ + items: list[TriageItem] = [] + page = 1 + + while len(items) < MAX_ITEMS: + path = ( + f"/issues?state=open&per_page={API_PAGE_SIZE}" + f"&sort=created&direction=desc&page={page}" + ) + data = github_api_get(path) + + if not data: + break + + for raw in data: + if len(items) >= MAX_ITEMS: + break + + body = raw.get("body", "") or "" + items.append(TriageItem( + number=raw["number"], + title=raw["title"], + html_url=raw["html_url"], + is_pr="pull_request" in raw, + labels=[lbl["name"] for lbl in raw.get("labels", [])], + created_at=raw["created_at"], + text=f"{raw['title']}\n\n{body}", + )) + + if len(data) < API_PAGE_SIZE: + break + + page += 1 + + return items + + +class RepoLabel(TypedDict): + """A label from the repo with its embedding text.""" + name: str + description: str + # "name: description" concatenated for embedding + text: str + + +def fetch_repo_labels() -> list[RepoLabel]: + """Fetch all labels from the repository. + + Returns labels with name, description, and a text field suitable + for embedding ("name: description"). Labels with no description + use just the name. + """ + data = github_api_get("/labels?per_page=100") + labels: list[RepoLabel] = [] + for raw in data: + name = raw["name"] + desc = raw.get("description", "") or "" + text = f"{name}: {desc}" if desc else name + labels.append(RepoLabel(name=name, description=desc, text=text)) + return labels + + +def apply_labels_to_item(item_number: int, labels: list[str]) -> None: + """Add labels to a single issue/PR via the GitHub API. + + Skips silently if labels list is empty. Uses POST which adds labels + without removing existing ones. + """ + if not labels: + return + + token = os.environ["GITHUB_TOKEN"] + repo = os.environ["GITHUB_REPOSITORY"] + url = f"https://api.github.com/repos/{repo}/issues/{item_number}/labels" + + payload = json.dumps({"labels": labels}).encode("utf-8") + req = urllib.request.Request(url, data=payload, method="POST") + req.add_header("Accept", "application/vnd.github+json") + req.add_header("Authorization", f"Bearer {token}") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + req.add_header("Content-Type", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + resp.read() + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + # Non-fatal: log warning but don't abort the sweep + print(f"::warning::Failed to label #{item_number}: {e.code} {body}") + + +def generate_report( + items: list[TriageItem], + outlier_indices: list[int], + duplicate_pairs: list[tuple[int, int, float]], + label_suggestions: list[list[tuple[str, float]]] | None = None, +) -> str: + """Generate a structured markdown triage report.""" + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + repo = os.environ.get("GITHUB_REPOSITORY", "unknown/repo") + + lines: list[str] = [ + "## Triage Sweep Report", + "", + f"**Run:** {now} UTC", + f"**Items analyzed:** {len(items)}", + f"**Thresholds:** Mahalanobis > {MAHALANOBIS_THRESHOLD}, Cosine > {COSINE_THRESHOLD}", + "", + f"### Potential Outliers / Spam ({len(outlier_indices)})", + "", + "Items with unusually high Mahalanobis distance from the distribution center.", + "These may be spam, off-topic, or poorly described.", + "", + ] + + if outlier_indices: + lines.append("| # | Type | Title | Distance |") + lines.append("|---|------|-------|----------|") + for idx in outlier_indices: + item = items[idx] + kind = "PR" if item["is_pr"] else "Issue" + lines.append( + f"| [#{item['number']}]({item['html_url']}) " + f"| {kind} | {item['title']} | flagged |" + ) + else: + lines.append("None found.") + + lines.extend([ + "", + f"### Potential Duplicates ({len(duplicate_pairs)} pairs)", + "", + "Pairs of items with cosine similarity above the threshold.", + "", + ]) + + if duplicate_pairs: + lines.append("| Item A | Item B | Similarity |") + lines.append("|--------|--------|------------|") + for i, j, sim in duplicate_pairs: + a = items[i] + b = items[j] + kind_a = "PR" if a["is_pr"] else "Issue" + kind_b = "PR" if b["is_pr"] else "Issue" + lines.append( + f"| [#{a['number']}]({a['html_url']}) {kind_a}: {a['title']} " + f"| [#{b['number']}]({b['html_url']}) {kind_b}: {b['title']} " + f"| {sim:.3f} |" + ) + else: + lines.append("None found.") + + # ── Label suggestions section ──────────────────────────────────── + outlier_set = set(outlier_indices) + if label_suggestions is not None: + # Only unlabeled, non-outlier items — spam shouldn't get categorized + items_with_suggestions = [ + (i, sugs) for i, sugs in enumerate(label_suggestions) + if sugs and not items[i]["labels"] and i not in outlier_set + ] + lines.extend([ + "", + f"### Suggested Labels ({len(items_with_suggestions)} unlabeled items)", + "", + "Labels suggested by embedding similarity against repo label descriptions.", + "Only shown for unlabeled items that were not flagged as outliers.", + "", + ]) + + if items_with_suggestions: + lines.append("| # | Type | Title | Suggested Labels |") + lines.append("|---|------|-------|-----------------|") + for idx, sugs in items_with_suggestions: + item = items[idx] + kind = "PR" if item["is_pr"] else "Issue" + label_strs = [f"`{name}` ({score:.2f})" for name, score in sugs] + lines.append( + f"| [#{item['number']}]({item['html_url']}) " + f"| {kind} | {item['title']} | {', '.join(label_strs)} |" + ) + else: + lines.append("No unlabeled items need suggestions.") + + lines.extend([ + "", + "### Summary", + "", + f"- {len(outlier_indices)} outliers flagged for review", + f"- {len(duplicate_pairs)} duplicate pairs found", + f"- {len(items)} items analyzed in total", + ]) + + if label_suggestions is not None: + applied = sum( + 1 for i, s in enumerate(label_suggestions) + if s and not items[i]["labels"] and i not in outlier_set + ) + lines.append(f"- {applied} items suggested for labeling") + + lines.extend([ + "", + "---", + f"*Generated by [triage-sweep](https://github.com/{repo}/actions) — no LLM was used.*", + ]) + + return "\n".join(lines) + + +def create_report_issue(report_body: str) -> None: + """Create a GitHub issue with the triage report. + + Posts to the issues API with the triage-report label. + Raises SystemExit on non-201 response. + """ + token = os.environ["GITHUB_TOKEN"] + repo = os.environ["GITHUB_REPOSITORY"] + url = f"https://api.github.com/repos/{repo}/issues" + + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + payload = json.dumps({ + "title": f"Triage Sweep Report — {today}", + "body": report_body, + "labels": [REPORT_LABEL], + }).encode("utf-8") + + req = urllib.request.Request(url, data=payload, method="POST") + req.add_header("Accept", "application/vnd.github+json") + req.add_header("Authorization", f"Bearer {token}") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + req.add_header("Content-Type", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + resp_body = resp.read().decode("utf-8") + if resp.status != 201: + print(f"::error::Failed to create issue: {resp.status} {resp_body}") + sys.exit(1) + result = json.loads(resp_body) + print(f"Created issue: {result.get('html_url', 'unknown')}") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + print(f"::error::Failed to create issue: {e.code} {body}") + sys.exit(1) + + +def write_report(report: str) -> None: + """Write the report to the file system for the summary step.""" + with open(REPORT_FILE, "w", encoding="utf-8") as f: + f.write(report) + + +def main() -> None: + """Orchestrate the full triage sweep.""" + # 1. Validate environment + for var in ("GITHUB_TOKEN", "GITHUB_REPOSITORY"): + if not os.environ.get(var): + print(f"::error::Missing required environment variable: {var}") + sys.exit(1) + + # 2. Fetch all open issues + PRs + items = fetch_all_open_items() + print(f"Fetched {len(items)} open items") + + if len(items) == 0: + report = "## Triage Sweep Report\n\nNo open issues or PRs found." + write_report(report) + print("No items to analyze.") + return + + # 3. Extract texts for embedding + texts: list[str] = [item["text"] for item in items] + + # 4. Embed all texts (returns numpy float32 array of shape [n, 384]) + embeddings = embed_texts(texts) + + # 5. L2-normalize + embeddings = normalize_rows(embeddings) + + # 6. Outlier detection (Mahalanobis via EllipticEnvelope) + outlier_indices: list[int] = [] + if len(items) >= MIN_SAMPLES_FOR_OUTLIER_DETECTION: + reduced = reduce_dimensions(embeddings, PCA_VARIANCE_RATIO, PCA_MAX_COMPONENTS) + outlier_indices = detect_outliers(reduced, MAHALANOBIS_THRESHOLD) + else: + print( + f"Skipping outlier detection: {len(items)} items < " + f"{MIN_SAMPLES_FOR_OUTLIER_DETECTION} minimum" + ) + + # 7. Duplicate detection (pairwise cosine similarity) + duplicate_pairs = find_duplicate_pairs(embeddings, COSINE_THRESHOLD) + + # 8. Label suggestion via embedding similarity + label_suggestions: list[list[tuple[str, float]]] | None = None + repo_labels = fetch_repo_labels() + if repo_labels: + label_texts = [lbl["text"] for lbl in repo_labels] + label_names = [lbl["name"] for lbl in repo_labels] + label_embeddings = embed_texts(label_texts) + label_embeddings = normalize_rows(label_embeddings) + label_suggestions = suggest_labels(embeddings, label_embeddings, label_names) + print(f"Computed label suggestions against {len(repo_labels)} repo labels") + + # Apply top label to unlabeled items (unless dry run) + # Skip outliers — flagged items shouldn't get categorized + outlier_set = set(outlier_indices) + if not DRY_RUN: + applied_count = 0 + for i, sugs in enumerate(label_suggestions): + if sugs and not items[i]["labels"] and i not in outlier_set: + # Apply only the top-1 label (highest confidence) + apply_labels_to_item(items[i]["number"], [sugs[0][0]]) + applied_count += 1 + print(f"Applied labels to {applied_count} unlabeled items") + else: + print("No repo labels found — skipping label suggestions") + + # 9. Generate report + report = generate_report(items, outlier_indices, duplicate_pairs, label_suggestions) + + # 10. Write report to file (for summary step) + write_report(report) + + # 11. Create report issue (unless dry run) + if DRY_RUN: + print("Dry run — skipping issue creation and label application.") + print(report) + else: + create_report_issue(report) + print("Report issue created.") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/triage/test_embedding_utils.py b/.github/scripts/triage/test_embedding_utils.py new file mode 100644 index 000000000..c192f63ab --- /dev/null +++ b/.github/scripts/triage/test_embedding_utils.py @@ -0,0 +1,311 @@ +"""Tests for embedding_utils.py — all embedding model calls are mocked.""" +from __future__ import annotations + +import sys +from unittest.mock import patch, MagicMock +import numpy as np +import pytest + +# Mock fastembed before importing the module under test (persistent) +if "fastembed" not in sys.modules: + sys.modules["fastembed"] = MagicMock() + +from embedding_utils import ( + embed_texts, + normalize_rows, + reduce_dimensions, + detect_outliers, + find_duplicate_pairs, + suggest_labels, + EMBEDDING_DIM, + EMBEDDING_MODEL, + EMBEDDING_BATCH_SIZE, + LABEL_SIMILARITY_THRESHOLD, + MAX_LABELS_PER_ITEM, +) + + +class TestEmbedTexts: + """Tests for the embed_texts function.""" + + def test_empty_list_returns_empty_array(self): + result = embed_texts([]) + assert result.shape == (0, EMBEDDING_DIM) + assert result.dtype == np.float32 + + @patch("embedding_utils.TextEmbedding") + def test_single_text(self, mock_cls): + mock_model = MagicMock() + mock_cls.return_value = mock_model + vec = np.random.randn(EMBEDDING_DIM).astype(np.float32) + mock_model.embed.return_value = iter([vec]) + + result = embed_texts(["hello world"]) + + mock_cls.assert_called_once_with(model_name=EMBEDDING_MODEL) + mock_model.embed.assert_called_once_with( + ["hello world"], batch_size=EMBEDDING_BATCH_SIZE + ) + assert result.shape == (1, EMBEDDING_DIM) + assert result.dtype == np.float32 + np.testing.assert_array_almost_equal(result[0], vec) + + @patch("embedding_utils.TextEmbedding") + def test_multiple_texts(self, mock_cls): + mock_model = MagicMock() + mock_cls.return_value = mock_model + vecs = [ + np.random.randn(EMBEDDING_DIM).astype(np.float32) + for _ in range(5) + ] + mock_model.embed.return_value = iter(vecs) + + result = embed_texts(["a", "b", "c", "d", "e"]) + assert result.shape == (5, EMBEDDING_DIM) + assert result.dtype == np.float32 + + +class TestNormalizeRows: + """Tests for L2 row normalization.""" + + def test_empty_matrix(self): + m = np.empty((0, 10), dtype=np.float32) + result = normalize_rows(m) + assert result.shape == (0, 10) + + def test_single_row(self): + m = np.array([[3.0, 4.0]], dtype=np.float32) + result = normalize_rows(m) + # Norm should be ~1.0 + norm = np.linalg.norm(result[0]) + assert abs(norm - 1.0) < 1e-5 + + def test_multiple_rows(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((10, 50)).astype(np.float32) + result = normalize_rows(m) + norms = np.linalg.norm(result, axis=1) + np.testing.assert_allclose(norms, 1.0, atol=1e-5) + + def test_zero_row_stays_near_zero(self): + m = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0]], dtype=np.float32) + result = normalize_rows(m) + # Zero row divided by eps -> very small values + assert np.linalg.norm(result[0]) < 1e-3 + # Non-zero row should be unit norm + assert abs(np.linalg.norm(result[1]) - 1.0) < 1e-5 + + def test_preserves_direction(self): + m = np.array([[2.0, 0.0], [0.0, 3.0]], dtype=np.float32) + result = normalize_rows(m) + np.testing.assert_allclose(result[0], [1.0, 0.0], atol=1e-5) + np.testing.assert_allclose(result[1], [0.0, 1.0], atol=1e-5) + + +class TestReduceDimensions: + """Tests for PCA dimensionality reduction.""" + + def test_single_sample_returns_unchanged(self): + m = np.random.randn(1, 50).astype(np.float32) + result = reduce_dimensions(m, 0.95, 10) + np.testing.assert_array_equal(result, m) + + def test_reduces_dimensions(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((100, 50)).astype(np.float32) + result = reduce_dimensions(m, 0.95, 10) + assert result.shape == (100, 10) + assert result.dtype == np.float32 + + def test_caps_at_n_minus_1(self): + rng = np.random.default_rng(42) + # 5 samples, 20 features -> max components = 4 (n-1) + m = rng.standard_normal((5, 20)).astype(np.float32) + result = reduce_dimensions(m, 0.95, 50) + assert result.shape == (5, 4) + + def test_caps_at_d(self): + rng = np.random.default_rng(42) + # 100 samples, 3 features -> max components = 3 + m = rng.standard_normal((100, 3)).astype(np.float32) + result = reduce_dimensions(m, 0.95, 50) + assert result.shape == (100, 3) + + def test_max_components_respected(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((50, 30)).astype(np.float32) + result = reduce_dimensions(m, 0.95, 5) + assert result.shape[1] == 5 + + +class TestDetectOutliers: + """Tests for Mahalanobis-based outlier detection.""" + + def test_single_sample_returns_empty(self): + m = np.random.randn(1, 5).astype(np.float32) + result = detect_outliers(m, 3.0) + assert result == [] + + def test_empty_returns_empty(self): + # n < 2 case + m = np.empty((0, 5), dtype=np.float32) + result = detect_outliers(m, 3.0) + assert result == [] + + def test_finds_outliers_in_synthetic_data(self): + rng = np.random.default_rng(42) + # Create a tight cluster with one obvious outlier + cluster = rng.standard_normal((50, 3)).astype(np.float32) * 0.1 + outlier = np.array([[100.0, 100.0, 100.0]], dtype=np.float32) + m = np.vstack([cluster, outlier]) + result = detect_outliers(m, 3.0) + # The outlier (index 50) should be detected + assert 50 in result + + def test_returns_list_of_ints(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((20, 3)).astype(np.float32) + result = detect_outliers(m, 3.0) + assert isinstance(result, list) + for idx in result: + assert isinstance(idx, (int, np.integer)) + + def test_low_threshold_flags_more(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((30, 3)).astype(np.float32) + low = detect_outliers(m, 1.0) + high = detect_outliers(m, 10.0) + assert len(low) >= len(high) + + +class TestFindDuplicatePairs: + """Tests for cosine similarity duplicate detection.""" + + def test_single_item_returns_empty(self): + m = np.random.randn(1, 10).astype(np.float32) + result = find_duplicate_pairs(m, 0.9) + assert result == [] + + def test_empty_returns_empty(self): + m = np.empty((0, 10), dtype=np.float32) + result = find_duplicate_pairs(m, 0.9) + assert result == [] + + def test_identical_vectors_detected(self): + vec = np.random.randn(10).astype(np.float32) + vec = vec / np.linalg.norm(vec) + m = np.vstack([vec, vec, np.random.randn(10).astype(np.float32)]) + result = find_duplicate_pairs(m, 0.99) + # Items 0 and 1 are identical, should be found + assert any(i == 0 and j == 1 for i, j, _ in result) + + def test_orthogonal_vectors_not_detected(self): + m = np.eye(5, dtype=np.float32) + result = find_duplicate_pairs(m, 0.5) + assert result == [] + + def test_returns_correct_format(self): + vec = np.random.randn(10).astype(np.float32) + vec = vec / np.linalg.norm(vec) + m = np.vstack([vec, vec]) + result = find_duplicate_pairs(m, 0.5) + assert len(result) >= 1 + for item in result: + assert len(item) == 3 + i, j, sim = item + assert isinstance(i, int) + assert isinstance(j, int) + assert isinstance(sim, float) + assert i < j + + def test_i_less_than_j(self): + rng = np.random.default_rng(42) + # Create some similar vectors + base = rng.standard_normal(10).astype(np.float32) + m = np.vstack([base + rng.standard_normal(10) * 0.01 for _ in range(5)]) + result = find_duplicate_pairs(m, 0.5) + for i, j, _ in result: + assert i < j + + def test_high_threshold_fewer_pairs(self): + rng = np.random.default_rng(42) + m = rng.standard_normal((10, 20)).astype(np.float32) + # Normalize for meaningful cosine similarities + norms = np.linalg.norm(m, axis=1, keepdims=True) + m = m / norms + low = find_duplicate_pairs(m, 0.3) + high = find_duplicate_pairs(m, 0.9) + assert len(low) >= len(high) + + +class TestSuggestLabels: + """Tests for embedding-based label suggestion.""" + + def test_empty_items_returns_empty_lists(self): + items = np.empty((0, 10), dtype=np.float32) + labels = np.random.randn(3, 10).astype(np.float32) + result = suggest_labels(items, labels, ["a", "b", "c"]) + assert result == [] + + def test_empty_labels_returns_empty_per_item(self): + items = np.random.randn(5, 10).astype(np.float32) + labels = np.empty((0, 10), dtype=np.float32) + result = suggest_labels(items, labels, []) + assert len(result) == 5 + assert all(s == [] for s in result) + + def test_identical_embedding_gets_that_label(self): + """If an item embedding equals a label embedding, it should suggest that label.""" + vec = np.array([1.0, 0.0, 0.0], dtype=np.float32) + items = np.array([vec], dtype=np.float32) + labels = np.array([vec, [0, 1, 0], [0, 0, 1]], dtype=np.float32) + result = suggest_labels(items, labels, ["bug", "feature", "docs"], threshold=0.5) + assert len(result) == 1 + assert result[0][0][0] == "bug" + assert result[0][0][1] > 0.99 + + def test_threshold_filters_low_similarity(self): + """With a high threshold, orthogonal vectors should get no suggestions.""" + items = np.eye(3, dtype=np.float32) + labels = np.eye(3, dtype=np.float32) + # threshold=0.99 means only near-exact matches + result = suggest_labels(items, labels, ["a", "b", "c"], threshold=0.99) + # Each item should match exactly one label (itself) + for sugs in result: + assert len(sugs) == 1 + + def test_max_per_item_respected(self): + """Even if all labels are similar, max_per_item caps the results.""" + rng = np.random.default_rng(42) + base = rng.standard_normal(10).astype(np.float32) + items = np.array([base]) + # All labels very similar to item + labels = np.array([base + rng.standard_normal(10) * 0.01 for _ in range(10)]) + names = [f"label-{i}" for i in range(10)] + result = suggest_labels(items, labels, names, threshold=0.1, max_per_item=2) + assert len(result[0]) <= 2 + + def test_returns_sorted_by_similarity_descending(self): + """Suggestions should be ordered highest similarity first.""" + items = np.array([[1.0, 0.5, 0.0]], dtype=np.float32) + labels = np.array([ + [1.0, 0.0, 0.0], # decent match + [1.0, 0.5, 0.0], # exact match + [0.0, 0.0, 1.0], # poor match + ], dtype=np.float32) + result = suggest_labels(items, labels, ["a", "b", "c"], threshold=0.1) + scores = [s for _, s in result[0]] + assert scores == sorted(scores, reverse=True) + + def test_returns_correct_format(self): + rng = np.random.default_rng(42) + items = rng.standard_normal((3, 10)).astype(np.float32) + labels = rng.standard_normal((5, 10)).astype(np.float32) + names = ["bug", "feature", "docs", "ci", "test"] + result = suggest_labels(items, labels, names, threshold=0.0) + assert len(result) == 3 + for sugs in result: + for name, score in sugs: + assert isinstance(name, str) + assert isinstance(score, float) + assert name in names diff --git a/.github/scripts/triage/test_sweep.py b/.github/scripts/triage/test_sweep.py new file mode 100644 index 000000000..ea5e8cf7c --- /dev/null +++ b/.github/scripts/triage/test_sweep.py @@ -0,0 +1,616 @@ +"""Tests for sweep.py — all external calls (API, embedding) are mocked.""" +from __future__ import annotations + +import json +import os +import sys +from io import BytesIO +from unittest.mock import patch, MagicMock, mock_open +from urllib.error import HTTPError + +import numpy as np +import pytest + +# Mock fastembed before importing sweep (which imports embedding_utils) +sys.modules["fastembed"] = MagicMock() + +# Set required env vars before importing sweep (module-level constants read env) +os.environ.setdefault("GITHUB_TOKEN", "test-token") +os.environ.setdefault("GITHUB_REPOSITORY", "owner/repo") + +from sweep import ( + github_api_get, + fetch_all_open_items, + fetch_repo_labels, + apply_labels_to_item, + generate_report, + create_report_issue, + write_report, + main, + TriageItem, + RepoLabel, + REPORT_FILE, + REPORT_LABEL, + API_PAGE_SIZE, + MIN_SAMPLES_FOR_OUTLIER_DETECTION, +) + + +def _make_api_issue(number: int, title: str = "Test issue", is_pr: bool = False, + body: str = "Issue body", labels: list[str] | None = None) -> dict: + """Helper to build a mock GitHub API issue response object.""" + result: dict = { + "number": number, + "title": title, + "html_url": f"https://github.com/owner/repo/issues/{number}", + "body": body, + "created_at": "2026-03-21T00:00:00Z", + "labels": [{"name": lbl} for lbl in (labels or [])], + } + if is_pr: + result["pull_request"] = {"url": "..."} + return result + + +class TestGithubApiGet: + """Tests for the github_api_get function.""" + + @patch("sweep.urllib.request.urlopen") + def test_successful_request(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.read.return_value = json.dumps([{"id": 1}]).encode() + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.return_value = mock_resp + + result = github_api_get("/issues?state=open") + assert result == [{"id": 1}] + + @patch("sweep.urllib.request.urlopen") + def test_http_error_exits(self, mock_urlopen): + error = HTTPError( + url="https://api.github.com/repos/owner/repo/issues", + code=403, + msg="Forbidden", + hdrs=None, # type: ignore[arg-type] + fp=BytesIO(b'{"message": "rate limited"}'), + ) + mock_urlopen.side_effect = error + + with pytest.raises(SystemExit) as exc_info: + github_api_get("/issues") + assert exc_info.value.code == 1 + + +class TestFetchAllOpenItems: + """Tests for fetch_all_open_items.""" + + @patch("sweep.github_api_get") + def test_empty_repo(self, mock_get): + mock_get.return_value = [] + items = fetch_all_open_items() + assert items == [] + + @patch("sweep.github_api_get") + def test_single_page(self, mock_get): + mock_get.return_value = [ + _make_api_issue(1, "Bug report"), + _make_api_issue(2, "Feature request", is_pr=True), + ] + items = fetch_all_open_items() + assert len(items) == 2 + assert items[0]["number"] == 1 + assert items[0]["is_pr"] is False + assert items[1]["is_pr"] is True + + @patch("sweep.github_api_get") + def test_text_field_constructed(self, mock_get): + mock_get.return_value = [ + _make_api_issue(1, "My Title", body="My Body"), + ] + items = fetch_all_open_items() + assert items[0]["text"] == "My Title\n\nMy Body" + + @patch("sweep.github_api_get") + def test_null_body_handled(self, mock_get): + issue = _make_api_issue(1, "No body") + issue["body"] = None + mock_get.return_value = [issue] + items = fetch_all_open_items() + assert items[0]["text"] == "No body\n\n" + + @patch("sweep.github_api_get") + def test_labels_extracted(self, mock_get): + mock_get.return_value = [ + _make_api_issue(1, "Labeled", labels=["bug", "high-priority"]), + ] + items = fetch_all_open_items() + assert items[0]["labels"] == ["bug", "high-priority"] + + @patch("sweep.MAX_ITEMS", 3) + @patch("sweep.github_api_get") + def test_max_items_cap(self, mock_get): + mock_get.return_value = [_make_api_issue(i) for i in range(100)] + items = fetch_all_open_items() + assert len(items) == 3 + + @patch("sweep.API_PAGE_SIZE", 2) + @patch("sweep.github_api_get") + def test_pagination(self, mock_get): + # First page: 2 items (full page), second page: 1 item (partial -> stop) + mock_get.side_effect = [ + [_make_api_issue(1), _make_api_issue(2)], + [_make_api_issue(3)], + ] + items = fetch_all_open_items() + assert len(items) == 3 + assert mock_get.call_count == 2 + + +class TestGenerateReport: + """Tests for the markdown report generator.""" + + def test_no_findings(self): + items = [ + TriageItem( + number=1, title="Test", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="Test", + ), + ] + report = generate_report(items, [], []) + assert "## Triage Sweep Report" in report + assert "Items analyzed:** 1" in report + assert "None found." in report + assert "0 outliers flagged" in report + assert "0 duplicate pairs found" in report + + def test_with_outliers(self): + items = [ + TriageItem( + number=10, title="Spam Issue", html_url="https://example.com/10", + is_pr=False, labels=[], created_at="2026-01-01", text="spam", + ), + TriageItem( + number=20, title="Good Issue", html_url="https://example.com/20", + is_pr=False, labels=[], created_at="2026-01-01", text="good", + ), + ] + report = generate_report(items, [0], []) + assert "#10" in report + assert "Spam Issue" in report + assert "1 outliers flagged" in report + + def test_with_duplicates(self): + items = [ + TriageItem( + number=1, title="First", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="a", + ), + TriageItem( + number=2, title="Second", html_url="https://example.com/2", + is_pr=True, labels=[], created_at="2026-01-01", text="b", + ), + ] + report = generate_report(items, [], [(0, 1, 0.954)]) + assert "#1" in report + assert "#2" in report + assert "0.954" in report + assert "1 duplicate pairs found" in report + + def test_pr_type_label(self): + items = [ + TriageItem( + number=5, title="PR Title", html_url="https://example.com/5", + is_pr=True, labels=[], created_at="2026-01-01", text="pr", + ), + ] + report = generate_report(items, [0], []) + assert "| PR |" in report + + def test_footer_present(self): + items = [ + TriageItem( + number=1, title="T", html_url="u", + is_pr=False, labels=[], created_at="d", text="t", + ), + ] + report = generate_report(items, [], []) + assert "no LLM was used" in report + + +class TestCreateReportIssue: + """Tests for creating the report GitHub issue.""" + + @patch("sweep.urllib.request.urlopen") + def test_successful_creation(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.status = 201 + mock_resp.read.return_value = json.dumps({ + "html_url": "https://github.com/owner/repo/issues/99", + }).encode() + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.return_value = mock_resp + + # Should not raise + create_report_issue("# Test Report") + + @patch("sweep.urllib.request.urlopen") + def test_http_error_exits(self, mock_urlopen): + error = HTTPError( + url="https://api.github.com/repos/owner/repo/issues", + code=422, + msg="Unprocessable", + hdrs=None, # type: ignore[arg-type] + fp=BytesIO(b'{"message": "validation failed"}'), + ) + mock_urlopen.side_effect = error + + with pytest.raises(SystemExit) as exc_info: + create_report_issue("# Test Report") + assert exc_info.value.code == 1 + + +class TestWriteReport: + """Tests for the write_report helper.""" + + @patch("builtins.open", mock_open()) + def test_writes_to_file(self): + write_report("# Report Content") + from builtins import open as builtin_open # noqa + # Verify open was called with the right path + from unittest.mock import call + open_mock = open # The patched version + open_mock.assert_called_once_with(REPORT_FILE, "w", encoding="utf-8") # type: ignore[attr-defined] + open_mock().write.assert_called_once_with("# Report Content") # type: ignore[attr-defined] + + +class TestFetchRepoLabels: + """Tests for fetch_repo_labels.""" + + @patch("sweep.github_api_get") + def test_fetches_and_constructs_labels(self, mock_get): + mock_get.return_value = [ + {"name": "bug", "description": "Something isn't working"}, + {"name": "enhancement", "description": "New feature or request"}, + {"name": "docs", "description": ""}, + ] + labels = fetch_repo_labels() + assert len(labels) == 3 + assert labels[0]["name"] == "bug" + assert labels[0]["text"] == "bug: Something isn't working" + assert labels[2]["text"] == "docs" # no description, just name + + @patch("sweep.github_api_get") + def test_empty_repo_labels(self, mock_get): + mock_get.return_value = [] + labels = fetch_repo_labels() + assert labels == [] + + @patch("sweep.github_api_get") + def test_null_description_handled(self, mock_get): + mock_get.return_value = [ + {"name": "wontfix", "description": None}, + ] + labels = fetch_repo_labels() + assert labels[0]["text"] == "wontfix" + + +class TestApplyLabelsToItem: + """Tests for apply_labels_to_item.""" + + def test_empty_labels_skips(self): + # Should not make any API call + apply_labels_to_item(1, []) + + @patch("sweep.urllib.request.urlopen") + def test_successful_label_application(self, mock_urlopen): + mock_resp = MagicMock() + mock_resp.read.return_value = b'[{"name": "bug"}]' + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.return_value = mock_resp + + # Should not raise + apply_labels_to_item(42, ["bug", "enhancement"]) + + @patch("sweep.urllib.request.urlopen") + def test_http_error_is_non_fatal(self, mock_urlopen): + error = HTTPError( + url="https://api.github.com/repos/owner/repo/issues/1/labels", + code=404, + msg="Not Found", + hdrs=None, # type: ignore[arg-type] + fp=BytesIO(b'{"message": "not found"}'), + ) + mock_urlopen.side_effect = error + + # Should NOT raise — labeling failures are warnings, not fatal + apply_labels_to_item(1, ["bug"]) + + +class TestGenerateReportWithLabels: + """Tests for label suggestions in the report.""" + + def test_report_includes_label_section(self): + items = [ + TriageItem( + number=1, title="Fix crash", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="crash", + ), + ] + suggestions = [[("bug", 0.85), ("enhancement", 0.42)]] + report = generate_report(items, [], [], label_suggestions=suggestions) + assert "Suggested Labels" in report + assert "`bug` (0.85)" in report + assert "1 items suggested for labeling" in report + + def test_report_skips_already_labeled_items(self): + items = [ + TriageItem( + number=1, title="Already labeled", html_url="https://example.com/1", + is_pr=False, labels=["bug"], created_at="2026-01-01", text="bug", + ), + ] + suggestions = [[("bug", 0.95)]] + report = generate_report(items, [], [], label_suggestions=suggestions) + assert "0 items suggested for labeling" in report + assert "No unlabeled items" in report + + def test_report_excludes_outliers_from_suggestions(self): + items = [ + TriageItem( + number=1, title="Spam garbage", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="spam", + ), + TriageItem( + number=2, title="Real bug", html_url="https://example.com/2", + is_pr=False, labels=[], created_at="2026-01-01", text="bug", + ), + ] + suggestions = [[("bug", 0.85)], [("bug", 0.90)]] + # Item 0 is an outlier — should be excluded from label suggestions + report = generate_report(items, [0], [], label_suggestions=suggestions) + assert "1 unlabeled items" in report # only item 2 + assert "#2" in report + # Item 1 (outlier) should NOT be in the suggestions table + assert "Spam garbage" not in report.split("Suggested Labels")[1] + + def test_report_without_label_suggestions(self): + items = [ + TriageItem( + number=1, title="T", html_url="u", + is_pr=False, labels=[], created_at="d", text="t", + ), + ] + report = generate_report(items, [], [], label_suggestions=None) + assert "Suggested Labels" not in report + + +class TestMain: + """Tests for the main orchestration function.""" + + @patch.dict(os.environ, {"GITHUB_TOKEN": "", "GITHUB_REPOSITORY": "owner/repo"}) + def test_missing_token_exits(self): + with pytest.raises(SystemExit) as exc_info: + main() + assert exc_info.value.code == 1 + + @patch.dict(os.environ, {"GITHUB_TOKEN": "tok", "GITHUB_REPOSITORY": ""}) + def test_missing_repo_exits(self): + with pytest.raises(SystemExit) as exc_info: + main() + assert exc_info.value.code == 1 + + @patch("sweep.write_report") + @patch("sweep.fetch_all_open_items", return_value=[]) + def test_no_items(self, mock_fetch, mock_write): + main() + mock_write.assert_called_once() + report = mock_write.call_args[0][0] + assert "No open issues or PRs found" in report + + @patch("sweep.create_report_issue") + @patch("sweep.write_report") + @patch("sweep.suggest_labels", return_value=[]) + @patch("sweep.find_duplicate_pairs", return_value=[]) + @patch("sweep.detect_outliers", return_value=[]) + @patch("sweep.reduce_dimensions") + @patch("sweep.normalize_rows") + @patch("sweep.embed_texts") + @patch("sweep.fetch_repo_labels") + @patch("sweep.fetch_all_open_items") + def test_full_flow_with_enough_items( + self, mock_fetch, mock_labels, mock_embed, mock_norm, mock_reduce, + mock_outliers, mock_dupes, mock_suggest, mock_write, mock_create, + ): + """Test the full flow with >= MIN_SAMPLES items (outlier detection runs).""" + items = [ + TriageItem( + number=i, title=f"Item {i}", html_url=f"https://example.com/{i}", + is_pr=False, labels=[], created_at="2026-01-01", text=f"text {i}", + ) + for i in range(15) + ] + mock_fetch.return_value = items + mock_labels.return_value = [ + RepoLabel(name="bug", description="Something broken", text="bug: Something broken"), + ] + + embeddings = np.random.randn(15, 384).astype(np.float32) + mock_embed.return_value = embeddings + mock_norm.return_value = embeddings + mock_reduce.return_value = np.random.randn(15, 10).astype(np.float32) + + main() + + mock_fetch.assert_called_once() + mock_labels.assert_called_once() + # embed_texts called twice: once for items, once for labels + assert mock_embed.call_count == 2 + mock_norm.assert_called() + mock_reduce.assert_called_once() + mock_outliers.assert_called_once() + mock_dupes.assert_called_once() + mock_suggest.assert_called_once() + mock_write.assert_called_once() + mock_create.assert_called_once() + + @patch("sweep.create_report_issue") + @patch("sweep.write_report") + @patch("sweep.suggest_labels", return_value=[]) + @patch("sweep.find_duplicate_pairs", return_value=[]) + @patch("sweep.detect_outliers") + @patch("sweep.reduce_dimensions") + @patch("sweep.normalize_rows") + @patch("sweep.embed_texts") + @patch("sweep.fetch_repo_labels", return_value=[]) + @patch("sweep.fetch_all_open_items") + def test_skips_outlier_detection_for_few_items( + self, mock_fetch, mock_labels, mock_embed, mock_norm, mock_reduce, + mock_outliers, mock_dupes, mock_suggest, mock_write, mock_create, + ): + """With < MIN_SAMPLES items, outlier detection should be skipped.""" + items = [ + TriageItem( + number=i, title=f"Item {i}", html_url=f"https://example.com/{i}", + is_pr=False, labels=[], created_at="2026-01-01", text=f"text {i}", + ) + for i in range(5) + ] + mock_fetch.return_value = items + + embeddings = np.random.randn(5, 384).astype(np.float32) + mock_embed.return_value = embeddings + mock_norm.return_value = embeddings + + main() + + # Outlier detection should not have been called + mock_reduce.assert_not_called() + mock_outliers.assert_not_called() + # But duplicates should still be checked + mock_dupes.assert_called_once() + + @patch.dict(os.environ, {"INPUT_DRY_RUN": "true"}) + @patch("sweep.DRY_RUN", True) + @patch("sweep.write_report") + @patch("sweep.create_report_issue") + @patch("sweep.apply_labels_to_item") + @patch("sweep.suggest_labels", return_value=[[("bug", 0.85)]]) + @patch("sweep.find_duplicate_pairs", return_value=[]) + @patch("sweep.normalize_rows") + @patch("sweep.embed_texts") + @patch("sweep.fetch_repo_labels") + @patch("sweep.fetch_all_open_items") + def test_dry_run_skips_issue_creation_and_labeling( + self, mock_fetch, mock_labels, mock_embed, mock_norm, + mock_dupes, mock_suggest, mock_apply, mock_create, mock_write, + ): + items = [ + TriageItem( + number=1, title="Item", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="text", + ) + ] + mock_fetch.return_value = items + mock_labels.return_value = [ + RepoLabel(name="bug", description="Broken", text="bug: Broken"), + ] + embeddings = np.random.randn(1, 384).astype(np.float32) + mock_embed.return_value = embeddings + mock_norm.return_value = embeddings + + main() + + mock_create.assert_not_called() + mock_apply.assert_not_called() + mock_write.assert_called_once() + + @patch("sweep.create_report_issue") + @patch("sweep.write_report") + @patch("sweep.apply_labels_to_item") + @patch("sweep.suggest_labels") + @patch("sweep.find_duplicate_pairs", return_value=[]) + @patch("sweep.normalize_rows") + @patch("sweep.embed_texts") + @patch("sweep.fetch_repo_labels") + @patch("sweep.fetch_all_open_items") + def test_applies_labels_to_unlabeled_items( + self, mock_fetch, mock_labels, mock_embed, mock_norm, + mock_dupes, mock_suggest, mock_apply, mock_write, mock_create, + ): + """When not dry run, top-1 label should be applied to unlabeled items.""" + items = [ + TriageItem( + number=1, title="Crash bug", html_url="https://example.com/1", + is_pr=False, labels=[], created_at="2026-01-01", text="crash", + ), + TriageItem( + number=2, title="Already labeled", html_url="https://example.com/2", + is_pr=False, labels=["enhancement"], created_at="2026-01-01", text="feat", + ), + ] + mock_fetch.return_value = items + mock_labels.return_value = [ + RepoLabel(name="bug", description="Broken", text="bug: Broken"), + ] + mock_suggest.return_value = [ + [("bug", 0.90)], # item 1: unlabeled, should get labeled + [("bug", 0.45)], # item 2: already labeled, skip + ] + + embeddings = np.random.randn(2, 384).astype(np.float32) + mock_embed.return_value = embeddings + mock_norm.return_value = embeddings + + main() + + # Only item 1 (unlabeled) should get a label applied + mock_apply.assert_called_once_with(1, ["bug"]) + + @patch("sweep.create_report_issue") + @patch("sweep.write_report") + @patch("sweep.apply_labels_to_item") + @patch("sweep.suggest_labels") + @patch("sweep.find_duplicate_pairs", return_value=[]) + @patch("sweep.detect_outliers") + @patch("sweep.reduce_dimensions") + @patch("sweep.normalize_rows") + @patch("sweep.embed_texts") + @patch("sweep.fetch_repo_labels") + @patch("sweep.fetch_all_open_items") + def test_outliers_do_not_get_labeled( + self, mock_fetch, mock_labels, mock_embed, mock_norm, mock_reduce, + mock_outliers, mock_dupes, mock_suggest, mock_apply, mock_write, mock_create, + ): + """Items flagged as outliers should not receive label suggestions.""" + items = [ + TriageItem( + number=i, title=f"Item {i}", html_url=f"https://example.com/{i}", + is_pr=False, labels=[], created_at="2026-01-01", text=f"text {i}", + ) + for i in range(15) + ] + mock_fetch.return_value = items + mock_labels.return_value = [ + RepoLabel(name="bug", description="Broken", text="bug: Broken"), + ] + # Outlier detection flags items 0 and 5 + mock_outliers.return_value = [0, 5] + # Every item gets a suggestion + mock_suggest.return_value = [[("bug", 0.85)] for _ in range(15)] + + embeddings = np.random.randn(15, 384).astype(np.float32) + mock_embed.return_value = embeddings + mock_norm.return_value = embeddings + mock_reduce.return_value = np.random.randn(15, 10).astype(np.float32) + + main() + + # Items 0 and 5 are outliers — should NOT be labeled + labeled_numbers = [call.args[0] for call in mock_apply.call_args_list] + assert 0 not in labeled_numbers + assert 5 not in labeled_numbers + # Other items should be labeled (13 items: 15 total - 2 outliers) + assert mock_apply.call_count == 13 diff --git a/.github/workflows/pr-description-check.yml b/.github/workflows/pr-description-check.yml new file mode 100644 index 000000000..ac7817a86 --- /dev/null +++ b/.github/workflows/pr-description-check.yml @@ -0,0 +1,94 @@ +name: PR Description Check + +on: + pull_request: + types: [opened, edited, reopened] + branches: [main] + +permissions: + pull-requests: write + +concurrency: + group: pr-desc-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-description: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check PR description quality + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + with: + script: | + const MIN_BODY_LENGTH = 50; + const LABEL = 'needs-description'; + + const pr = context.payload.pull_request; + const body = (pr.body || '').trim(); + const owner = context.repo.owner; + const repo = context.repo.repo; + const number = pr.number; + + const hasLabel = pr.labels.some(l => l.name === LABEL); + + if (body.length < MIN_BODY_LENGTH) { + // Add label if not already present + if (!hasLabel) { + await github.rest.issues.addLabels({ + owner, repo, issue_number: number, + labels: [LABEL], + }); + } + + // Post or update a comment + const marker = ''; + const message = [ + marker, + `### PR description is too short`, + '', + `This PR's description is **${body.length}** characters, ` + + `but the minimum is **${MIN_BODY_LENGTH}**.`, + '', + 'Please update the PR description to explain:', + '- **What** this PR changes', + '- **Why** the change is needed', + '', + 'Use the PR template as a guide. This check will re-run when you edit the description.', + ].join('\n'); + + // Find existing bot comment to update (avoid spam) + const comments = await github.rest.issues.listComments({ + owner, repo, issue_number: number, + }); + const existing = comments.data.find(c => + c.body && c.body.includes(marker) + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner, repo, comment_id: existing.id, + body: message, + }); + } else { + await github.rest.issues.createComment({ + owner, repo, issue_number: number, + body: message, + }); + } + + core.setFailed( + `PR description is ${body.length} chars (minimum: ${MIN_BODY_LENGTH})` + ); + } else { + // Description is acceptable — remove the label if present + if (hasLabel) { + await github.rest.issues.removeLabel({ + owner, repo, issue_number: number, + name: LABEL, + }).catch(() => {}); + // .catch: label may have been removed manually + } + + core.info(`PR description OK (${body.length} chars)`); + } diff --git a/.github/workflows/triage-sweep.yml b/.github/workflows/triage-sweep.yml new file mode 100644 index 000000000..5ae12536b --- /dev/null +++ b/.github/workflows/triage-sweep.yml @@ -0,0 +1,87 @@ +name: Triage Sweep + +on: + workflow_dispatch: + inputs: + mahalanobis_threshold: + description: >- + Mahalanobis distance threshold for outlier detection. + Items with distance above this are flagged as potential spam/noise. + Lower = more aggressive flagging. + type: number + default: 3.0 + cosine_threshold: + description: >- + Cosine similarity threshold for duplicate detection. + Pairs with similarity above this are flagged as potential duplicates. + Higher = only very similar pairs flagged. + type: number + default: 0.92 + max_items: + description: >- + Maximum number of open issues + PRs to process. + Hard cap to prevent runaway costs on very large repos. + type: number + default: 500 + dry_run: + description: >- + If true, print the report to workflow logs but do not create + a GitHub issue. + type: boolean + default: false + +permissions: + contents: read + issues: write + +concurrency: + group: triage-sweep + cancel-in-progress: true + +jobs: + sweep: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + sparse-checkout: .github/scripts/triage + sparse-checkout-cone-mode: false + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: .github/scripts/triage/requirements.txt + + - name: Install dependencies + run: pip install -r .github/scripts/triage/requirements.txt + + - name: Cache FastEmbed model weights + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: ~/.cache/fastembed_cache + key: fastembed-bge-small-en-v1.5 + + - name: Run triage sweep + id: sweep + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + INPUT_MAHALANOBIS_THRESHOLD: ${{ inputs.mahalanobis_threshold }} + INPUT_COSINE_THRESHOLD: ${{ inputs.cosine_threshold }} + INPUT_MAX_ITEMS: ${{ inputs.max_items }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + run: python .github/scripts/triage/sweep.py + + - name: Post summary + if: always() + run: | + if [ -f /tmp/triage-report.md ]; then + cat /tmp/triage-report.md >> "$GITHUB_STEP_SUMMARY" + else + echo "No report generated." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.gitignore b/.gitignore index 509bdbb1e..1d3abf074 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,5 @@ gitnexus/test/fixtures/lang-resolution/**/bin GitNexus.sln # Git worktrees .worktrees/ + +/github/scripts/triage/__pycache__/ \ No newline at end of file