Merge pull request #9 from zander-raycraft/gh/issue-pr-filter

token trunking

.github/scripts/triage/sweep.py

@@ -51,6 +51,11 @@ DRY_RUN: bool = os.environ.get("INPUT_DRY_RUN", "false").lower() == "true"
 PCA_MAX_COMPONENTS: int = 33
 MIN_SAMPLES_FOR_OUTLIER_DETECTION: int = 100
 
+# Max character length for embedding input text. bge-small-en-v1.5 has a
+# 512-token context window (~4 chars/token). We keep title + body under
+# this limit so the model sees the full text instead of silently truncating.
+MAX_EMBED_CHARS: int = 2000
+
 # GitHub REST API page size (max allowed is 100).
 API_PAGE_SIZE: int = 100
 
@@ -121,6 +126,11 @@ def fetch_all_open_items() -> list[TriageItem]:
                 break
 
             body = raw.get("body", "") or ""
+            full_text = f"{raw['title']}\n\n{body}"
+            # Truncate to fit the embedding model's token window.
+            # Title is always preserved; body gets clipped if needed.
+            if len(full_text) > MAX_EMBED_CHARS:
+                full_text = full_text[:MAX_EMBED_CHARS]
             items.append(TriageItem(
                 number=raw["number"],
                 title=raw["title"],
@@ -128,7 +138,7 @@ def fetch_all_open_items() -> list[TriageItem]:
                 is_pr="pull_request" in raw,
                 labels=[lbl["name"] for lbl in raw.get("labels", [])],
                 created_at=raw["created_at"],
-                text=f"{raw['title']}\n\n{body}",
+                text=full_text,
             ))
 
         if len(data) < API_PAGE_SIZE:

.github/scripts/triage/test_sweep.py

@@ -34,6 +34,7 @@ from sweep import (
     API_PAGE_SIZE,
     MIN_SAMPLES_FOR_OUTLIER_DETECTION,
     PCA_MAX_COMPONENTS,
+    MAX_EMBED_CHARS,
 )
 
 
@@ -123,6 +124,25 @@ class TestFetchAllOpenItems:
         items = fetch_all_open_items()
         assert items[0]["text"] == "My Title\n\nMy Body"
 
+    @patch("sweep.github_api_get")
+    def test_long_body_truncated(self, mock_get):
+        """Bodies exceeding MAX_EMBED_CHARS are truncated to fit the token window."""
+        long_body = "x" * (MAX_EMBED_CHARS + 500)
+        mock_get.return_value = [
+            _make_api_issue(1, "Title", body=long_body),
+        ]
+        items = fetch_all_open_items()
+        assert len(items[0]["text"]) == MAX_EMBED_CHARS
+
+    @patch("sweep.github_api_get")
+    def test_short_body_not_truncated(self, mock_get):
+        """Bodies under the limit are left intact."""
+        mock_get.return_value = [
+            _make_api_issue(1, "Title", body="Short body"),
+        ]
+        items = fetch_all_open_items()
+        assert items[0]["text"] == "Title\n\nShort body"
+
     @patch("sweep.github_api_get")
     def test_null_body_handled(self, mock_get):
         issue = _make_api_issue(1, "No body")