fix(ci): parse paginated gh api output without splitting on unicode line breaks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/scripts/close_duplicate_issues.py

@@ -39,6 +39,23 @@ def gh(*args: str) -> str:
     return result.stdout
 
 
+def parse_concatenated_json(raw: str) -> list[dict]:
+    """Parse the concatenated JSON documents that `gh api --paginate` emits."""
+    decoder = json.JSONDecoder()
+    issues: list[dict] = []
+    idx = 0
+    while idx < len(raw):
+        if raw[idx].isspace():
+            idx += 1
+            continue
+        parsed, idx = decoder.raw_decode(raw, idx)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+    return issues
+
+
 def fetch_open_issues(repo: str | None) -> list[dict]:
     """Fetch all open issues (excluding PRs) via gh api --paginate."""
     if repo:
@@ -49,18 +66,7 @@ def fetch_open_issues(repo: str | None) -> list[dict]:
         endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
     cmd = ["api", "--paginate", endpoint]
 
-    raw = gh(*cmd)
-    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
-    issues = []
-    for line in raw.strip().splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        parsed = json.loads(line)
-        if isinstance(parsed, list):
-            issues.extend(parsed)
-        else:
-            issues.append(parsed)
+    issues = parse_concatenated_json(gh(*cmd))
 
     # Filter out pull requests (they also appear in the issues endpoint)
     return [i for i in issues if "pull_request" not in i]

tests/test_litellm/test_github_close_duplicate_issues.py

@@ -0,0 +1,46 @@
+"""Unit tests for `.github/scripts/close_duplicate_issues.py`."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+SCRIPT_PATH = (
+    Path(__file__).resolve().parents[2]
+    / ".github"
+    / "scripts"
+    / "close_duplicate_issues.py"
+)
+
+
+@pytest.fixture(scope="module")
+def dedupe_module():
+    spec = importlib.util.spec_from_file_location("close_duplicate_issues", SCRIPT_PATH)
+    assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}"
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["close_duplicate_issues"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_parse_concatenated_json_joins_paginated_arrays(dedupe_module):
+    page_one = json.dumps([{"number": 1, "title": "a"}, {"number": 2, "title": "b"}])
+    page_two = json.dumps([{"number": 3, "title": "c"}])
+    issues = dedupe_module.parse_concatenated_json(page_one + page_two)
+    assert [i["number"] for i in issues] == [1, 2, 3]
+
+
+@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\x85"])
+def test_parse_concatenated_json_survives_unicode_line_breaks_in_bodies(
+    dedupe_module, separator
+):
+    body = f"first{separator}second"
+    page_one = json.dumps([{"number": 1, "title": "a", "body": body}])
+    page_two = json.dumps([{"number": 2, "title": "b", "body": "plain"}])
+    issues = dedupe_module.parse_concatenated_json(page_one + page_two)
+    assert [i["number"] for i in issues] == [1, 2]
+    assert issues[0]["body"] == body