fix(ci): verify complete and fresh DeepWiki crawl results (#1015)

Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
XiaoSeS 2026-10-25 18:52:03 +08:00 • committed by GitHub
parent 219dccae2a
commit 16fea87abf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 205 additions and 17 deletions

View file

@ -20,12 +20,19 @@ jobs:
env:
CRAWLER_DIR: ${{ github.workspace }}/.deepwiki-crawler
REPOSITORY: ${{ github.repository }}
MAX_CONCURRENCY: '3'
# The pinned crawler schedules extra workers before updating its active
# count. Use its sequential path to avoid an unbounded scheduling loop.
MAX_CONCURRENCY: '1'
REQUEST_TIMEOUT: '30000'
RETRY_ATTEMPTS: '3'
USER_AGENT: DeepWiki-Crawler/1.0
steps:
- name: Check out workflow helpers
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check out pinned crawler source
uses: actions/checkout@v4
with:
@ -51,6 +58,4 @@ jobs:
run: |
set -euo pipefail
[[ "$REPOSITORY" =~ ^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$ ]]
cd "$CRAWLER_DIR"
node scripts/check-deepwiki.js "$REPOSITORY"
node scripts/crawl-deepwiki.js "$REPOSITORY"
node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs" "$CRAWLER_DIR" "$REPOSITORY"

57
scripts/run-deepwiki.cjs Normal file
View file

@ -0,0 +1,57 @@
#!/usr/bin/env node
// The pinned upstream crawler exits zero for failed or empty crawls. Fail closed
// on its real result rather than treating repository discovery as documentation.
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const [crawlerDirectory, repository] = process.argv.slice(2);
if (!crawlerDirectory || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(repository ?? '')) {
console.error('DeepWiki requires a crawler directory and owner/repository.');
process.exit(1);
}
const cwd = path.resolve(crawlerDirectory);
const deadline = Date.now() + 9 * 60 * 1000;
function run(name) {
const result = spawnSync(process.execPath, [path.join(cwd, 'scripts', name), repository], {
cwd,
encoding: 'utf8',
maxBuffer: 4 * 1024 * 1024,
timeout: Math.max(1, deadline - Date.now()),
});
if (result.stdout) process.stdout.write(result.stdout);
if (result.stderr) process.stderr.write(result.stderr);
if (result.error || result.status !== 0) {
console.error(`DeepWiki ${name} failed or exceeded its execution limit.`);
process.exit(result.status > 0 && result.status <= 255 ? result.status : 1);
}
return result.stdout;
}
run('check-deepwiki.js');
const content = path.join(cwd, 'deepwiki-content.md');
const previousContent = fs.lstatSync(content, { throwIfNoEntry: false });
if (previousContent && !previousContent.isFile()) {
console.error('DeepWiki crawl refused: output path must not be a symlink or other non-regular file.');
process.exit(1);
}
// Remove only this generated output so a stale file cannot validate a new crawl.
if (previousContent) fs.unlinkSync(content);
const output = run('crawl-deepwiki.js');
const summaries = output.split(/\r?\n/).filter(line => line.startsWith('Crawl complete.'));
const match = summaries.length === 1 && summaries[0].match(
/^Crawl complete\. Attempted: (\d+), Successful: (\d+), Failed: (\d+)$/,
);
const counts = match ? match.slice(1).map(Number) : [];
if (!match || !counts.every(Number.isSafeInteger) || counts[0] < 1 || counts[1] < 1
|| counts[2] !== 0 || counts[0] !== counts[1] + counts[2]) {
console.error('DeepWiki crawl failed: missing, inconsistent, empty or unsuccessful page summary.');
process.exit(1);
}
if (!fs.existsSync(content) || !fs.lstatSync(content).isFile()
|| !fs.readFileSync(content, 'utf8').trim()) {
console.error('DeepWiki crawl failed: no extracted documentation content.');
process.exit(1);
}
console.log(`DeepWiki verified: ${counts[1]} pages fetched, non-empty documentation content, no failed pages.`);

View file

@ -34,7 +34,7 @@ class DeepWikiWorkflowTest(unittest.TestCase):
(self.crawler / "scripts").mkdir(parents=True)
self.env = dict(os.environ, CRAWLER_DIR=str(self.crawler),
REPOSITORY="iflytek/skillhub", NODE_OPTIONS="--no-global-search-paths",
GITHUB_OUTPUT=str(self.base / "output"))
GITHUB_OUTPUT=str(self.base / "output"), GITHUB_WORKSPACE=str(ROOT))
self.env.pop("NODE_PATH", None)
self.env.update(dict(re.findall(
r"^ (MAX_CONCURRENCY|REQUEST_TIMEOUT|RETRY_ATTEMPTS): '([^']+)'$",
@ -46,8 +46,12 @@ class DeepWikiWorkflowTest(unittest.TestCase):
"require('cheerio');\n"
"if (process.env.FAIL_STAGE === '" + name + "') process.exit(17);\n"
"if (process.argv[2] !== 'iflytek/skillhub') process.exit(18);\n"
"fetch(process.env.TEST_URL).then(text => console.log('" + name
+ " ' + text)).catch(() => process.exit(19));\n"
"fetch(process.env.TEST_URL).then(text => { console.log('" + name + " ' + text);\n"
+ ("console.log(process.env.CRAWL_SUMMARY ?? 'Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n"
"if (!process.env.SKIP_CONTENT) require('fs').writeFileSync('deepwiki-content.md', process.env.EMPTY_CONTENT ? '' : '## Page: fixture\\n\\nExtracted content.\\n');\n"
"process.exitCode = Number(process.env.CRAWL_EXIT || '0');\n"
if name == "crawl" else "")
+ "}).catch(() => process.exit(19));\n"
)
def shell(self, script):
@ -66,18 +70,16 @@ class DeepWikiWorkflowTest(unittest.TestCase):
"res.on('end', () => resolve(body)); }).on('error', reject); });\n"
)
def test_original_split_directory_failure_then_fixed_resolution(self):
self.install_fixture(self.workspace)
result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"')
self.assertNotEqual(result.returncode, 0)
self.assertIn("Cannot find module 'node-fetch'", result.stderr)
self.install_fixture(self.crawler)
def serve_fixture(self, linked=False):
self.requests = []
requests = self.requests
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
requests.append(self.path)
self.send_response(200)
self.end_headers()
self.wfile.write(b"mock-http-ok")
self.wfile.write(b'<a href="/child">Child</a>' if linked and self.path == '/'
else b"mock-http-ok")
def log_message(self, *_):
pass
@ -88,14 +90,136 @@ class DeepWikiWorkflowTest(unittest.TestCase):
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
self.env["TEST_URL"] = f"http://127.0.0.1:{server.server_port}/"
def test_original_split_directory_failure_then_fixed_resolution(self):
self.install_fixture(self.workspace)
result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"')
self.assertNotEqual(result.returncode, 0)
self.assertIn("Cannot find module 'node-fetch'", result.stderr)
self.install_fixture(self.crawler)
self.serve_fixture()
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok", "crawl mock-http-ok"])
self.assertEqual(result.stdout.splitlines()[:2], ["check mock-http-ok", "crawl mock-http-ok"])
self.assertIn("Attempted: 2, Successful: 2, Failed: 0", result.stdout)
self.env["FAIL_STAGE"] = "crawl"
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertEqual(result.returncode, 17)
self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok"])
def test_crawl_summary_fails_closed(self):
self.install_fixture(self.crawler)
self.serve_fixture()
summaries = (
"Crawl complete. Attempted: 1, Successful: 0, Failed: 1",
"Crawl complete. Attempted: 2, Successful: 1, Failed: 1",
"Crawl complete. Attempted: 0, Successful: 0, Failed: 0",
"",
"Crawl complete. Attempted: unknown, Successful: 1, Failed: 0",
"Crawl complete. Attempted: 2, Successful: 1, Failed: 0",
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n"
"Crawl complete. Attempted: 1, Successful: 0, Failed: 1",
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n"
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0",
)
for summary in summaries:
with self.subTest(summary=summary):
self.env["CRAWL_SUMMARY"] = summary
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertNotEqual(result.returncode, 0, result.stdout)
def test_success_without_extracted_content_fails(self):
self.install_fixture(self.crawler)
self.serve_fixture()
self.env["EMPTY_CONTENT"] = "1"
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertNotEqual(result.returncode, 0, result.stdout)
def test_success_summary_without_output_fails(self):
self.install_fixture(self.crawler)
self.serve_fixture()
self.env["SKIP_CONTENT"] = "1"
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertIn('Successful: 2, Failed: 0', result.stdout)
self.assertNotEqual(result.returncode, 0)
def test_old_output_cannot_satisfy_fresh_crawl(self):
self.install_fixture(self.crawler)
self.serve_fixture()
content = self.crawler / 'deepwiki-content.md'
content.write_text('Old cached documentation.\n')
self.env["SKIP_CONTENT"] = "1"
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertFalse(content.exists())
def test_old_regular_output_is_replaced_by_fresh_crawl(self):
self.install_fixture(self.crawler)
self.serve_fixture()
content = self.crawler / 'deepwiki-content.md'
content.write_text('Old cached documentation.\n')
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('Extracted content.', content.read_text())
self.assertNotIn('Old cached documentation.', content.read_text())
def test_output_symlink_is_rejected_without_touching_target(self):
self.install_fixture(self.crawler)
self.serve_fixture()
target = self.base / 'protected-target.md'
target.write_text('Do not modify this target.\n')
content = self.crawler / 'deepwiki-content.md'
absent = self.base / 'absent-target.md'
for link_target in (target, absent):
with self.subTest(target=link_target.name):
content.symlink_to(link_target)
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertNotEqual(result.returncode, 0)
self.assertNotIn('crawl mock-http-ok', result.stdout)
self.assertTrue(content.is_symlink())
self.assertEqual(target.read_text(), 'Do not modify this target.\n')
self.assertFalse(absent.exists())
content.unlink()
def test_output_directory_is_rejected_without_cleanup(self):
self.install_fixture(self.crawler)
self.serve_fixture()
content = self.crawler / 'deepwiki-content.md'
content.mkdir()
retained = content / 'retained.txt'
retained.write_text('Retain directory contents.\n')
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertNotEqual(result.returncode, 0)
self.assertNotIn('crawl mock-http-ok', result.stdout)
self.assertTrue(content.is_dir())
self.assertEqual(retained.read_text(), 'Retain directory contents.\n')
def test_success_summary_does_not_mask_process_failure(self):
self.install_fixture(self.crawler)
self.serve_fixture()
self.env["CRAWL_EXIT"] = "23"
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertIn("Successful: 2, Failed: 0", result.stdout)
self.assertEqual(result.returncode, 23, result.stderr)
def test_wrapper_preserves_multi_page_crawl(self):
self.install_fixture(self.crawler)
self.serve_fixture(linked=True)
(self.crawler / "scripts/crawl-deepwiki.js").write_text(
"const fetch = require('node-fetch'); const fs = require('node:fs');\n"
"(async () => {\n"
"const root = await fetch(process.env.TEST_URL);\n"
"if (!root.includes('/child') || process.env.MAX_CONCURRENCY !== '1') process.exit(21);\n"
"const child = await fetch(new URL('/child', process.env.TEST_URL).href);\n"
"fs.writeFileSync('deepwiki-content.md', root + child);\n"
"console.log('Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n"
"})().catch(() => process.exit(22));\n"
)
result = self.shell(run_block("Check repository and crawl DeepWiki"))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(self.requests, ['/', '/', '/child'])
self.assertIn('2 pages fetched', result.stdout)
def test_failures_propagate_and_bad_repository_is_rejected(self):
self.install_fixture(self.crawler)
self.env["FAIL_STAGE"] = "check"
@ -133,9 +257,11 @@ class DeepWikiWorkflowTest(unittest.TestCase):
"ref: b9fc8378235f298d736d12c5f4651d6ea0820c61",
"contents: read", "REPOSITORY: ${{ github.repository }}"):
self.assertIn(expected, WORKFLOW)
self.assertEqual(self.env["MAX_CONCURRENCY"], "3")
self.assertEqual(self.env["MAX_CONCURRENCY"], "1")
self.assertEqual(self.env["REQUEST_TIMEOUT"], "30000")
self.assertEqual(self.env["RETRY_ATTEMPTS"], "3")
pr = (ROOT / ".github/workflows/pr-scripts.yml").read_text()
self.assertIn("'.github/workflows/deepwiki.yml'", pr)
self.assertIn("-p 'test_deepwiki_workflow.py'", pr)
self.assertIn("'scripts/**'", pr)
self.assertIn('node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs"', WORKFLOW)