diff --git a/.github/workflows/deepwiki.yml b/.github/workflows/deepwiki.yml index c843ec191..cba91681a 100644 --- a/.github/workflows/deepwiki.yml +++ b/.github/workflows/deepwiki.yml @@ -20,12 +20,19 @@ jobs: env: CRAWLER_DIR: ${{ github.workspace }}/.deepwiki-crawler REPOSITORY: ${{ github.repository }} - MAX_CONCURRENCY: '3' + # The pinned crawler schedules extra workers before updating its active + # count. Use its sequential path to avoid an unbounded scheduling loop. + MAX_CONCURRENCY: '1' REQUEST_TIMEOUT: '30000' RETRY_ATTEMPTS: '3' USER_AGENT: DeepWiki-Crawler/1.0 steps: + - name: Check out workflow helpers + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Check out pinned crawler source uses: actions/checkout@v4 with: @@ -51,6 +58,4 @@ jobs: run: | set -euo pipefail [[ "$REPOSITORY" =~ ^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$ ]] - cd "$CRAWLER_DIR" - node scripts/check-deepwiki.js "$REPOSITORY" - node scripts/crawl-deepwiki.js "$REPOSITORY" + node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs" "$CRAWLER_DIR" "$REPOSITORY" diff --git a/scripts/run-deepwiki.cjs b/scripts/run-deepwiki.cjs new file mode 100644 index 000000000..760d60112 --- /dev/null +++ b/scripts/run-deepwiki.cjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +// The pinned upstream crawler exits zero for failed or empty crawls. Fail closed +// on its real result rather than treating repository discovery as documentation. +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const [crawlerDirectory, repository] = process.argv.slice(2); +if (!crawlerDirectory || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(repository ?? '')) { + console.error('DeepWiki requires a crawler directory and owner/repository.'); + process.exit(1); +} +const cwd = path.resolve(crawlerDirectory); +const deadline = Date.now() + 9 * 60 * 1000; + +function run(name) { + const result = spawnSync(process.execPath, [path.join(cwd, 'scripts', name), repository], { + cwd, + encoding: 'utf8', + maxBuffer: 4 * 1024 * 1024, + timeout: Math.max(1, deadline - Date.now()), + }); + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + if (result.error || result.status !== 0) { + console.error(`DeepWiki ${name} failed or exceeded its execution limit.`); + process.exit(result.status > 0 && result.status <= 255 ? result.status : 1); + } + return result.stdout; +} + +run('check-deepwiki.js'); +const content = path.join(cwd, 'deepwiki-content.md'); +const previousContent = fs.lstatSync(content, { throwIfNoEntry: false }); +if (previousContent && !previousContent.isFile()) { + console.error('DeepWiki crawl refused: output path must not be a symlink or other non-regular file.'); + process.exit(1); +} +// Remove only this generated output so a stale file cannot validate a new crawl. +if (previousContent) fs.unlinkSync(content); +const output = run('crawl-deepwiki.js'); +const summaries = output.split(/\r?\n/).filter(line => line.startsWith('Crawl complete.')); +const match = summaries.length === 1 && summaries[0].match( + /^Crawl complete\. Attempted: (\d+), Successful: (\d+), Failed: (\d+)$/, +); +const counts = match ? match.slice(1).map(Number) : []; +if (!match || !counts.every(Number.isSafeInteger) || counts[0] < 1 || counts[1] < 1 + || counts[2] !== 0 || counts[0] !== counts[1] + counts[2]) { + console.error('DeepWiki crawl failed: missing, inconsistent, empty or unsuccessful page summary.'); + process.exit(1); +} +if (!fs.existsSync(content) || !fs.lstatSync(content).isFile() + || !fs.readFileSync(content, 'utf8').trim()) { + console.error('DeepWiki crawl failed: no extracted documentation content.'); + process.exit(1); +} +console.log(`DeepWiki verified: ${counts[1]} pages fetched, non-empty documentation content, no failed pages.`); diff --git a/scripts/tests/test_deepwiki_workflow.py b/scripts/tests/test_deepwiki_workflow.py index 598cbe8e2..ede8d3a9d 100644 --- a/scripts/tests/test_deepwiki_workflow.py +++ b/scripts/tests/test_deepwiki_workflow.py @@ -34,7 +34,7 @@ class DeepWikiWorkflowTest(unittest.TestCase): (self.crawler / "scripts").mkdir(parents=True) self.env = dict(os.environ, CRAWLER_DIR=str(self.crawler), REPOSITORY="iflytek/skillhub", NODE_OPTIONS="--no-global-search-paths", - GITHUB_OUTPUT=str(self.base / "output")) + GITHUB_OUTPUT=str(self.base / "output"), GITHUB_WORKSPACE=str(ROOT)) self.env.pop("NODE_PATH", None) self.env.update(dict(re.findall( r"^ (MAX_CONCURRENCY|REQUEST_TIMEOUT|RETRY_ATTEMPTS): '([^']+)'$", @@ -46,8 +46,12 @@ class DeepWikiWorkflowTest(unittest.TestCase): "require('cheerio');\n" "if (process.env.FAIL_STAGE === '" + name + "') process.exit(17);\n" "if (process.argv[2] !== 'iflytek/skillhub') process.exit(18);\n" - "fetch(process.env.TEST_URL).then(text => console.log('" + name - + " ' + text)).catch(() => process.exit(19));\n" + "fetch(process.env.TEST_URL).then(text => { console.log('" + name + " ' + text);\n" + + ("console.log(process.env.CRAWL_SUMMARY ?? 'Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n" + "if (!process.env.SKIP_CONTENT) require('fs').writeFileSync('deepwiki-content.md', process.env.EMPTY_CONTENT ? '' : '## Page: fixture\\n\\nExtracted content.\\n');\n" + "process.exitCode = Number(process.env.CRAWL_EXIT || '0');\n" + if name == "crawl" else "") + + "}).catch(() => process.exit(19));\n" ) def shell(self, script): @@ -66,18 +70,16 @@ class DeepWikiWorkflowTest(unittest.TestCase): "res.on('end', () => resolve(body)); }).on('error', reject); });\n" ) - def test_original_split_directory_failure_then_fixed_resolution(self): - self.install_fixture(self.workspace) - result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"') - self.assertNotEqual(result.returncode, 0) - self.assertIn("Cannot find module 'node-fetch'", result.stderr) - self.install_fixture(self.crawler) - + def serve_fixture(self, linked=False): + self.requests = [] + requests = self.requests class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): + requests.append(self.path) self.send_response(200) self.end_headers() - self.wfile.write(b"mock-http-ok") + self.wfile.write(b'Child' if linked and self.path == '/' + else b"mock-http-ok") def log_message(self, *_): pass @@ -88,14 +90,136 @@ class DeepWikiWorkflowTest(unittest.TestCase): self.addCleanup(server.server_close) self.addCleanup(server.shutdown) self.env["TEST_URL"] = f"http://127.0.0.1:{server.server_port}/" + + def test_original_split_directory_failure_then_fixed_resolution(self): + self.install_fixture(self.workspace) + result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"') + self.assertNotEqual(result.returncode, 0) + self.assertIn("Cannot find module 'node-fetch'", result.stderr) + self.install_fixture(self.crawler) + self.serve_fixture() result = self.shell(run_block("Check repository and crawl DeepWiki")) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok", "crawl mock-http-ok"]) + self.assertEqual(result.stdout.splitlines()[:2], ["check mock-http-ok", "crawl mock-http-ok"]) + self.assertIn("Attempted: 2, Successful: 2, Failed: 0", result.stdout) self.env["FAIL_STAGE"] = "crawl" result = self.shell(run_block("Check repository and crawl DeepWiki")) self.assertEqual(result.returncode, 17) self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok"]) + def test_crawl_summary_fails_closed(self): + self.install_fixture(self.crawler) + self.serve_fixture() + summaries = ( + "Crawl complete. Attempted: 1, Successful: 0, Failed: 1", + "Crawl complete. Attempted: 2, Successful: 1, Failed: 1", + "Crawl complete. Attempted: 0, Successful: 0, Failed: 0", + "", + "Crawl complete. Attempted: unknown, Successful: 1, Failed: 0", + "Crawl complete. Attempted: 2, Successful: 1, Failed: 0", + "Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n" + "Crawl complete. Attempted: 1, Successful: 0, Failed: 1", + "Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n" + "Crawl complete. Attempted: 1, Successful: 1, Failed: 0", + ) + for summary in summaries: + with self.subTest(summary=summary): + self.env["CRAWL_SUMMARY"] = summary + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertNotEqual(result.returncode, 0, result.stdout) + + def test_success_without_extracted_content_fails(self): + self.install_fixture(self.crawler) + self.serve_fixture() + self.env["EMPTY_CONTENT"] = "1" + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertNotEqual(result.returncode, 0, result.stdout) + + def test_success_summary_without_output_fails(self): + self.install_fixture(self.crawler) + self.serve_fixture() + self.env["SKIP_CONTENT"] = "1" + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertIn('Successful: 2, Failed: 0', result.stdout) + self.assertNotEqual(result.returncode, 0) + + def test_old_output_cannot_satisfy_fresh_crawl(self): + self.install_fixture(self.crawler) + self.serve_fixture() + content = self.crawler / 'deepwiki-content.md' + content.write_text('Old cached documentation.\n') + self.env["SKIP_CONTENT"] = "1" + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(content.exists()) + + def test_old_regular_output_is_replaced_by_fresh_crawl(self): + self.install_fixture(self.crawler) + self.serve_fixture() + content = self.crawler / 'deepwiki-content.md' + content.write_text('Old cached documentation.\n') + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('Extracted content.', content.read_text()) + self.assertNotIn('Old cached documentation.', content.read_text()) + + def test_output_symlink_is_rejected_without_touching_target(self): + self.install_fixture(self.crawler) + self.serve_fixture() + target = self.base / 'protected-target.md' + target.write_text('Do not modify this target.\n') + content = self.crawler / 'deepwiki-content.md' + absent = self.base / 'absent-target.md' + for link_target in (target, absent): + with self.subTest(target=link_target.name): + content.symlink_to(link_target) + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('crawl mock-http-ok', result.stdout) + self.assertTrue(content.is_symlink()) + self.assertEqual(target.read_text(), 'Do not modify this target.\n') + self.assertFalse(absent.exists()) + content.unlink() + + def test_output_directory_is_rejected_without_cleanup(self): + self.install_fixture(self.crawler) + self.serve_fixture() + content = self.crawler / 'deepwiki-content.md' + content.mkdir() + retained = content / 'retained.txt' + retained.write_text('Retain directory contents.\n') + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('crawl mock-http-ok', result.stdout) + self.assertTrue(content.is_dir()) + self.assertEqual(retained.read_text(), 'Retain directory contents.\n') + + def test_success_summary_does_not_mask_process_failure(self): + self.install_fixture(self.crawler) + self.serve_fixture() + self.env["CRAWL_EXIT"] = "23" + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertIn("Successful: 2, Failed: 0", result.stdout) + self.assertEqual(result.returncode, 23, result.stderr) + + def test_wrapper_preserves_multi_page_crawl(self): + self.install_fixture(self.crawler) + self.serve_fixture(linked=True) + (self.crawler / "scripts/crawl-deepwiki.js").write_text( + "const fetch = require('node-fetch'); const fs = require('node:fs');\n" + "(async () => {\n" + "const root = await fetch(process.env.TEST_URL);\n" + "if (!root.includes('/child') || process.env.MAX_CONCURRENCY !== '1') process.exit(21);\n" + "const child = await fetch(new URL('/child', process.env.TEST_URL).href);\n" + "fs.writeFileSync('deepwiki-content.md', root + child);\n" + "console.log('Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n" + "})().catch(() => process.exit(22));\n" + ) + result = self.shell(run_block("Check repository and crawl DeepWiki")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.requests, ['/', '/', '/child']) + self.assertIn('2 pages fetched', result.stdout) + def test_failures_propagate_and_bad_repository_is_rejected(self): self.install_fixture(self.crawler) self.env["FAIL_STAGE"] = "check" @@ -133,9 +257,11 @@ class DeepWikiWorkflowTest(unittest.TestCase): "ref: b9fc8378235f298d736d12c5f4651d6ea0820c61", "contents: read", "REPOSITORY: ${{ github.repository }}"): self.assertIn(expected, WORKFLOW) - self.assertEqual(self.env["MAX_CONCURRENCY"], "3") + self.assertEqual(self.env["MAX_CONCURRENCY"], "1") self.assertEqual(self.env["REQUEST_TIMEOUT"], "30000") self.assertEqual(self.env["RETRY_ATTEMPTS"], "3") pr = (ROOT / ".github/workflows/pr-scripts.yml").read_text() self.assertIn("'.github/workflows/deepwiki.yml'", pr) self.assertIn("-p 'test_deepwiki_workflow.py'", pr) + self.assertIn("'scripts/**'", pr) + self.assertIn('node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs"', WORKFLOW)