mirror of
https://github.com/iflytek/skillhub.git
synced 2026-11-16 09:21:18 +00:00
fix(ci): verify complete and fresh DeepWiki crawl results (#1015)
Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>
This commit is contained in:
parent
219dccae2a
commit
16fea87abf
3 changed files with 205 additions and 17 deletions
13
.github/workflows/deepwiki.yml
vendored
13
.github/workflows/deepwiki.yml
vendored
|
|
@ -20,12 +20,19 @@ jobs:
|
|||
env:
|
||||
CRAWLER_DIR: ${{ github.workspace }}/.deepwiki-crawler
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
MAX_CONCURRENCY: '3'
|
||||
# The pinned crawler schedules extra workers before updating its active
|
||||
# count. Use its sequential path to avoid an unbounded scheduling loop.
|
||||
MAX_CONCURRENCY: '1'
|
||||
REQUEST_TIMEOUT: '30000'
|
||||
RETRY_ATTEMPTS: '3'
|
||||
USER_AGENT: DeepWiki-Crawler/1.0
|
||||
|
||||
steps:
|
||||
- name: Check out workflow helpers
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check out pinned crawler source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
|
|
@ -51,6 +58,4 @@ jobs:
|
|||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$REPOSITORY" =~ ^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$ ]]
|
||||
cd "$CRAWLER_DIR"
|
||||
node scripts/check-deepwiki.js "$REPOSITORY"
|
||||
node scripts/crawl-deepwiki.js "$REPOSITORY"
|
||||
node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs" "$CRAWLER_DIR" "$REPOSITORY"
|
||||
|
|
|
|||
57
scripts/run-deepwiki.cjs
Normal file
57
scripts/run-deepwiki.cjs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env node
|
||||
// The pinned upstream crawler exits zero for failed or empty crawls. Fail closed
|
||||
// on its real result rather than treating repository discovery as documentation.
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const [crawlerDirectory, repository] = process.argv.slice(2);
|
||||
if (!crawlerDirectory || !/^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/.test(repository ?? '')) {
|
||||
console.error('DeepWiki requires a crawler directory and owner/repository.');
|
||||
process.exit(1);
|
||||
}
|
||||
const cwd = path.resolve(crawlerDirectory);
|
||||
const deadline = Date.now() + 9 * 60 * 1000;
|
||||
|
||||
function run(name) {
|
||||
const result = spawnSync(process.execPath, [path.join(cwd, 'scripts', name), repository], {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
timeout: Math.max(1, deadline - Date.now()),
|
||||
});
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
if (result.error || result.status !== 0) {
|
||||
console.error(`DeepWiki ${name} failed or exceeded its execution limit.`);
|
||||
process.exit(result.status > 0 && result.status <= 255 ? result.status : 1);
|
||||
}
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
run('check-deepwiki.js');
|
||||
const content = path.join(cwd, 'deepwiki-content.md');
|
||||
const previousContent = fs.lstatSync(content, { throwIfNoEntry: false });
|
||||
if (previousContent && !previousContent.isFile()) {
|
||||
console.error('DeepWiki crawl refused: output path must not be a symlink or other non-regular file.');
|
||||
process.exit(1);
|
||||
}
|
||||
// Remove only this generated output so a stale file cannot validate a new crawl.
|
||||
if (previousContent) fs.unlinkSync(content);
|
||||
const output = run('crawl-deepwiki.js');
|
||||
const summaries = output.split(/\r?\n/).filter(line => line.startsWith('Crawl complete.'));
|
||||
const match = summaries.length === 1 && summaries[0].match(
|
||||
/^Crawl complete\. Attempted: (\d+), Successful: (\d+), Failed: (\d+)$/,
|
||||
);
|
||||
const counts = match ? match.slice(1).map(Number) : [];
|
||||
if (!match || !counts.every(Number.isSafeInteger) || counts[0] < 1 || counts[1] < 1
|
||||
|| counts[2] !== 0 || counts[0] !== counts[1] + counts[2]) {
|
||||
console.error('DeepWiki crawl failed: missing, inconsistent, empty or unsuccessful page summary.');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!fs.existsSync(content) || !fs.lstatSync(content).isFile()
|
||||
|| !fs.readFileSync(content, 'utf8').trim()) {
|
||||
console.error('DeepWiki crawl failed: no extracted documentation content.');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`DeepWiki verified: ${counts[1]} pages fetched, non-empty documentation content, no failed pages.`);
|
||||
|
|
@ -34,7 +34,7 @@ class DeepWikiWorkflowTest(unittest.TestCase):
|
|||
(self.crawler / "scripts").mkdir(parents=True)
|
||||
self.env = dict(os.environ, CRAWLER_DIR=str(self.crawler),
|
||||
REPOSITORY="iflytek/skillhub", NODE_OPTIONS="--no-global-search-paths",
|
||||
GITHUB_OUTPUT=str(self.base / "output"))
|
||||
GITHUB_OUTPUT=str(self.base / "output"), GITHUB_WORKSPACE=str(ROOT))
|
||||
self.env.pop("NODE_PATH", None)
|
||||
self.env.update(dict(re.findall(
|
||||
r"^ (MAX_CONCURRENCY|REQUEST_TIMEOUT|RETRY_ATTEMPTS): '([^']+)'$",
|
||||
|
|
@ -46,8 +46,12 @@ class DeepWikiWorkflowTest(unittest.TestCase):
|
|||
"require('cheerio');\n"
|
||||
"if (process.env.FAIL_STAGE === '" + name + "') process.exit(17);\n"
|
||||
"if (process.argv[2] !== 'iflytek/skillhub') process.exit(18);\n"
|
||||
"fetch(process.env.TEST_URL).then(text => console.log('" + name
|
||||
+ " ' + text)).catch(() => process.exit(19));\n"
|
||||
"fetch(process.env.TEST_URL).then(text => { console.log('" + name + " ' + text);\n"
|
||||
+ ("console.log(process.env.CRAWL_SUMMARY ?? 'Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n"
|
||||
"if (!process.env.SKIP_CONTENT) require('fs').writeFileSync('deepwiki-content.md', process.env.EMPTY_CONTENT ? '' : '## Page: fixture\\n\\nExtracted content.\\n');\n"
|
||||
"process.exitCode = Number(process.env.CRAWL_EXIT || '0');\n"
|
||||
if name == "crawl" else "")
|
||||
+ "}).catch(() => process.exit(19));\n"
|
||||
)
|
||||
|
||||
def shell(self, script):
|
||||
|
|
@ -66,18 +70,16 @@ class DeepWikiWorkflowTest(unittest.TestCase):
|
|||
"res.on('end', () => resolve(body)); }).on('error', reject); });\n"
|
||||
)
|
||||
|
||||
def test_original_split_directory_failure_then_fixed_resolution(self):
|
||||
self.install_fixture(self.workspace)
|
||||
result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("Cannot find module 'node-fetch'", result.stderr)
|
||||
self.install_fixture(self.crawler)
|
||||
|
||||
def serve_fixture(self, linked=False):
|
||||
self.requests = []
|
||||
requests = self.requests
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
requests.append(self.path)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b"mock-http-ok")
|
||||
self.wfile.write(b'<a href="/child">Child</a>' if linked and self.path == '/'
|
||||
else b"mock-http-ok")
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
|
@ -88,14 +90,136 @@ class DeepWikiWorkflowTest(unittest.TestCase):
|
|||
self.addCleanup(server.server_close)
|
||||
self.addCleanup(server.shutdown)
|
||||
self.env["TEST_URL"] = f"http://127.0.0.1:{server.server_port}/"
|
||||
|
||||
def test_original_split_directory_failure_then_fixed_resolution(self):
|
||||
self.install_fixture(self.workspace)
|
||||
result = self.shell('node "$CRAWLER_DIR/scripts/check-deepwiki.js" "$REPOSITORY"')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("Cannot find module 'node-fetch'", result.stderr)
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok", "crawl mock-http-ok"])
|
||||
self.assertEqual(result.stdout.splitlines()[:2], ["check mock-http-ok", "crawl mock-http-ok"])
|
||||
self.assertIn("Attempted: 2, Successful: 2, Failed: 0", result.stdout)
|
||||
self.env["FAIL_STAGE"] = "crawl"
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertEqual(result.returncode, 17)
|
||||
self.assertEqual(result.stdout.splitlines(), ["check mock-http-ok"])
|
||||
|
||||
def test_crawl_summary_fails_closed(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
summaries = (
|
||||
"Crawl complete. Attempted: 1, Successful: 0, Failed: 1",
|
||||
"Crawl complete. Attempted: 2, Successful: 1, Failed: 1",
|
||||
"Crawl complete. Attempted: 0, Successful: 0, Failed: 0",
|
||||
"",
|
||||
"Crawl complete. Attempted: unknown, Successful: 1, Failed: 0",
|
||||
"Crawl complete. Attempted: 2, Successful: 1, Failed: 0",
|
||||
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n"
|
||||
"Crawl complete. Attempted: 1, Successful: 0, Failed: 1",
|
||||
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0\n"
|
||||
"Crawl complete. Attempted: 1, Successful: 1, Failed: 0",
|
||||
)
|
||||
for summary in summaries:
|
||||
with self.subTest(summary=summary):
|
||||
self.env["CRAWL_SUMMARY"] = summary
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
|
||||
def test_success_without_extracted_content_fails(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
self.env["EMPTY_CONTENT"] = "1"
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
|
||||
def test_success_summary_without_output_fails(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
self.env["SKIP_CONTENT"] = "1"
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertIn('Successful: 2, Failed: 0', result.stdout)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
|
||||
def test_old_output_cannot_satisfy_fresh_crawl(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
content = self.crawler / 'deepwiki-content.md'
|
||||
content.write_text('Old cached documentation.\n')
|
||||
self.env["SKIP_CONTENT"] = "1"
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
self.assertFalse(content.exists())
|
||||
|
||||
def test_old_regular_output_is_replaced_by_fresh_crawl(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
content = self.crawler / 'deepwiki-content.md'
|
||||
content.write_text('Old cached documentation.\n')
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn('Extracted content.', content.read_text())
|
||||
self.assertNotIn('Old cached documentation.', content.read_text())
|
||||
|
||||
def test_output_symlink_is_rejected_without_touching_target(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
target = self.base / 'protected-target.md'
|
||||
target.write_text('Do not modify this target.\n')
|
||||
content = self.crawler / 'deepwiki-content.md'
|
||||
absent = self.base / 'absent-target.md'
|
||||
for link_target in (target, absent):
|
||||
with self.subTest(target=link_target.name):
|
||||
content.symlink_to(link_target)
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn('crawl mock-http-ok', result.stdout)
|
||||
self.assertTrue(content.is_symlink())
|
||||
self.assertEqual(target.read_text(), 'Do not modify this target.\n')
|
||||
self.assertFalse(absent.exists())
|
||||
content.unlink()
|
||||
|
||||
def test_output_directory_is_rejected_without_cleanup(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
content = self.crawler / 'deepwiki-content.md'
|
||||
content.mkdir()
|
||||
retained = content / 'retained.txt'
|
||||
retained.write_text('Retain directory contents.\n')
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn('crawl mock-http-ok', result.stdout)
|
||||
self.assertTrue(content.is_dir())
|
||||
self.assertEqual(retained.read_text(), 'Retain directory contents.\n')
|
||||
|
||||
def test_success_summary_does_not_mask_process_failure(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture()
|
||||
self.env["CRAWL_EXIT"] = "23"
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertIn("Successful: 2, Failed: 0", result.stdout)
|
||||
self.assertEqual(result.returncode, 23, result.stderr)
|
||||
|
||||
def test_wrapper_preserves_multi_page_crawl(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.serve_fixture(linked=True)
|
||||
(self.crawler / "scripts/crawl-deepwiki.js").write_text(
|
||||
"const fetch = require('node-fetch'); const fs = require('node:fs');\n"
|
||||
"(async () => {\n"
|
||||
"const root = await fetch(process.env.TEST_URL);\n"
|
||||
"if (!root.includes('/child') || process.env.MAX_CONCURRENCY !== '1') process.exit(21);\n"
|
||||
"const child = await fetch(new URL('/child', process.env.TEST_URL).href);\n"
|
||||
"fs.writeFileSync('deepwiki-content.md', root + child);\n"
|
||||
"console.log('Crawl complete. Attempted: 2, Successful: 2, Failed: 0');\n"
|
||||
"})().catch(() => process.exit(22));\n"
|
||||
)
|
||||
result = self.shell(run_block("Check repository and crawl DeepWiki"))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(self.requests, ['/', '/', '/child'])
|
||||
self.assertIn('2 pages fetched', result.stdout)
|
||||
|
||||
def test_failures_propagate_and_bad_repository_is_rejected(self):
|
||||
self.install_fixture(self.crawler)
|
||||
self.env["FAIL_STAGE"] = "check"
|
||||
|
|
@ -133,9 +257,11 @@ class DeepWikiWorkflowTest(unittest.TestCase):
|
|||
"ref: b9fc8378235f298d736d12c5f4651d6ea0820c61",
|
||||
"contents: read", "REPOSITORY: ${{ github.repository }}"):
|
||||
self.assertIn(expected, WORKFLOW)
|
||||
self.assertEqual(self.env["MAX_CONCURRENCY"], "3")
|
||||
self.assertEqual(self.env["MAX_CONCURRENCY"], "1")
|
||||
self.assertEqual(self.env["REQUEST_TIMEOUT"], "30000")
|
||||
self.assertEqual(self.env["RETRY_ATTEMPTS"], "3")
|
||||
pr = (ROOT / ".github/workflows/pr-scripts.yml").read_text()
|
||||
self.assertIn("'.github/workflows/deepwiki.yml'", pr)
|
||||
self.assertIn("-p 'test_deepwiki_workflow.py'", pr)
|
||||
self.assertIn("'scripts/**'", pr)
|
||||
self.assertIn('node "$GITHUB_WORKSPACE/scripts/run-deepwiki.cjs"', WORKFLOW)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue