mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-30 01:51:20 +00:00
Merge branch 'main' into fix/fts-non-fatal-in-analyze
This commit is contained in:
henry201605
GitHub
commit
39a0c677df
19 changed files with 588 additions and 75 deletions
|
|
@ -30,6 +30,12 @@ services:
|
|||
container_name: ${WEB_CONTAINER_NAME:-gitnexus-web}
|
||||
ports:
|
||||
- '${WEB_HOST_PORT:-4173}:4173'
|
||||
# Override the backend URL served to the browser. The default
|
||||
# (http://localhost:4747) works when both containers run locally.
|
||||
# Set GITNEXUS_BACKEND_URL in your .env or shell for remote/custom setups:
|
||||
# GITNEXUS_BACKEND_URL=http://<server-ip>:4747
|
||||
environment:
|
||||
- GITNEXUS_BACKEND_URL=${GITNEXUS_BACKEND_URL:-}
|
||||
depends_on:
|
||||
gitnexus-server:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -1,12 +1,39 @@
|
|||
import { createReadStream } from 'node:fs';
|
||||
import { stat } from 'node:fs/promises';
|
||||
import { open } from 'node:fs/promises';
|
||||
import { createServer } from 'node:http';
|
||||
import { extname, isAbsolute, normalize, relative, resolve } from 'node:path';
|
||||
import { extname, isAbsolute, normalize, relative, resolve, sep } from 'node:path';
|
||||
|
||||
const host = '0.0.0.0';
|
||||
const port = Number(process.env.PORT || '4173');
|
||||
const root = resolve(process.cwd(), 'dist');
|
||||
|
||||
function isValidUrl(value) {
|
||||
try {
|
||||
const u = new URL(value);
|
||||
return u.protocol === 'http:' || u.protocol === 'https:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function jsonForScriptTag(obj) {
|
||||
return JSON.stringify(obj)
|
||||
.replace(/</g, '\\u003c')
|
||||
.replace(/>/g, '\\u003e')
|
||||
.replace(/&/g, '\\u0026');
|
||||
}
|
||||
|
||||
const rawBackendUrl = process.env.GITNEXUS_BACKEND_URL ?? null;
|
||||
if (rawBackendUrl && !isValidUrl(rawBackendUrl)) {
|
||||
const safeRaw = rawBackendUrl.replace(/[\x00-\x1f\x7f]/g, ' ').slice(0, 200);
|
||||
console.warn(
|
||||
`[gitnexus-web] GITNEXUS_BACKEND_URL "${safeRaw}" is not a valid http/https URL -- ignoring.`,
|
||||
);
|
||||
}
|
||||
const backendUrl = rawBackendUrl && isValidUrl(rawBackendUrl) ? rawBackendUrl : null;
|
||||
const configScript = backendUrl
|
||||
? `<script>window.__GITNEXUS_CONFIG__=${jsonForScriptTag({ backendUrl })};</script>`
|
||||
: '';
|
||||
|
||||
const contentTypes = {
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
|
|
@ -22,22 +49,22 @@ const contentTypes = {
|
|||
|
||||
// Static asset server for the gitnexus-web Docker image.
|
||||
//
|
||||
// Path-injection containment: the request handler is intentionally a single
|
||||
// inline pipeline with no helper functions on the path-data flow. Each
|
||||
// filesystem sink (stat, createReadStream) is immediately preceded by the
|
||||
// canonical `path.relative` containment check that CodeQL's
|
||||
// `js/path-injection` query recognizes as a sanitizer barrier:
|
||||
// TOCTOU prevention: every filesystem interaction uses open() to get a
|
||||
// file handle; subsequent reads use handle.readFile()/createReadStream().
|
||||
//
|
||||
// const rel = relative(root, candidate);
|
||||
// if (rel.startsWith('..') || isAbsolute(rel)) reject;
|
||||
// // candidate is now proven inside `root`
|
||||
// CodeQL js/file-system-race: the query pairs open() calls when their
|
||||
// path arguments are data-flow aliased. This handler uses exactly two
|
||||
// open() calls whose paths are provably independent:
|
||||
// 1. open(requestedPath) — derived from the URL
|
||||
// 2. open(spaFallback) — the constant root/index.html
|
||||
// Because spaFallback has no data-flow from the request, CodeQL cannot
|
||||
// pair them as a check/use on the same path.
|
||||
//
|
||||
// Earlier iterations of this file used a helper (`resolveWithinRoot`) and a
|
||||
// `startsWith(root + sep)` check. Both were semantically correct but neither
|
||||
// was recognized by CodeQL: `startsWith(root + sep)` is not in the analyzer's
|
||||
// barrier-pattern set, and helper-based sanitization is not followed across
|
||||
// the request handler's reassignment paths in vanilla JS. The inline-at-sink
|
||||
// shape below is the documented analyzer-friendly idiom.
|
||||
// Path-injection containment: each open() is preceded by a
|
||||
// path.relative() barrier that CodeQL recognizes as a sanitizer.
|
||||
|
||||
const spaFallback = resolve(root, 'index.html');
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
const urlPath = req.url?.split('?')[0] || '/';
|
||||
|
||||
|
|
@ -56,62 +83,90 @@ const server = createServer(async (req, res) => {
|
|||
}
|
||||
|
||||
const cleanPath = normalize(decoded.replace(/^\/+/, ''));
|
||||
const initialPath = resolve(root, cleanPath);
|
||||
const requestedPath = resolve(root, cleanPath);
|
||||
|
||||
// Sanitizer barrier #1 — guards the first stat() sink.
|
||||
const initialRel = relative(root, initialPath);
|
||||
if (initialRel.startsWith('..') || isAbsolute(initialRel)) {
|
||||
const rel = relative(root, requestedPath);
|
||||
if (rel.startsWith('..') || isAbsolute(rel)) {
|
||||
res.writeHead(400);
|
||||
res.end('Bad request');
|
||||
return;
|
||||
}
|
||||
|
||||
let handle;
|
||||
try {
|
||||
const initialStat = await stat(initialPath).catch(() => null);
|
||||
let servePath = requestedPath;
|
||||
|
||||
// Pick the path we actually serve. Note: any branch reassigns to a
|
||||
// freshly-resolved path; the next sanitizer barrier re-validates.
|
||||
let finalPath;
|
||||
if (initialStat?.isDirectory()) {
|
||||
finalPath = resolve(initialPath, 'index.html');
|
||||
} else if (!initialStat?.isFile()) {
|
||||
finalPath = resolve(root, 'index.html');
|
||||
// Try to open the exact path the client asked for.
|
||||
handle = await open(requestedPath, 'r').catch(() => null);
|
||||
if (handle) {
|
||||
const s = await handle.stat();
|
||||
if (!s.isFile()) {
|
||||
// Directories and other non-files fall through to SPA fallback.
|
||||
await handle.close();
|
||||
handle = null;
|
||||
}
|
||||
}
|
||||
|
||||
// If the requested path wasn't a regular file, serve the SPA entry
|
||||
// point. spaFallback is a module-level constant with no data-flow
|
||||
// from the request, so this open() is independent of the one above.
|
||||
if (!handle) {
|
||||
servePath = spaFallback;
|
||||
handle = await open(spaFallback, 'r').catch(() => null);
|
||||
if (!handle) {
|
||||
res.writeHead(404);
|
||||
res.end('Not found');
|
||||
return;
|
||||
}
|
||||
const s = await handle.stat();
|
||||
if (!s.isFile()) {
|
||||
res.writeHead(404);
|
||||
res.end('Not found');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const isHtml = extname(servePath) === '.html' || !extname(servePath);
|
||||
const cacheControl = servePath.includes(`${sep}assets${sep}`)
|
||||
? 'public, max-age=31536000, immutable'
|
||||
: 'no-cache';
|
||||
const contentType = contentTypes[extname(servePath)] || 'application/octet-stream';
|
||||
|
||||
if (isHtml && configScript) {
|
||||
const raw = await handle.readFile('utf8');
|
||||
await handle.close();
|
||||
handle = null;
|
||||
if (!raw.includes('</head>')) {
|
||||
console.warn('[gitnexus-web] Could not inject config: no </head> tag found in HTML');
|
||||
}
|
||||
const html = raw.includes('</head>') ? raw.replace('</head>', `${configScript}</head>`) : raw;
|
||||
const buf = Buffer.from(html, 'utf8');
|
||||
res.writeHead(200, {
|
||||
'Cache-Control': cacheControl,
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Content-Length': buf.length,
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
});
|
||||
res.end(buf);
|
||||
} else {
|
||||
finalPath = initialPath;
|
||||
res.writeHead(200, {
|
||||
'Cache-Control': cacheControl,
|
||||
'Content-Type': contentType,
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
});
|
||||
const stream = handle.createReadStream();
|
||||
handle = null;
|
||||
stream.on('error', () => res.destroy());
|
||||
stream.pipe(res);
|
||||
}
|
||||
|
||||
// Sanitizer barrier #2 — guards both the second stat() and the
|
||||
// createReadStream() sinks. No reassignment of finalPath happens
|
||||
// between this guard and either sink, so the analyzer can prove
|
||||
// containment for both.
|
||||
const finalRel = relative(root, finalPath);
|
||||
if (finalRel.startsWith('..') || isAbsolute(finalRel)) {
|
||||
res.writeHead(400);
|
||||
res.end('Bad request');
|
||||
return;
|
||||
}
|
||||
|
||||
const finalStat = await stat(finalPath).catch(() => null);
|
||||
if (!finalStat?.isFile()) {
|
||||
res.writeHead(404);
|
||||
res.end('Not found');
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Cache-Control': finalPath.includes('/assets/')
|
||||
? 'public, max-age=31536000, immutable'
|
||||
: 'no-cache',
|
||||
'Content-Type': contentTypes[extname(finalPath)] || 'application/octet-stream',
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
});
|
||||
const stream = createReadStream(finalPath);
|
||||
stream.on('error', () => res.destroy());
|
||||
stream.pipe(res);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
res.writeHead(500);
|
||||
res.end(error instanceof Error ? error.message : 'Internal server error');
|
||||
res.end('Internal server error');
|
||||
} finally {
|
||||
if (handle) await handle.close().catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -70,8 +70,20 @@ before(async () => {
|
|||
await waitForServer(serverPort);
|
||||
});
|
||||
|
||||
function killAndWait(proc) {
|
||||
return new Promise((resolve) => {
|
||||
if (!proc || proc.exitCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
proc.once('exit', resolve);
|
||||
proc.kill();
|
||||
if (proc.exitCode !== null) resolve();
|
||||
});
|
||||
}
|
||||
|
||||
after(async () => {
|
||||
child?.kill();
|
||||
await killAndWait(child);
|
||||
if (tmpDir) await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
|
|
@ -122,3 +134,132 @@ it('returns 404 when dist/index.html is missing', async () => {
|
|||
const res = await rawGet(serverPort, '/nonexistent-page');
|
||||
assert.equal(res.status, 404);
|
||||
});
|
||||
|
||||
// -- Config injection: server-level integration tests ---
|
||||
|
||||
function spawnServerWithEnv(cwd, port, env) {
|
||||
const proc = spawn(process.execPath, [serverScript], {
|
||||
cwd,
|
||||
env: { ...process.env, PORT: String(port), ...env },
|
||||
stdio: 'pipe',
|
||||
});
|
||||
proc.on('error', (err) => {
|
||||
throw err;
|
||||
});
|
||||
return proc;
|
||||
}
|
||||
|
||||
async function withInjectionServer(envOverrides, fn) {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'gitnexus-inject-'));
|
||||
const distDir = join(dir, 'dist');
|
||||
const assetsDir = join(distDir, 'assets');
|
||||
await mkdir(assetsDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(distDir, 'index.html'),
|
||||
'<!doctype html><html><head><meta charset="utf-8"></head><body>app</body></html>',
|
||||
);
|
||||
await writeFile(join(assetsDir, 'style.abc.css'), 'body{}');
|
||||
|
||||
const port = await getFreePort();
|
||||
const proc = spawnServerWithEnv(dir, port, envOverrides);
|
||||
try {
|
||||
await waitForServer(port);
|
||||
await fn(port);
|
||||
} finally {
|
||||
await killAndWait(proc);
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it('injects __GITNEXUS_CONFIG__ into / when GITNEXUS_BACKEND_URL is valid', async () => {
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: 'http://10.0.0.1:4747' }, async (port) => {
|
||||
const res = await rawGet(port, '/');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
res.body.includes('window.__GITNEXUS_CONFIG__'),
|
||||
'Expected __GITNEXUS_CONFIG__ in response body',
|
||||
);
|
||||
assert.ok(res.body.includes('http://10.0.0.1:4747'), 'Expected backend URL in response body');
|
||||
});
|
||||
});
|
||||
|
||||
it('injects __GITNEXUS_CONFIG__ into SPA fallback routes', async () => {
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: 'http://10.0.0.1:4747' }, async (port) => {
|
||||
const res = await rawGet(port, '/some/deep/link');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
res.body.includes('window.__GITNEXUS_CONFIG__'),
|
||||
'Expected __GITNEXUS_CONFIG__ in SPA fallback response',
|
||||
);
|
||||
assert.ok(
|
||||
res.body.includes('http://10.0.0.1:4747'),
|
||||
'Expected backend URL in SPA fallback response',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not inject when GITNEXUS_BACKEND_URL is not set', async () => {
|
||||
await withInjectionServer({}, async (port) => {
|
||||
const res = await rawGet(port, '/');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
!res.body.includes('__GITNEXUS_CONFIG__'),
|
||||
'Expected no __GITNEXUS_CONFIG__ when env var is unset',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not inject when GITNEXUS_BACKEND_URL is invalid', async () => {
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: 'not-a-url' }, async (port) => {
|
||||
const res = await rawGet(port, '/');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
!res.body.includes('__GITNEXUS_CONFIG__'),
|
||||
'Expected no __GITNEXUS_CONFIG__ for invalid URL',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('does not inject when GITNEXUS_BACKEND_URL uses a non-http protocol', async () => {
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: 'ftp://somehost:21' }, async (port) => {
|
||||
const res = await rawGet(port, '/');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
!res.body.includes('__GITNEXUS_CONFIG__'),
|
||||
'Expected no __GITNEXUS_CONFIG__ for non-http protocol',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('escapes </script> in GITNEXUS_BACKEND_URL to prevent XSS', async () => {
|
||||
const xssUrl = 'http://example.com/?x=</script><script>alert(1)</script>';
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: xssUrl }, async (port) => {
|
||||
const res = await rawGet(port, '/');
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const scriptMatches = res.body.match(/<script>/gi) || [];
|
||||
assert.equal(
|
||||
scriptMatches.length,
|
||||
1,
|
||||
`Expected exactly 1 <script> tag but found ${scriptMatches.length}: XSS breakout detected`,
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
!res.body.includes('</script><script>'),
|
||||
'</script> must not appear unescaped -- would allow script breakout',
|
||||
);
|
||||
assert.ok(res.body.includes('\\u003c'), 'Angle brackets must be escaped as \\u003c');
|
||||
});
|
||||
});
|
||||
|
||||
it('does not inject config into static assets', async () => {
|
||||
await withInjectionServer({ GITNEXUS_BACKEND_URL: 'http://10.0.0.1:4747' }, async (port) => {
|
||||
const res = await rawGet(port, '/assets/style.abc.css');
|
||||
assert.equal(res.status, 200);
|
||||
assert.ok(
|
||||
!res.body.includes('__GITNEXUS_CONFIG__'),
|
||||
'Static assets must not contain injected config',
|
||||
);
|
||||
assert.equal(res.body, 'body{}');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export type PipelinePhase =
|
|||
| 'imports'
|
||||
| 'calls'
|
||||
| 'heritage'
|
||||
| 'scopeResolution'
|
||||
| 'communities'
|
||||
| 'processes'
|
||||
| 'enriching'
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
export const ERROR_RESET_DELAY_MS = 3000;
|
||||
export const BACKEND_URL_DEBOUNCE_MS = 500;
|
||||
|
||||
export const DEFAULT_BACKEND_URL = 'http://localhost:4747';
|
||||
export const DEFAULT_BACKEND_URL =
|
||||
(typeof window !== 'undefined' && window.__GITNEXUS_CONFIG__?.backendUrl) ||
|
||||
'http://localhost:4747';
|
||||
export const DEFAULT_OLLAMA_BASE_URL = 'http://localhost:11434';
|
||||
export const DEFAULT_OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@
|
|||
"imports": "Resolving imports",
|
||||
"calls": "Tracing calls",
|
||||
"heritage": "Extracting inheritance",
|
||||
"scopeResolution": "Resolving types",
|
||||
"communities": "Detecting communities",
|
||||
"processes": "Detecting processes",
|
||||
"complete": "Pipeline complete",
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@
|
|||
"imports": "正在解析导入",
|
||||
"calls": "正在追踪调用",
|
||||
"heritage": "正在提取继承关系",
|
||||
"scopeResolution": "正在解析类型",
|
||||
"communities": "正在检测社区",
|
||||
"processes": "正在检测流程",
|
||||
"complete": "流水线完成",
|
||||
|
|
|
|||
6
gitnexus-web/src/vite-env.d.ts
vendored
6
gitnexus-web/src/vite-env.d.ts
vendored
|
|
@ -1 +1,7 @@
|
|||
/// <reference types="vite/client" />
|
||||
|
||||
interface Window {
|
||||
__GITNEXUS_CONFIG__?: {
|
||||
backendUrl?: string;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -172,6 +172,37 @@ describe('fetchGraph', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('DEFAULT_BACKEND_URL resolution', () => {
|
||||
afterEach(() => {
|
||||
delete window.__GITNEXUS_CONFIG__;
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it('falls back to localhost:4747 when no config is injected', async () => {
|
||||
delete window.__GITNEXUS_CONFIG__;
|
||||
const { DEFAULT_BACKEND_URL } = await import('../../src/config/ui-constants');
|
||||
expect(DEFAULT_BACKEND_URL).toBe('http://localhost:4747');
|
||||
});
|
||||
|
||||
it('uses window.__GITNEXUS_CONFIG__.backendUrl when set', async () => {
|
||||
window.__GITNEXUS_CONFIG__ = { backendUrl: 'http://10.0.0.1:4747' };
|
||||
const { DEFAULT_BACKEND_URL } = await import('../../src/config/ui-constants');
|
||||
expect(DEFAULT_BACKEND_URL).toBe('http://10.0.0.1:4747');
|
||||
});
|
||||
|
||||
it('falls back to localhost:4747 when config object has no backendUrl', async () => {
|
||||
window.__GITNEXUS_CONFIG__ = {};
|
||||
const { DEFAULT_BACKEND_URL } = await import('../../src/config/ui-constants');
|
||||
expect(DEFAULT_BACKEND_URL).toBe('http://localhost:4747');
|
||||
});
|
||||
|
||||
it('falls back to localhost:4747 when backendUrl is an empty string', async () => {
|
||||
window.__GITNEXUS_CONFIG__ = { backendUrl: '' };
|
||||
const { DEFAULT_BACKEND_URL } = await import('../../src/config/ui-constants');
|
||||
expect(DEFAULT_BACKEND_URL).toBe('http://localhost:4747');
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateBackendUrl', () => {
|
||||
it('allows http:// URLs', () => {
|
||||
expect(() => validateBackendUrl('http://localhost:4747')).not.toThrow();
|
||||
|
|
@ -225,7 +256,6 @@ describe('setBackendUrl', () => {
|
|||
it('does not mutate _backendUrl when validation fails', () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
expect(() => setBackendUrl('javascript:alert(1)')).toThrow();
|
||||
// State must be preserved — validation must happen before the assignment
|
||||
expect(getBackendUrl()).toBe('http://localhost:4747');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import { warnMissingOptionalGrammars } from './optional-grammars.js';
|
|||
import { glob } from 'glob';
|
||||
import fs from 'fs/promises';
|
||||
import { cliError } from './cli-message.js';
|
||||
import { formatElapsed } from './format-elapsed.js';
|
||||
import { isHfDownloadFailure } from '../core/embeddings/hf-env.js';
|
||||
|
||||
// Capture stderr.write at module load BEFORE anything (LadybugDB native
|
||||
|
|
@ -916,14 +917,14 @@ const analyzeCommandImpl = async (inputPath?: string, options?: AnalyzeOptions):
|
|||
phaseStart = Date.now();
|
||||
}
|
||||
const elapsed = Math.round((Date.now() - phaseStart) / 1000);
|
||||
const display = elapsed >= 3 ? `${phaseLabel} (${elapsed}s)` : phaseLabel;
|
||||
const display = elapsed >= 3 ? `${phaseLabel} (${formatElapsed(elapsed)})` : phaseLabel;
|
||||
bar.update(value, { phase: display });
|
||||
};
|
||||
|
||||
const elapsedTimer = setInterval(() => {
|
||||
const elapsed = Math.round((Date.now() - phaseStart) / 1000);
|
||||
if (elapsed >= 3) {
|
||||
bar.update({ phase: `${lastPhaseLabel} (${elapsed}s)` });
|
||||
bar.update({ phase: `${lastPhaseLabel} (${formatElapsed(elapsed)})` });
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
|
|
|
|||
7
gitnexus/src/cli/format-elapsed.ts
Normal file
7
gitnexus/src/cli/format-elapsed.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export function formatElapsed(secs: number): string {
|
||||
if (secs < 60) return `${secs}s`;
|
||||
if (secs < 3600) return `${Math.floor(secs / 60)}m ${secs % 60}s`;
|
||||
const h = Math.floor(secs / 3600);
|
||||
const m = Math.floor((secs % 3600) / 60);
|
||||
return `${h}h ${m}m`;
|
||||
}
|
||||
|
|
@ -32,13 +32,13 @@ export const communitiesPhase: PipelinePhase<CommunitiesOutput> = {
|
|||
|
||||
ctx.onProgress({
|
||||
phase: 'communities',
|
||||
percent: 84,
|
||||
percent: 98,
|
||||
message: 'Detecting code communities...',
|
||||
stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
|
||||
});
|
||||
|
||||
const communityResult = await processCommunities(ctx.graph, (message, progress) => {
|
||||
const communityProgress = 84 + progress * 0.09;
|
||||
const communityProgress = 98 + progress * 0.01;
|
||||
ctx.onProgress({
|
||||
phase: 'communities',
|
||||
percent: Math.round(communityProgress),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* Computes Method Resolution Order (MRO) and creates METHOD_OVERRIDES
|
||||
* and METHOD_IMPLEMENTS edges.
|
||||
*
|
||||
* @deps crossFile
|
||||
* @deps crossFile, scopeResolution
|
||||
* @reads graph (all nodes and relationships)
|
||||
* @writes graph (METHOD_OVERRIDES, METHOD_IMPLEMENTS edges)
|
||||
*/
|
||||
|
|
@ -25,7 +25,7 @@ export interface MROOutput {
|
|||
|
||||
export const mroPhase: PipelinePhase<MROOutput> = {
|
||||
name: 'mro',
|
||||
deps: ['crossFile', 'structure'],
|
||||
deps: ['crossFile', 'scopeResolution', 'structure'],
|
||||
|
||||
async execute(
|
||||
ctx: PipelineContext,
|
||||
|
|
@ -35,7 +35,7 @@ export const mroPhase: PipelinePhase<MROOutput> = {
|
|||
|
||||
ctx.onProgress({
|
||||
phase: 'enriching',
|
||||
percent: 83,
|
||||
percent: 98,
|
||||
message: 'Computing method resolution order...',
|
||||
stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
|
||||
});
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ export const processesPhase: PipelinePhase<ProcessesOutput> = {
|
|||
|
||||
ctx.onProgress({
|
||||
phase: 'processes',
|
||||
percent: 94,
|
||||
percent: 99,
|
||||
message: 'Detecting execution flows...',
|
||||
stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
|
||||
});
|
||||
|
|
@ -56,7 +56,7 @@ export const processesPhase: PipelinePhase<ProcessesOutput> = {
|
|||
ctx.graph,
|
||||
communityResult.memberships,
|
||||
(message, progress) => {
|
||||
const processProgress = 94 + progress * 0.05;
|
||||
const processProgress = 99 + progress * 0.01;
|
||||
ctx.onProgress({
|
||||
phase: 'processes',
|
||||
percent: Math.round(processProgress),
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ import type { ParseOutput } from '../../pipeline-phases/parse.js';
|
|||
import { isRegistryPrimary } from '../../registry-primary-flag.js';
|
||||
import { SupportedLanguages, getLanguageFromFilename } from 'gitnexus-shared';
|
||||
import { readFileContents } from '../../filesystem-walker.js';
|
||||
import { runScopeResolution } from './run.js';
|
||||
import { runScopeResolution, type ScopeResolutionSubPhase } from './run.js';
|
||||
import { SCOPE_RESOLVERS } from './registry.js';
|
||||
import { isDev, isSemanticModelValidatorEnabled } from '../../utils/env.js';
|
||||
import type { ResolutionOutcome } from '../resolution-outcome.js';
|
||||
|
|
@ -130,6 +130,31 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
}
|
||||
>();
|
||||
|
||||
// Pre-count files and languages for progress reporting. This avoids
|
||||
// a frozen progress bar during long scope-resolution runs (#1741).
|
||||
let totalScopeFiles = 0;
|
||||
let totalScopeLangs = 0;
|
||||
for (const [lang] of SCOPE_RESOLVERS) {
|
||||
if (!isRegistryPrimary(lang)) continue;
|
||||
const count = scannedFiles.filter((f) => getLanguageFromFilename(f.path) === lang).length;
|
||||
if (count > 0) {
|
||||
totalScopeLangs++;
|
||||
totalScopeFiles += count;
|
||||
}
|
||||
}
|
||||
const SCOPE_PCT_START = 90;
|
||||
const SCOPE_PCT_RANGE = 8; // 90-98 internal → 54-59% display
|
||||
let processedScopeFiles = 0;
|
||||
let currentLangIdx = 0;
|
||||
|
||||
if (totalScopeFiles > 0) {
|
||||
ctx.onProgress({
|
||||
phase: 'scopeResolution',
|
||||
percent: SCOPE_PCT_START,
|
||||
message: 'Resolving types',
|
||||
});
|
||||
}
|
||||
|
||||
for (const [lang, provider] of SCOPE_RESOLVERS) {
|
||||
if (!isRegistryPrimary(lang)) continue;
|
||||
|
||||
|
|
@ -153,6 +178,23 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
? await provider.loadResolutionConfig(ctx.repoPath)
|
||||
: undefined;
|
||||
|
||||
const langFileCount = files.length;
|
||||
const langLabel = lang.charAt(0).toUpperCase() + lang.slice(1);
|
||||
currentLangIdx++;
|
||||
const langTag =
|
||||
totalScopeLangs > 1 ? `${langLabel} [${currentLangIdx}/${totalScopeLangs}]` : langLabel;
|
||||
|
||||
if (totalScopeFiles > 0) {
|
||||
const pct =
|
||||
SCOPE_PCT_START + Math.round((processedScopeFiles / totalScopeFiles) * SCOPE_PCT_RANGE);
|
||||
ctx.onProgress({
|
||||
phase: 'scopeResolution',
|
||||
percent: pct,
|
||||
message: 'Resolving types',
|
||||
detail: `${langTag}, ${langFileCount.toLocaleString()} files`,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = runScopeResolution(
|
||||
{
|
||||
graph: ctx.graph,
|
||||
|
|
@ -169,6 +211,44 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
logger.warn(`[scope-resolution:${lang}] ${msg}`);
|
||||
}
|
||||
},
|
||||
onProgress:
|
||||
totalScopeFiles > 0
|
||||
? (subPhase: ScopeResolutionSubPhase, current, total) => {
|
||||
let langRatio: number;
|
||||
switch (subPhase) {
|
||||
case 'extracting':
|
||||
langRatio = total > 0 ? (current / total) * 0.5 : 0;
|
||||
break;
|
||||
case 'analyzing types':
|
||||
langRatio = 0.5;
|
||||
break;
|
||||
case 'resolving references':
|
||||
langRatio = 0.7;
|
||||
break;
|
||||
case 'linking symbols':
|
||||
langRatio = 0.85;
|
||||
break;
|
||||
default: {
|
||||
const _exhaustive: never = subPhase;
|
||||
langRatio = 0.85;
|
||||
}
|
||||
}
|
||||
const overallRatio = Math.min(
|
||||
1,
|
||||
(processedScopeFiles + langRatio * langFileCount) / totalScopeFiles,
|
||||
);
|
||||
const pct = SCOPE_PCT_START + Math.round(overallRatio * SCOPE_PCT_RANGE);
|
||||
ctx.onProgress({
|
||||
phase: 'scopeResolution',
|
||||
percent: pct,
|
||||
message: 'Resolving types',
|
||||
detail:
|
||||
subPhase === 'extracting'
|
||||
? `${langTag} — extracting ${current.toLocaleString()}/${total.toLocaleString()} files`
|
||||
: `${langTag} — ${subPhase}`,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
provider,
|
||||
);
|
||||
|
|
@ -183,6 +263,7 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
preExtractedByPath.delete(fp);
|
||||
}
|
||||
|
||||
processedScopeFiles += langFileCount;
|
||||
anyRan = true;
|
||||
totalFiles += stats.filesProcessed;
|
||||
totalImports += stats.importsEmitted;
|
||||
|
|
@ -200,6 +281,15 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
}
|
||||
}
|
||||
|
||||
if (totalScopeFiles > 0 && anyRan) {
|
||||
ctx.onProgress({
|
||||
phase: 'scopeResolution',
|
||||
percent: SCOPE_PCT_START + SCOPE_PCT_RANGE,
|
||||
message: 'Resolving types',
|
||||
detail: 'complete',
|
||||
});
|
||||
}
|
||||
|
||||
// Dispose the cross-phase Tree cache — scope-resolution is the
|
||||
// only consumer. Holding Trees past this point is pure memory
|
||||
// pressure: downstream phases (mro, community, csv-generator)
|
||||
|
|
|
|||
|
|
@ -116,6 +116,12 @@ function preEmitInheritanceEdges(
|
|||
return handledSites;
|
||||
}
|
||||
|
||||
export type ScopeResolutionSubPhase =
|
||||
| 'extracting'
|
||||
| 'analyzing types'
|
||||
| 'resolving references'
|
||||
| 'linking symbols';
|
||||
|
||||
interface RunScopeResolutionInput {
|
||||
readonly graph: KnowledgeGraph;
|
||||
/**
|
||||
|
|
@ -167,6 +173,16 @@ interface RunScopeResolutionInput {
|
|||
* intentionally suppress an edge; the graph remains unchanged.
|
||||
*/
|
||||
readonly recordResolutionOutcome?: ResolutionOutcomeRecorder;
|
||||
/**
|
||||
* Optional progress callback for UI updates during long-running scope
|
||||
* resolution. Called periodically during the extract loop and at each
|
||||
* sub-phase boundary (finalize, resolve, emit).
|
||||
*
|
||||
* @param subPhase Current sub-phase name for display
|
||||
* @param current Files processed so far (during extract) or total files (at phase boundaries)
|
||||
* @param total Total files in this language
|
||||
*/
|
||||
readonly onProgress?: (subPhase: ScopeResolutionSubPhase, current: number, total: number) => void;
|
||||
}
|
||||
|
||||
interface RunScopeResolutionStats {
|
||||
|
|
@ -207,7 +223,10 @@ export function runScopeResolution(
|
|||
const treeCache = input.treeCache;
|
||||
const preExtracted = input.preExtractedParsedFiles;
|
||||
let preExtractedHits = 0;
|
||||
for (const file of files) {
|
||||
const progressInterval = files.length > 0 ? Math.max(1, Math.floor(files.length / 50)) : 1;
|
||||
input.onProgress?.('extracting', 0, files.length);
|
||||
for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
|
||||
const file = files[fileIdx];
|
||||
let parsed: ParsedFile | undefined;
|
||||
// Fast path: a worker (during the parse phase) already produced a
|
||||
// ParsedFile for this file via `extractParsedFile`. Reuse it
|
||||
|
|
@ -232,6 +251,12 @@ export function runScopeResolution(
|
|||
}
|
||||
provider.populateOwners(parsed);
|
||||
parsedFiles.push(parsed);
|
||||
if (
|
||||
input.onProgress &&
|
||||
((fileIdx + 1) % progressInterval === 0 || fileIdx === files.length - 1)
|
||||
) {
|
||||
input.onProgress('extracting', fileIdx + 1, files.length);
|
||||
}
|
||||
}
|
||||
if (PROF && preExtracted !== undefined) {
|
||||
logger.warn(`[scope-resolution prof] pre-extracted hits: ${preExtractedHits}/${files.length}`);
|
||||
|
|
@ -267,6 +292,7 @@ export function runScopeResolution(
|
|||
const tExtract = PROF ? process.hrtime.bigint() : 0n;
|
||||
|
||||
// ── Phase 2: finalize → ScopeResolutionIndexes ─────────────────────────
|
||||
input.onProgress?.('analyzing types', files.length, files.length);
|
||||
const allFilePaths = new Set(parsedFiles.map((f) => f.filePath));
|
||||
const nodeLookup = buildGraphNodeLookup(graph);
|
||||
|
||||
|
|
@ -350,6 +376,7 @@ export function runScopeResolution(
|
|||
validateBindingsImmutability(indexes, onWarn);
|
||||
|
||||
// ── Phase 3: resolve references via Registry.lookup ────────────────────
|
||||
input.onProgress?.('resolving references', files.length, files.length);
|
||||
const registryProviders: RegistryProviders = {
|
||||
arityCompatibility: provider.arityCompatibility,
|
||||
};
|
||||
|
|
@ -362,6 +389,7 @@ export function runScopeResolution(
|
|||
const tResolve = PROF ? process.hrtime.bigint() : 0n;
|
||||
|
||||
// ── Phase 4: emit graph edges (LOAD-BEARING ORDER — see I1) ────────────
|
||||
input.onProgress?.('linking symbols', files.length, files.length);
|
||||
const handledSites = new Set<string>(preEmittedInheritanceSites);
|
||||
const receiverExtras = emitReceiverBoundCalls(
|
||||
graph,
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ export const PHASE_LABELS: Record<string, string> = {
|
|||
imports: 'Resolving imports',
|
||||
calls: 'Tracing calls',
|
||||
heritage: 'Extracting inheritance',
|
||||
scopeResolution: 'Resolving types',
|
||||
communities: 'Detecting communities',
|
||||
processes: 'Detecting processes',
|
||||
complete: 'Pipeline complete',
|
||||
|
|
|
|||
36
gitnexus/test/unit/format-elapsed.test.ts
Normal file
36
gitnexus/test/unit/format-elapsed.test.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { formatElapsed } from '../../src/cli/format-elapsed.js';
|
||||
|
||||
describe('formatElapsed', () => {
|
||||
it('formats 0 seconds', () => {
|
||||
expect(formatElapsed(0)).toBe('0s');
|
||||
});
|
||||
|
||||
it('formats seconds below 60', () => {
|
||||
expect(formatElapsed(1)).toBe('1s');
|
||||
expect(formatElapsed(59)).toBe('59s');
|
||||
});
|
||||
|
||||
it('formats exactly 60 seconds as 1m 0s', () => {
|
||||
expect(formatElapsed(60)).toBe('1m 0s');
|
||||
});
|
||||
|
||||
it('formats minutes and seconds', () => {
|
||||
expect(formatElapsed(61)).toBe('1m 1s');
|
||||
expect(formatElapsed(125)).toBe('2m 5s');
|
||||
});
|
||||
|
||||
it('formats the last second before an hour', () => {
|
||||
expect(formatElapsed(3599)).toBe('59m 59s');
|
||||
});
|
||||
|
||||
it('formats exactly 3600 seconds as 1h 0m', () => {
|
||||
expect(formatElapsed(3600)).toBe('1h 0m');
|
||||
});
|
||||
|
||||
it('formats hours and minutes', () => {
|
||||
expect(formatElapsed(3661)).toBe('1h 1m');
|
||||
expect(formatElapsed(7200)).toBe('2h 0m');
|
||||
expect(formatElapsed(7323)).toBe('2h 2m');
|
||||
});
|
||||
});
|
||||
107
gitnexus/test/unit/scope-resolution/run-progress.test.ts
Normal file
107
gitnexus/test/unit/scope-resolution/run-progress.test.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import type { ParsedFile, ScopeId, Scope } from 'gitnexus-shared';
|
||||
import {
|
||||
runScopeResolution,
|
||||
type ScopeResolutionSubPhase,
|
||||
} from '../../../src/core/ingestion/scope-resolution/pipeline/run.js';
|
||||
import { createKnowledgeGraph } from '../../../src/core/graph/graph.js';
|
||||
import { createSemanticModel } from '../../../src/core/ingestion/model/semantic-model.js';
|
||||
import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js';
|
||||
|
||||
const mkScope = (id: ScopeId, filePath: string): Scope => ({
|
||||
id,
|
||||
parent: null,
|
||||
kind: 'Module',
|
||||
range: { startLine: 1, startCol: 0, endLine: 10, endCol: 0 },
|
||||
filePath,
|
||||
bindings: new Map(),
|
||||
ownedDefs: [],
|
||||
imports: [],
|
||||
typeBindings: new Map(),
|
||||
});
|
||||
|
||||
const mkFile = (filePath: string): ParsedFile => ({
|
||||
filePath,
|
||||
moduleScope: `scope:${filePath}#module`,
|
||||
scopes: [mkScope(`scope:${filePath}#module`, filePath)],
|
||||
parsedImports: [],
|
||||
localDefs: [],
|
||||
referenceSites: [],
|
||||
});
|
||||
|
||||
const stubProvider = {
|
||||
language: 'python' as const,
|
||||
languageProvider: {} as ScopeResolver['languageProvider'],
|
||||
importEdgeReason: 'test',
|
||||
populateOwners: () => {},
|
||||
resolveImportTarget: () => null,
|
||||
mergeBindings: (existing: unknown) => existing,
|
||||
buildMro: () => new Map(),
|
||||
propagatesReturnTypesAcrossImports: false,
|
||||
} as unknown as ScopeResolver;
|
||||
|
||||
describe('runScopeResolution onProgress', () => {
|
||||
it('emits sub-phases in order for a 3-file input', () => {
|
||||
const files = [
|
||||
{ path: 'a.py', content: '' },
|
||||
{ path: 'b.py', content: '' },
|
||||
{ path: 'c.py', content: '' },
|
||||
];
|
||||
const preExtracted = new Map<string, ParsedFile>();
|
||||
for (const f of files) preExtracted.set(f.path, mkFile(f.path));
|
||||
|
||||
const calls: { subPhase: ScopeResolutionSubPhase; current: number; total: number }[] = [];
|
||||
const onProgress = (subPhase: ScopeResolutionSubPhase, current: number, total: number) => {
|
||||
calls.push({ subPhase, current, total });
|
||||
};
|
||||
|
||||
runScopeResolution(
|
||||
{
|
||||
graph: createKnowledgeGraph(),
|
||||
model: createSemanticModel(),
|
||||
files,
|
||||
preExtractedParsedFiles: preExtracted,
|
||||
onProgress,
|
||||
},
|
||||
stubProvider,
|
||||
);
|
||||
|
||||
const subPhases = calls.map((c) => c.subPhase);
|
||||
expect(subPhases).toContain('extracting');
|
||||
expect(subPhases).toContain('analyzing types');
|
||||
expect(subPhases).toContain('resolving references');
|
||||
expect(subPhases).toContain('linking symbols');
|
||||
|
||||
const extractCalls = calls.filter((c) => c.subPhase === 'extracting');
|
||||
expect(extractCalls.length).toBeGreaterThan(0);
|
||||
expect(extractCalls[0].total).toBe(3);
|
||||
expect(extractCalls[0].current).toBe(0);
|
||||
expect(extractCalls[extractCalls.length - 1].current).toBe(3);
|
||||
|
||||
const analyzeIdx = subPhases.indexOf('analyzing types');
|
||||
const resolveIdx = subPhases.indexOf('resolving references');
|
||||
const linkIdx = subPhases.indexOf('linking symbols');
|
||||
expect(analyzeIdx).toBeLessThan(resolveIdx);
|
||||
expect(resolveIdx).toBeLessThan(linkIdx);
|
||||
});
|
||||
|
||||
it('emits only extracting (0, 0) then returns early for 0-file input', () => {
|
||||
const calls: { subPhase: ScopeResolutionSubPhase; current: number; total: number }[] = [];
|
||||
const onProgress = (subPhase: ScopeResolutionSubPhase, current: number, total: number) => {
|
||||
calls.push({ subPhase, current, total });
|
||||
};
|
||||
|
||||
const stats = runScopeResolution(
|
||||
{
|
||||
graph: createKnowledgeGraph(),
|
||||
model: createSemanticModel(),
|
||||
files: [],
|
||||
onProgress,
|
||||
},
|
||||
stubProvider,
|
||||
);
|
||||
|
||||
expect(stats.filesProcessed).toBe(0);
|
||||
expect(calls).toEqual([{ subPhase: 'extracting', current: 0, total: 0 }]);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue