mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-03 02:21:44 +00:00
* feat(storage): add configurable index storage and content retention tiers Rebase #3060 onto current origin/main. Keep GITNEXUS_STORAGE_PATH, GITNEXUS_STORAGE_ROOT, and GITNEXUS_CONTENT_RETENTION, and fold in main's FTS skip, embed-session, and help-text updates. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep legacy registry rows on the local storage fallback, resolve symlinks before the destructive-path guard, and align hook lookup with CLI branch slugs, branch-slot metadata, and longest-path match. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Only list swept upload directories after a successful removal so callers cannot treat a permission or transient rm failure as gone. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Document that getStoragePath may consult registered storage while this module still does not mutate the global registry. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(storage): close review findings for external indexes and retention Re-inspect ownership under the analyze lock, fail-closed when the registry file is missing, and keep skip-git hook discovery plus retention fields on HTTP/MCP list surfaces. /api/file stays 410 unless contentRetention is full. Co-authored-by: Cursor <cursoragent@cursor.com> * chore(autofix): apply prettier + eslint fixes via /autofix command * Address PR review feedback (#3060) Treat lock-only index dirs as empty, honor HTTP --force storage policy, and prefer registered plus branch-aware slots in hooks and augment. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep hook fallbacks inside the current worktree, compare foreign-local slots canonically, and make storage fixtures survive ownership validation. Co-authored-by: Cursor <cursoragent@cursor.com> * Fix macOS hook test expecting realpath'd registry paths. resolveHookRepo returns the written registry path, not a filesystem realpath, so the assertion must match that. * Address gitnexus-check warnings on hook install docs and slot tests. The Cursor troubleshooting list omitted registry-query.cjs, and the writable-slot test only checked that isDirectory exists instead of that the path is a directory. * Align the HTTP catalog source-scan with skippable resolveRepo validation. resolveRepo lists fresh repos with validate: options.validateStorage !== false so DELETE can skip prune; the test still required a literal validate: true. * Harden storage path sinks so CodeQL path-injection and ReDoS alerts clear. Contain every filesystem probe inside the resolved storage slot with the inline path.relative idiom, reject filesystem-root slots, and trim slot basenames in linear time. * Settle bridge stamps before writing so CI size/mtime matches stay stable. LadybugDB can still flush into bridge.lbug after close+rename; persist whole-millisecond mtimes and wait for consecutive stats to agree so a freshly written pair matches. * Type the settled bridge stat as fs.Stats so tsc does not see bigint. Awaited<ReturnType<typeof fsp.stat>> collapsed the bigint overload and broke prepare/typecheck on CI. * Keep the bridge mtime stamp exact so same-size swaps still fail the pair check. Co-authored-by: Cursor <cursoragent@cursor.com> * Wrap the bridge stamp predicate so prettier --check stays green. Co-authored-by: Cursor <cursoragent@cursor.com> * Require a quiet interval before stamping a settled bridge file. Co-authored-by: Cursor <cursoragent@cursor.com> * Reuse shared storage and settle helpers instead of local copies. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
265 lines
9.7 KiB
TypeScript
265 lines
9.7 KiB
TypeScript
/**
|
|
* #3092 item 6 — ≥128 identity cache guards stay in a subprocess unless
|
|
* `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` is truthy or packageRoot /
|
|
* buildRoot fail `W_OK` with EACCES/EROFS. Persist/cache write failure is not
|
|
* that signal. Mutation still fail-closes on both paths.
|
|
*/
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { mkdir, realpath, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
import { writeFileSync } from 'node:fs';
|
|
|
|
const spawnCtx = vi.hoisted(() => ({
|
|
spawnSync: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('node:child_process', async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import('node:child_process')>();
|
|
spawnCtx.spawnSync.mockImplementation(((...args: Parameters<typeof actual.spawnSync>) =>
|
|
actual.spawnSync(...args)) as typeof actual.spawnSync);
|
|
return {
|
|
...actual,
|
|
spawnSync: ((...args: Parameters<typeof actual.spawnSync>) =>
|
|
spawnCtx.spawnSync(...args)) as typeof actual.spawnSync,
|
|
};
|
|
});
|
|
|
|
const fsCtx = vi.hoisted(() => ({
|
|
accessSync: vi.fn(),
|
|
unwritableExact: new Set<string>(),
|
|
wOkProbes: [] as string[],
|
|
}));
|
|
|
|
vi.mock('node:fs', async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import('node:fs')>();
|
|
fsCtx.accessSync.mockImplementation(((
|
|
p: Parameters<typeof actual.accessSync>[0],
|
|
mode?: number,
|
|
) => {
|
|
const pathStr = String(p);
|
|
if (mode === actual.constants.W_OK) fsCtx.wOkProbes.push(pathStr);
|
|
if (mode === actual.constants.W_OK && fsCtx.unwritableExact.has(pathStr)) {
|
|
const err = new Error('EACCES') as NodeJS.ErrnoException;
|
|
err.code = 'EACCES';
|
|
throw err;
|
|
}
|
|
return actual.accessSync(p, mode);
|
|
}) as typeof actual.accessSync);
|
|
return {
|
|
...actual,
|
|
accessSync: ((...args: Parameters<typeof actual.accessSync>) =>
|
|
fsCtx.accessSync(...args)) as typeof actual.accessSync,
|
|
};
|
|
});
|
|
|
|
import {
|
|
_clearAnalyzerIdentityProcessCacheForTests,
|
|
resolveAnalyzerRunnerIdentity,
|
|
} from '../../src/core/analyzer-identity.js';
|
|
import { createTempDir } from '../helpers/test-db.js';
|
|
|
|
const ENV_KEY = 'GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS';
|
|
|
|
const isCacheGuardSpawn = (call: unknown[]): boolean => {
|
|
const argv = call[1];
|
|
return Array.isArray(argv) && argv.includes('--input-type=commonjs') && argv.includes('-e');
|
|
};
|
|
|
|
const cacheGuardSpawnCount = (): number =>
|
|
spawnCtx.spawnSync.mock.calls.filter((call) => isCacheGuardSpawn(call as unknown[])).length;
|
|
|
|
async function seedWideBuildTree(root: string): Promise<{
|
|
modulePath: string;
|
|
sourceRoot: string;
|
|
packageRoot: string;
|
|
mutatedPath: string;
|
|
}> {
|
|
const packageRoot = root;
|
|
const sourceRoot = path.join(root, 'src');
|
|
const modulePath = path.join(sourceRoot, 'core', 'analyzer.ts');
|
|
await mkdir(path.dirname(modulePath), { recursive: true });
|
|
await writeFile(
|
|
path.join(root, 'package.json'),
|
|
'{"name":"fixture-analyzer","version":"9.8.7"}\n',
|
|
);
|
|
await writeFile(modulePath, 'export const analyzer = 1;\n');
|
|
for (let i = 0; i < 140; i += 1) {
|
|
await writeFile(path.join(sourceRoot, `wide-${i}.ts`), `export const n${i} = ${i};\n`);
|
|
}
|
|
return {
|
|
modulePath,
|
|
sourceRoot,
|
|
packageRoot,
|
|
mutatedPath: path.join(sourceRoot, 'wide-0.ts'),
|
|
};
|
|
}
|
|
|
|
describe('analyzer identity in-process cache guards (#3092)', () => {
|
|
let previousEnv: string | undefined;
|
|
|
|
beforeEach(() => {
|
|
previousEnv = process.env[ENV_KEY];
|
|
delete process.env[ENV_KEY];
|
|
spawnCtx.spawnSync.mockClear();
|
|
fsCtx.unwritableExact.clear();
|
|
fsCtx.wOkProbes.length = 0;
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (previousEnv === undefined) delete process.env[ENV_KEY];
|
|
else process.env[ENV_KEY] = previousEnv;
|
|
fsCtx.unwritableExact.clear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
});
|
|
|
|
it('uses spawnSync for ≥128 guards on a writable tree when env is unset', async () => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
let guardCount = 0;
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
onCacheValidationPass: ({ guardCount: n }) => {
|
|
guardCount = n;
|
|
},
|
|
});
|
|
expect(guardCount).toBeGreaterThanOrEqual(128);
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
expect(cacheGuardSpawnCount()).toBeGreaterThan(0);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it.each(['1', 'true', 'yes'])('skips spawn when env is %j', async (value) => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
process.env[ENV_KEY] = value;
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
const warm = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
});
|
|
expect(cacheGuardSpawnCount()).toBe(0);
|
|
expect(warm.schemaVersion).toBe(4);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it.each(['0', 'false', 'off', ''])('still spawns when env is %j', async (value) => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
process.env[ENV_KEY] = value;
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
expect(cacheGuardSpawnCount()).toBeGreaterThan(0);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it('uses in-process snapshots when packageRoot W_OK fails with EACCES', async () => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
fsCtx.unwritableExact.add(await realpath(tree.packageRoot));
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
expect(cacheGuardSpawnCount()).toBe(0);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it('does not treat persist-cache W_OK failure as an unwritable install', async () => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
fsCtx.unwritableExact.add(cacheDirectory);
|
|
spawnCtx.spawnSync.mockClear();
|
|
fsCtx.wOkProbes.length = 0;
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, { cacheDirectory });
|
|
expect(fsCtx.wOkProbes).toEqual(
|
|
expect.arrayContaining([await realpath(tree.packageRoot), await realpath(tree.sourceRoot)]),
|
|
);
|
|
expect(fsCtx.wOkProbes).not.toContain(cacheDirectory);
|
|
expect(cacheGuardSpawnCount()).toBeGreaterThan(0);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it('fail-closes on mutation with the default spawn path', async () => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
const first = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
});
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
let mutated = false;
|
|
const second = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
onCacheValidationPass: () => {
|
|
if (!mutated) {
|
|
mutated = true;
|
|
writeFileSync(tree.mutatedPath, 'export const n0 = 999;\n');
|
|
}
|
|
},
|
|
});
|
|
expect(cacheGuardSpawnCount()).toBeGreaterThan(0);
|
|
expect(second.build.digest).not.toBe(first.build.digest);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
|
|
it('fail-closes on mutation when in-process guards are opted in', async () => {
|
|
const fixture = await createTempDir();
|
|
try {
|
|
process.env[ENV_KEY] = '1';
|
|
const tree = await seedWideBuildTree(fixture.dbPath);
|
|
const cacheDirectory = path.join(fixture.dbPath, 'identity-cache');
|
|
const first = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
});
|
|
spawnCtx.spawnSync.mockClear();
|
|
_clearAnalyzerIdentityProcessCacheForTests();
|
|
let mutated = false;
|
|
const second = resolveAnalyzerRunnerIdentity(pathToFileURL(tree.modulePath).href, {
|
|
cacheDirectory,
|
|
onCacheValidationPass: () => {
|
|
if (!mutated) {
|
|
mutated = true;
|
|
writeFileSync(tree.mutatedPath, 'export const n0 = 1000;\n');
|
|
}
|
|
},
|
|
});
|
|
expect(cacheGuardSpawnCount()).toBe(0);
|
|
expect(second.build.digest).not.toBe(first.build.digest);
|
|
} finally {
|
|
await fixture.cleanup();
|
|
}
|
|
});
|
|
});
|