mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-02 02:11:29 +00:00
* feat(storage): add configurable index storage and content retention tiers Rebase #3060 onto current origin/main. Keep GITNEXUS_STORAGE_PATH, GITNEXUS_STORAGE_ROOT, and GITNEXUS_CONTENT_RETENTION, and fold in main's FTS skip, embed-session, and help-text updates. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep legacy registry rows on the local storage fallback, resolve symlinks before the destructive-path guard, and align hook lookup with CLI branch slugs, branch-slot metadata, and longest-path match. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Only list swept upload directories after a successful removal so callers cannot treat a permission or transient rm failure as gone. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Document that getStoragePath may consult registered storage while this module still does not mutate the global registry. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(storage): close review findings for external indexes and retention Re-inspect ownership under the analyze lock, fail-closed when the registry file is missing, and keep skip-git hook discovery plus retention fields on HTTP/MCP list surfaces. /api/file stays 410 unless contentRetention is full. Co-authored-by: Cursor <cursoragent@cursor.com> * chore(autofix): apply prettier + eslint fixes via /autofix command * Address PR review feedback (#3060) Treat lock-only index dirs as empty, honor HTTP --force storage policy, and prefer registered plus branch-aware slots in hooks and augment. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep hook fallbacks inside the current worktree, compare foreign-local slots canonically, and make storage fixtures survive ownership validation. Co-authored-by: Cursor <cursoragent@cursor.com> * Fix macOS hook test expecting realpath'd registry paths. resolveHookRepo returns the written registry path, not a filesystem realpath, so the assertion must match that. * Address gitnexus-check warnings on hook install docs and slot tests. The Cursor troubleshooting list omitted registry-query.cjs, and the writable-slot test only checked that isDirectory exists instead of that the path is a directory. * Align the HTTP catalog source-scan with skippable resolveRepo validation. resolveRepo lists fresh repos with validate: options.validateStorage !== false so DELETE can skip prune; the test still required a literal validate: true. * Harden storage path sinks so CodeQL path-injection and ReDoS alerts clear. Contain every filesystem probe inside the resolved storage slot with the inline path.relative idiom, reject filesystem-root slots, and trim slot basenames in linear time. * Settle bridge stamps before writing so CI size/mtime matches stay stable. LadybugDB can still flush into bridge.lbug after close+rename; persist whole-millisecond mtimes and wait for consecutive stats to agree so a freshly written pair matches. * Type the settled bridge stat as fs.Stats so tsc does not see bigint. Awaited<ReturnType<typeof fsp.stat>> collapsed the bigint overload and broke prepare/typecheck on CI. * Keep the bridge mtime stamp exact so same-size swaps still fail the pair check. Co-authored-by: Cursor <cursoragent@cursor.com> * Wrap the bridge stamp predicate so prettier --check stays green. Co-authored-by: Cursor <cursoragent@cursor.com> * Require a quiet interval before stamping a settled bridge file. Co-authored-by: Cursor <cursoragent@cursor.com> * Reuse shared storage and settle helpers instead of local copies. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
140 lines
5.6 KiB
TypeScript
140 lines
5.6 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import path from 'node:path';
|
|
import fs from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import { sweepStaleUploads } from '../../src/server/upload-sweep.js';
|
|
|
|
let root: string;
|
|
let home: string;
|
|
let previousHome: string | undefined;
|
|
|
|
beforeEach(async () => {
|
|
root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-sweep-test-'));
|
|
home = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-sweep-home-'));
|
|
await fs.writeFile(path.join(home, 'registry.json'), '[]');
|
|
previousHome = process.env.GITNEXUS_HOME;
|
|
process.env.GITNEXUS_HOME = home;
|
|
});
|
|
afterEach(async () => {
|
|
await fs.rm(root, { recursive: true, force: true }).catch(() => {});
|
|
await fs.rm(home, { recursive: true, force: true }).catch(() => {});
|
|
if (previousHome === undefined) delete process.env.GITNEXUS_HOME;
|
|
else process.env.GITNEXUS_HOME = previousHome;
|
|
});
|
|
|
|
describe('sweepStaleUploads', () => {
|
|
it('removes stale staging dirs but keeps recent ones and non-staging dirs', async () => {
|
|
await fs.mkdir(path.join(root, '.staging-old'));
|
|
await fs.mkdir(path.join(root, '.staging-new'));
|
|
await fs.mkdir(path.join(root, 'myrepo')); // a promoted (persistent) upload dir
|
|
|
|
const now = 1_000_000_000_000;
|
|
// Age the "old" staging dir well past the threshold.
|
|
const old = new Date(now - 10 * 60 * 60 * 1000);
|
|
await fs.utimes(path.join(root, '.staging-old'), old, old);
|
|
const recent = new Date(now - 60 * 1000);
|
|
await fs.utimes(path.join(root, '.staging-new'), recent, recent);
|
|
|
|
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
|
|
|
|
expect(removed).toHaveLength(1);
|
|
expect(removed[0]).toContain('.staging-old');
|
|
await expect(fs.access(path.join(root, '.staging-old'))).rejects.toBeTruthy();
|
|
// Recent staging and the promoted repo dir survive.
|
|
await expect(fs.access(path.join(root, '.staging-new'))).resolves.toBeUndefined();
|
|
await expect(fs.access(path.join(root, 'myrepo'))).resolves.toBeUndefined();
|
|
});
|
|
|
|
it('removes an unregistered stale promoted dir, keeps a registered one without local index', async () => {
|
|
const now = 2_000_000_000_000;
|
|
const old = new Date(now - 10 * 60 * 60 * 1000);
|
|
|
|
// Orphan: a failed analysis that never registered its source directory.
|
|
await fs.mkdir(path.join(root, 'orphan'));
|
|
await fs.utimes(path.join(root, 'orphan'), old, old);
|
|
|
|
// Registered: stale and its index is external, so it has no local
|
|
// `.gitnexus` directory at all. Registry membership is the persistence
|
|
// signal for promoted uploads.
|
|
const registered = path.join(root, 'registered');
|
|
await fs.mkdir(registered);
|
|
await fs.writeFile(
|
|
path.join(home, 'registry.json'),
|
|
JSON.stringify([
|
|
{
|
|
name: 'registered',
|
|
path: registered,
|
|
storagePath: path.join(home, 'storage', 'registered'),
|
|
indexedAt: '',
|
|
lastCommit: '',
|
|
},
|
|
]),
|
|
);
|
|
await fs.utimes(registered, old, old);
|
|
|
|
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
|
|
|
|
expect(removed.some((r) => r.endsWith('orphan'))).toBe(true);
|
|
await expect(fs.access(path.join(root, 'orphan'))).rejects.toBeTruthy();
|
|
await expect(fs.access(path.join(root, 'registered'))).resolves.toBeUndefined();
|
|
});
|
|
|
|
it('keeps a stale promoted dir when registry.json is absent (ENOENT)', async () => {
|
|
const now = 2_000_000_000_000;
|
|
const old = new Date(now - 10 * 60 * 60 * 1000);
|
|
const staging = path.join(root, '.staging-old');
|
|
const promoted = path.join(root, 'promoted');
|
|
await fs.mkdir(staging);
|
|
await fs.mkdir(promoted);
|
|
await fs.utimes(staging, old, old);
|
|
await fs.utimes(promoted, old, old);
|
|
await fs.rm(path.join(home, 'registry.json'));
|
|
|
|
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
|
|
|
|
expect(removed).toContain(staging);
|
|
await expect(fs.access(staging)).rejects.toBeTruthy();
|
|
await expect(fs.access(promoted)).resolves.toBeUndefined();
|
|
});
|
|
|
|
it('still removes stale staging dirs but preserves promoted source dirs when the registry is corrupt', async () => {
|
|
const now = 2_000_000_000_000;
|
|
const old = new Date(now - 10 * 60 * 60 * 1000);
|
|
const staging = path.join(root, '.staging-old');
|
|
const promoted = path.join(root, 'promoted');
|
|
await fs.mkdir(staging);
|
|
await fs.mkdir(promoted);
|
|
await fs.utimes(staging, old, old);
|
|
await fs.utimes(promoted, old, old);
|
|
await fs.writeFile(path.join(home, 'registry.json'), '{"truncated":');
|
|
|
|
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
|
|
|
|
expect(removed).toContain(staging);
|
|
await expect(fs.access(staging)).rejects.toBeTruthy();
|
|
await expect(fs.access(promoted)).resolves.toBeUndefined();
|
|
});
|
|
|
|
it('does not report a path as removed when deletion fails', async () => {
|
|
const now = 3_000_000_000_000;
|
|
const old = new Date(now - 10 * 60 * 60 * 1000);
|
|
const orphan = path.join(root, 'orphan');
|
|
await fs.mkdir(orphan);
|
|
await fs.utimes(orphan, old, old);
|
|
const spy = vi
|
|
.spyOn(fs, 'rm')
|
|
.mockRejectedValueOnce(Object.assign(new Error('EACCES'), { code: 'EACCES' }));
|
|
|
|
try {
|
|
const { removed } = await sweepStaleUploads({ root, now, maxAgeMs: 6 * 60 * 60 * 1000 });
|
|
expect(removed).toEqual([]);
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('tolerates a missing root', async () => {
|
|
const { removed } = await sweepStaleUploads({ root: path.join(root, 'does-not-exist') });
|
|
expect(removed).toEqual([]);
|
|
});
|
|
});
|