mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-10 03:27:59 +00:00
* feat(storage): add configurable index storage and content retention tiers Rebase #3060 onto current origin/main. Keep GITNEXUS_STORAGE_PATH, GITNEXUS_STORAGE_ROOT, and GITNEXUS_CONTENT_RETENTION, and fold in main's FTS skip, embed-session, and help-text updates. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep legacy registry rows on the local storage fallback, resolve symlinks before the destructive-path guard, and align hook lookup with CLI branch slugs, branch-slot metadata, and longest-path match. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Only list swept upload directories after a successful removal so callers cannot treat a permission or transient rm failure as gone. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Document that getStoragePath may consult registered storage while this module still does not mutate the global registry. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(storage): close review findings for external indexes and retention Re-inspect ownership under the analyze lock, fail-closed when the registry file is missing, and keep skip-git hook discovery plus retention fields on HTTP/MCP list surfaces. /api/file stays 410 unless contentRetention is full. Co-authored-by: Cursor <cursoragent@cursor.com> * chore(autofix): apply prettier + eslint fixes via /autofix command * Address PR review feedback (#3060) Treat lock-only index dirs as empty, honor HTTP --force storage policy, and prefer registered plus branch-aware slots in hooks and augment. Co-authored-by: Cursor <cursoragent@cursor.com> * Address PR review feedback (#3060) Keep hook fallbacks inside the current worktree, compare foreign-local slots canonically, and make storage fixtures survive ownership validation. Co-authored-by: Cursor <cursoragent@cursor.com> * Fix macOS hook test expecting realpath'd registry paths. resolveHookRepo returns the written registry path, not a filesystem realpath, so the assertion must match that. * Address gitnexus-check warnings on hook install docs and slot tests. The Cursor troubleshooting list omitted registry-query.cjs, and the writable-slot test only checked that isDirectory exists instead of that the path is a directory. * Align the HTTP catalog source-scan with skippable resolveRepo validation. resolveRepo lists fresh repos with validate: options.validateStorage !== false so DELETE can skip prune; the test still required a literal validate: true. * Harden storage path sinks so CodeQL path-injection and ReDoS alerts clear. Contain every filesystem probe inside the resolved storage slot with the inline path.relative idiom, reject filesystem-root slots, and trim slot basenames in linear time. * Settle bridge stamps before writing so CI size/mtime matches stay stable. LadybugDB can still flush into bridge.lbug after close+rename; persist whole-millisecond mtimes and wait for consecutive stats to agree so a freshly written pair matches. * Type the settled bridge stat as fs.Stats so tsc does not see bigint. Awaited<ReturnType<typeof fsp.stat>> collapsed the bigint overload and broke prepare/typecheck on CI. * Keep the bridge mtime stamp exact so same-size swaps still fail the pair check. Co-authored-by: Cursor <cursoragent@cursor.com> * Wrap the bridge stamp predicate so prettier --check stays green. Co-authored-by: Cursor <cursoragent@cursor.com> * Require a quiet interval before stamping a settled bridge file. Co-authored-by: Cursor <cursoragent@cursor.com> * Reuse shared storage and settle helpers instead of local copies. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
261 lines
8.7 KiB
TypeScript
261 lines
8.7 KiB
TypeScript
import express from 'express';
|
|
import { EventEmitter } from 'node:events';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
|
|
const mocks = vi.hoisted(() => ({
|
|
loadMeta: vi.fn(),
|
|
listRegisteredRepos: vi.fn(),
|
|
withLbugDb: vi.fn(),
|
|
search: vi.fn(),
|
|
updateJob: vi.fn(),
|
|
}));
|
|
|
|
vi.mock('../../src/storage/repo-manager.js', async (importOriginal) => ({
|
|
...(await importOriginal<typeof import('../../src/storage/repo-manager.js')>()),
|
|
loadMeta: mocks.loadMeta,
|
|
listRegisteredRepos: mocks.listRegisteredRepos,
|
|
}));
|
|
vi.mock('../../src/storage/storage-resolver.js', async (importOriginal) => ({
|
|
...(await importOriginal<typeof import('../../src/storage/storage-resolver.js')>()),
|
|
requireRegisteredStoragePath: vi.fn(async (entry: { storagePath: string }) => entry.storagePath),
|
|
}));
|
|
vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({
|
|
withLbugDb: mocks.withLbugDb,
|
|
executeQuery: vi.fn(async () => []),
|
|
executePrepared: vi.fn(async () => [{ value: 1 }]),
|
|
executeWithReusedStatement: vi.fn(async () => []),
|
|
streamQuery: vi.fn(async () => 0),
|
|
flushWAL: vi.fn(),
|
|
closeLbug: vi.fn(),
|
|
isReadOnlyDbError: vi.fn(() => false),
|
|
}));
|
|
vi.mock('../../src/core/search/bm25-index.js', () => ({ searchFTSFromLbug: mocks.search }));
|
|
vi.mock('../../src/mcp/local/local-backend.js', () => ({
|
|
LocalBackend: class {
|
|
async init() {
|
|
return true;
|
|
}
|
|
},
|
|
}));
|
|
vi.mock('../../src/server/mcp-http.js', () => ({
|
|
installServeMcpAuth: vi.fn(),
|
|
mountMCPEndpoints: vi.fn(async () => vi.fn()),
|
|
}));
|
|
vi.mock('../../src/server/upload-sweep.js', () => ({ sweepStaleUploads: vi.fn(async () => {}) }));
|
|
vi.mock('../../src/server/update-controller.js', () => ({
|
|
createServeUpdateController: vi.fn(() => ({ stop: vi.fn() })),
|
|
bindServeUpdateControllerLifecycle: vi.fn(),
|
|
buildServerInfo: vi.fn(),
|
|
}));
|
|
vi.mock('../../src/server/grep-scan.js', () => ({
|
|
runGrepScanInWorker: vi.fn(async () => ({ results: [], timedOut: false })),
|
|
}));
|
|
vi.mock('../../src/server/sse-progress.js', () => ({ mountSSEProgress: vi.fn() }));
|
|
vi.mock('../../src/server/analyze-job.js', () => ({
|
|
isTerminalJobStatus: vi.fn(() => true),
|
|
JobManager: class {
|
|
createJob() {
|
|
return { id: 'embed-job', status: 'queued' };
|
|
}
|
|
updateJob = mocks.updateJob;
|
|
registerAbortController() {}
|
|
getJob() {
|
|
return { status: 'complete' };
|
|
}
|
|
listJobs() {
|
|
return [];
|
|
}
|
|
},
|
|
}));
|
|
|
|
import { createServer } from '../../src/server/api.js';
|
|
import { FTS_DISABLED_MESSAGE } from '../../src/core/search/fts-policy.js';
|
|
|
|
const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-mode-fixture-'));
|
|
const entry = {
|
|
name: 'fts-mode-fixture',
|
|
path: fixtureRoot,
|
|
storagePath: path.join(fixtureRoot, '.gitnexus'),
|
|
};
|
|
fs.mkdirSync(entry.storagePath, { recursive: true });
|
|
let app: express.Express;
|
|
const events = ['SIGINT', 'SIGTERM', 'uncaughtException', 'unhandledRejection'] as const;
|
|
const originalListeners = new Map(events.map((event) => [event, process.listeners(event)]));
|
|
|
|
beforeAll(async () => {
|
|
// Capture the real registered handlers without binding a socket or starting MCP/native work.
|
|
const listen = vi.spyOn(express.application, 'listen').mockImplementation(function (
|
|
this: express.Express,
|
|
...args: any[]
|
|
) {
|
|
app = this;
|
|
queueMicrotask(args.at(-1));
|
|
return new EventEmitter() as any;
|
|
});
|
|
try {
|
|
await createServer(0);
|
|
} finally {
|
|
listen.mockRestore();
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
for (const event of events) {
|
|
for (const listener of process.listeners(event)) {
|
|
if (!originalListeners.get(event)!.includes(listener))
|
|
process.removeListener(event, listener);
|
|
}
|
|
}
|
|
vi.unstubAllEnvs();
|
|
fs.rmSync(fixtureRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
mocks.listRegisteredRepos.mockResolvedValue([entry]);
|
|
mocks.withLbugDb.mockImplementation(async (_path, callback) => callback());
|
|
mocks.search.mockImplementation(async (_query, _limit, _exec, reason) => ({
|
|
results: [],
|
|
ftsAvailable: !reason,
|
|
}));
|
|
});
|
|
|
|
async function invoke(route: string, query: Record<string, unknown> = {}) {
|
|
const layer = app.router.stack.find((item: any) => item.route?.path === route);
|
|
expect(layer, route).toBeDefined();
|
|
const handler = layer.route.stack.at(-1).handle;
|
|
const req = Object.assign(new EventEmitter(), {
|
|
query,
|
|
body: { cypher: 'RETURN 1 AS value', query: 'handler', mode: 'bm25', enrich: false },
|
|
});
|
|
const res = Object.assign(new EventEmitter(), {
|
|
statusCode: 200,
|
|
body: undefined as any,
|
|
writableEnded: false,
|
|
destroyed: false,
|
|
status(code: number) {
|
|
this.statusCode = code;
|
|
return this;
|
|
},
|
|
json(body: unknown) {
|
|
this.body = body;
|
|
return this;
|
|
},
|
|
set() {
|
|
return this;
|
|
},
|
|
setHeader() {
|
|
return this;
|
|
},
|
|
flushHeaders() {},
|
|
write() {
|
|
return true;
|
|
},
|
|
end() {
|
|
this.writableEnded = true;
|
|
this.emit('finish');
|
|
},
|
|
});
|
|
await handler(req, res);
|
|
expect(res.statusCode, JSON.stringify(res.body)).toBe(route === '/api/embed' ? 202 : 200);
|
|
return res;
|
|
}
|
|
|
|
const cases = [
|
|
{
|
|
name: 'flag-disabled',
|
|
fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'disabled-by-flag' },
|
|
skip: true,
|
|
},
|
|
{
|
|
name: 'env-disabled',
|
|
fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'disabled-by-env' },
|
|
skip: true,
|
|
},
|
|
{ name: 'normal', fts: { provider: 'ladybugdb-fts', status: 'available' }, skip: false },
|
|
{ name: 'legacy', fts: undefined, skip: false },
|
|
{
|
|
name: 'degraded',
|
|
fts: { provider: 'ladybugdb-fts', status: 'degraded', skipReason: 'build-failed' },
|
|
skip: false,
|
|
},
|
|
] as const;
|
|
|
|
describe('serve uses one metadata-derived FTS mode on every DB-open path', () => {
|
|
it.each(cases)(
|
|
'keeps mixed read requests consistent for $name indexes',
|
|
async ({ fts, skip }) => {
|
|
// The server process environment must not override persisted per-index intent.
|
|
vi.stubEnv('GITNEXUS_SKIP_FTS', skip ? undefined : '1');
|
|
mocks.loadMeta.mockResolvedValue({ capabilities: { fts } });
|
|
const sequence = [
|
|
['/api/search', {}],
|
|
['/api/query', {}],
|
|
['/api/graph', {}],
|
|
['/api/search', {}],
|
|
['/api/graph', { stream: 'true' }],
|
|
['/api/grep', { pattern: 'handler' }],
|
|
['/api/query', {}],
|
|
['/api/search', {}],
|
|
] as const;
|
|
for (const [route, query] of sequence) {
|
|
const response = await invoke(route, query);
|
|
if (route === '/api/search') {
|
|
expect(response.body.warning).toBe(skip ? FTS_DISABLED_MESSAGE : undefined);
|
|
}
|
|
}
|
|
expect(mocks.withLbugDb).toHaveBeenCalledTimes(sequence.length);
|
|
// Grep also loads metadata for getSourceAvailability before the FTS session.
|
|
expect(mocks.loadMeta).toHaveBeenCalledTimes(sequence.length + 1);
|
|
for (const [dbPath, , options] of mocks.withLbugDb.mock.calls) {
|
|
expect(dbPath).toBe(path.join(entry.storagePath, 'lbug'));
|
|
expect(options).toEqual({ readOnly: true, ...(skip ? { skipFts: true } : {}) });
|
|
}
|
|
},
|
|
);
|
|
|
|
it('reads mode changes between requests instead of caching stale metadata', async () => {
|
|
for (const mode of [cases[0], cases[2], cases[1]]) {
|
|
mocks.loadMeta.mockResolvedValue({ capabilities: { fts: mode.fts } });
|
|
await invoke('/api/query');
|
|
expect(mocks.withLbugDb.mock.lastCall?.[2]).toEqual({
|
|
readOnly: true,
|
|
...(mode.skip ? { skipFts: true } : {}),
|
|
});
|
|
}
|
|
});
|
|
|
|
it.each(cases)(
|
|
'preserves write mode while honoring $name metadata for embed',
|
|
async ({ fts, skip }) => {
|
|
mocks.loadMeta.mockResolvedValue({ capabilities: { fts } });
|
|
// This test stops at the DB boundary; it must not generate vectors or write an index.
|
|
mocks.withLbugDb.mockResolvedValue(undefined);
|
|
await invoke('/api/embed');
|
|
await vi.waitFor(() =>
|
|
expect(mocks.updateJob).toHaveBeenCalledWith(
|
|
'embed-job',
|
|
expect.objectContaining({ status: 'complete' }),
|
|
),
|
|
);
|
|
expect(mocks.withLbugDb).toHaveBeenCalledExactlyOnceWith(
|
|
path.join(entry.storagePath, 'lbug'),
|
|
expect.any(Function),
|
|
skip ? { skipFts: true } : {},
|
|
);
|
|
expect(mocks.loadMeta).toHaveBeenCalledExactlyOnceWith(entry.storagePath);
|
|
},
|
|
);
|
|
});
|
|
|
|
describe('GET /api/repos catalog validation', () => {
|
|
it('lists registered repos with validate: true', async () => {
|
|
mocks.loadMeta.mockResolvedValue({});
|
|
await invoke('/api/repos');
|
|
expect(mocks.listRegisteredRepos).toHaveBeenCalledWith({ validate: true });
|
|
expect(mocks.listRegisteredRepos).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|