diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 5f43df3e2..6c6b79b51 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1728,20 +1728,33 @@ async function runFullAnalysisInner( ? (table, indexName) => log(`FTS: ready ${table}.${indexName}`) : undefined, }); - const missing = await verifySearchFTSIndexes(executeQuery, ftsIndexes); - if (missing.length > 0) { - // #2889: name WHY each index is missing when the build itself said so. - // Repair now rebuilds every table it can before reporting, so the tables - // absent from this list were genuinely repaired even on a failed run — - // previously the first failure aborted the sweep and the message could - // only ever list "missing", never a reason. Same sentence the analyze - // degrade path prints, so one failure does not read two ways. - const reasons = + const catalogDiagnostics: string[] = []; + let missing: string[] = []; + let catalogError: string | undefined; + try { + missing = await verifySearchFTSIndexes(executeQuery, ftsIndexes, (diagnostic) => { + catalogDiagnostics.push(diagnostic.message); + }); + } catch (error) { + // An unreadable catalog is unknown, not proof of missing indexes. + // Keep earlier native build errors when this read also fails. + catalogError = error instanceof Error ? error.message : String(error); + } + if (repairFailures.length > 0 || missing.length > 0 || catalogError !== undefined) { + // A failed DROP may leave a valid old catalog entry. Its presence must + // not certify that this repair succeeded or clear the recovery marker. + const reasons = [ + missing.length > 0 ? `missing indexes after rebuild: ${missing.join(', ')}` : '', repairFailures.length > 0 - ? ` ${summarizeFtsIndexBuildFailures(repairFailures, ftsIndexes)}.` - : ''; + ? summarizeFtsIndexBuildFailures(repairFailures, ftsIndexes) + : missing.length > 0 + ? 'no build error was returned' + : '', + ...catalogDiagnostics, + catalogError !== undefined ? `catalog verification failed: ${catalogError}` : '', + ].filter(Boolean); throw new Error( - `FTS repair failed - missing indexes after rebuild: ${missing.join(', ')}.${reasons} ` + + `FTS repair failed - ${reasons.join('. ')}. ` + 'Run `gitnexus analyze --force` to perform a full graph+FTS rebuild; ' + 'if that also fails, verify FTS extension availability via `gitnexus doctor`.', ); diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 5f8b59f13..00d08f23a 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -3,6 +3,7 @@ import { dropFTSIndex, indexRowName, indexRowTable, + indexRowType, resolveGateRows, DEFAULT_FTS_STEMMER, type IndexCatalogSnapshot, @@ -394,9 +395,40 @@ export const summarizeFtsIndexBuildFailures = ( `FTS index build failed for ${failures.length} of ${indexes.length} tables: ` + failures.map((f) => `${f.table}.${f.indexName} (${f.error})`).join(', '); +/** Bounded catalog evidence for CLI/log diagnostics, never source or index definitions. */ +export interface FtsIndexVerificationDiagnostic { + table: string; + indexName: string; + message: string; +} + +const boundCatalogText = (text: string, limit: number): string => + text.length > limit ? `${text.slice(0, limit - 14)}… [truncated]` : text; + +const catalogFieldEvidence = (value: unknown): string => + typeof value === 'string' + ? JSON.stringify(boundCatalogText(value, 80)) + : value === undefined + ? '' + : ``; + +const catalogPropertiesEvidence = (value: unknown): string => { + if (!Array.isArray(value)) return ``; + return ( + `[${value.slice(0, 8).map(catalogFieldEvidence).join(', ')}]` + + (value.length > 8 ? ` (${value.length - 8} properties truncated)` : '') + ); +}; + +/** + * Return missing or invalid configured indexes. An optional callback receives + * bounded evidence from the same catalog read; a failed read still throws and + * proves neither presence nor absence. + */ export async function verifySearchFTSIndexes( executeQuery: (cypher: string) => Promise, indexes: readonly FTSIndexDefinition[] = FTS_INDEXES, + onMismatch?: (diagnostic: FtsIndexVerificationDiagnostic) => void, ): Promise { // Read the catalog once and check each configured index both EXISTS and // covers its expected columns. A queryability-only probe (CALL QUERY_FTS_INDEX @@ -404,34 +436,51 @@ export async function verifySearchFTSIndexes( // pre-#2299 DB stays queryable yet silently misses `description`, so the probe // would pass while doc-comment search is still broken (#2299). SHOW_INDEXES // exposes `property_names` (STRING[]) per index, so we assert coverage directly. - const rows = await executeQuery('CALL SHOW_INDEXES() RETURN *'); - - const propsByIndex = new Map(); - for (const row of rows) { - if (typeof row !== 'object' || row === null) continue; - const record = row as Record; - const indexName = indexRowName(record); - // LADYBUGDB-CONTRACT: `property_names` is the one SHOW_INDEXES column with a - // single reader, so it has no shared accessor — see {@link IndexCatalogRow} - // in lbug-adapter.ts for the full column list and the re-validation rule. - // Unlike the gates, an unreadable shape here is safe: it reports the index as - // not covering its columns, i.e. "missing", which degrades keyword search - // loudly rather than passing a broken index off as verified. - const propertyNames = record.property_names; - if (typeof indexName !== 'string' || !Array.isArray(propertyNames)) continue; - propsByIndex.set( - indexName, - propertyNames.filter((p): p is string => typeof p === 'string'), - ); - } + const rows = (await executeQuery('CALL SHOW_INDEXES() RETURN *')).filter( + (row): row is Record => typeof row === 'object' && row !== null, + ); const missing: string[] = []; for (const { table, indexName, properties } of indexes) { - const actual = propsByIndex.get(indexName); - // Absent from the catalog, or present but not covering every expected column. - if (!actual || !properties.every((p) => actual.includes(p))) { - missing.push(`${table}.${indexName}`); - } + const valid = rows.some((row) => { + // LADYBUGDB-CONTRACT: `property_names` is a named STRING[] column; + // identity/type use the shared catalog readers in lbug-adapter.ts. + const actual = row.property_names; + return ( + indexRowTable(row) === table && + indexRowName(row) === indexName && + indexRowType(row) === 'FTS' && + Array.isArray(actual) && + actual.every((property) => typeof property === 'string') && + properties.every((property) => actual.includes(property)) + ); + }); + if (valid) continue; + missing.push(`${table}.${indexName}`); + if (!onMismatch) continue; + + // Show only rows that could explain this identity mismatch, and only + // catalog identity/type/property fields. Malformed objects are represented + // by their type rather than serialized (they can carry arbitrary content). + const relevant = rows.filter( + (row) => indexRowTable(row) === table || indexRowName(row) === indexName, + ); + const observed = relevant + .slice(0, 3) + .map( + (row) => + `{table=${catalogFieldEvidence(indexRowTable(row))}, ` + + `index=${catalogFieldEvidence(indexRowName(row))}, ` + + `type=${catalogFieldEvidence(indexRowType(row))}, ` + + `properties=${catalogPropertiesEvidence(row.property_names)}}`, + ); + const message = + `${table}.${indexName}: expected table=${catalogFieldEvidence(table)}, ` + + `index=${catalogFieldEvidence(indexName)}, type=FTS, ` + + `properties=${catalogPropertiesEvidence(properties)}; observed ` + + (observed.length > 0 ? observed.join(', ') : 'no matching table or index name') + + (relevant.length > 3 ? ` (${relevant.length - 3} catalog rows truncated)` : ''); + onMismatch({ table, indexName, message: boundCatalogText(message, 2048) }); } return missing; } @@ -535,19 +584,39 @@ export async function buildSearchIndexesOrDegrade( // means description search is broken (#2299). const failures = await createSearchFTSIndexes(options); const indexes = options?.indexes ?? FTS_INDEXES; - const missing = await verifySearchFTSIndexes(executeQuery, indexes); - if (failures.length === 0 && missing.length === 0) return { ok: true }; + const diagnostics: FtsIndexVerificationDiagnostic[] = []; + let missing: string[] = []; + let verificationError: string | undefined; + try { + missing = await verifySearchFTSIndexes(executeQuery, indexes, (diagnostic) => + diagnostics.push(diagnostic), + ); + } catch (e) { + // A failed catalog read is unknown, not absence. Keep the native build + // failures already collected rather than replacing them with this error. + verificationError = e instanceof Error ? e.message : String(e); + } + if (failures.length === 0 && missing.length === 0 && verificationError === undefined) { + return { ok: true }; + } - // A table that failed to build is necessarily missing too — report it once, - // with its reason, and keep `missing` for indexes nothing explains. + // A failed DROP may leave an old catalog row present. Build failures stand + // on their own; avoid naming the same table again as an unexplained failure. const named = new Set(failures.map((f) => `${f.table}.${f.indexName}`)); - const unexplained = missing.filter((name) => !named.has(name)); + const unexplained = diagnostics.filter((d) => !named.has(`${d.table}.${d.indexName}`)); const error = [ failures.length > 0 ? summarizeFtsIndexBuildFailures(failures, indexes) : '', // Structural incompleteness with no thrown error — classified capability // (degrade) below, matching prior behavior; a broken *write* surfaces as // a thrown IO/checkpoint error and is classified integrity there. - unexplained.length > 0 ? `missing indexes after build: ${unexplained.join(', ')}` : '', + unexplained.length > 0 + ? 'missing indexes or mismatched catalog rows after build ' + + '(no build error was returned for these indexes): ' + + unexplained.map((d) => d.message).join('; ') + : '', + verificationError !== undefined + ? `FTS catalog verification failed: ${verificationError}` + : '', ] .filter((part) => part.length > 0) .join('; '); @@ -555,9 +624,11 @@ export async function buildSearchIndexesOrDegrade( // Classify per failure, not over the joined text: capability signatures are // checked first, so folding the messages together would let an untokenizable // row mask a genuinely broken write and downgrade an abort into a degrade. - const failureClass = failures.some((f) => classifyFtsBuildError(f.error) === 'integrity') - ? 'integrity' - : classifyFtsBuildError(error); + const failureClass = + failures.some((f) => classifyFtsBuildError(f.error) === 'integrity') || + (verificationError !== undefined && classifyFtsBuildError(verificationError) === 'integrity') + ? 'integrity' + : 'capability'; return { ok: false, error, failureClass }; } catch (e) { const error = e instanceof Error ? e.message : String(e); diff --git a/gitnexus/test/integration/incremental-worktree-context.test.ts b/gitnexus/test/integration/incremental-worktree-context.test.ts new file mode 100644 index 000000000..d8292813b --- /dev/null +++ b/gitnexus/test/integration/incremental-worktree-context.test.ts @@ -0,0 +1,288 @@ +/** + * Characterize #3423's public workflow against the current analyzer. The + * original private database is unavailable; this fixture does not reproduce + * or establish the cause of that historical incident. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it, vi } from 'vitest'; +import { SupportedLanguages } from '../../src/config/supported-languages.js'; +import * as adapter from '../../src/core/lbug/lbug-adapter.js'; +import { runFullAnalysis } from '../../src/core/run-analyze.js'; +import { isLanguageAvailable } from '../../src/core/tree-sitter/parser-loader.js'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { getStoragePaths, loadMeta } from '../../src/storage/repo-manager.js'; +import { CLI_SPAWN_PREFIX, tsxLoaderUrl } from '../helpers/cli-entry.js'; +import { commitAll, initGitRepo } from '../helpers/temp-git-repo.js'; +import { createTempDir } from '../helpers/test-db.js'; + +const SWIFT_PATH = 'Sources/Numbers.swift'; +const TEST_PATH = 'checks/test_numbers.py'; +const IMPORTER_PATH = 'checks/runner.py'; +const TARGET_NAME = 'extractLeadingNumber'; +const PROPERTY_NAME = 'preservedSearchNeedle'; +const CALLER_NAMES = ['parsePrimaryNumber', 'parseSecondaryNumber']; +const options = { skipAgentsMd: true, skipSkills: true }; +const callbacks = { onProgress: () => {} }; + +const SWIFT_SOURCE = `func extractLeadingNumber(_ raw: String) -> Int { + return 7 +} + +func parsePrimaryNumber() -> Int { + return extractLeadingNumber("17") +} + +func parseSecondaryNumber() -> Int { + return extractLeadingNumber("23") +} + +class NumberSettings { + var preservedSearchNeedle: String = "retained property" +} +`; + +type SymbolRef = { uid: string; name: string; filePath: string }; +type ContextResult = { + status: string; + symbol: SymbolRef; + incoming: { calls?: SymbolRef[] }; +}; +type StatusResult = { + repository: string; + status: string; + contentDrift: { status: string; coveredFiles?: number }; + index: { incompleteReasons: string[]; runnerIdentityStatus: string }; +}; + +const git = (cwd: string, ...args: string[]): string => + execFileSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }).trim(); + +function runCli(cwd: string, args: string[], prefix = CLI_SPAWN_PREFIX): T { + const stdout = execFileSync(process.execPath, [...prefix, ...args], { + cwd, + env: process.env, + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 4 * 1024 * 1024, + windowsHide: true, + }); + // Context prints a human-readable banner before its JSON response. + const start = stdout.indexOf('{'); + expect(start, stdout).toBeGreaterThanOrEqual(0); + return JSON.parse(stdout.slice(start)) as T; +} + +function expectContext(result: ContextResult, target: SymbolRef, callers: SymbolRef[]): void { + expect(result).not.toHaveProperty('error'); + expect(result.status).toBe('found'); + expect(result.symbol).toMatchObject(target); + const actual = (result.incoming.calls ?? []).map(({ uid, name, filePath }) => ({ + uid, + name, + filePath, + })); + expect(actual.sort((a, b) => a.uid.localeCompare(b.uid))).toEqual( + [...callers].sort((a, b) => a.uid.localeCompare(b.uid)), + ); +} + +describe.skipIf(!isLanguageAvailable(SupportedLanguages.Swift))( + 'unchanged Swift context across a selective linked-worktree refresh (#3423)', + () => { + it('retains CLI/MCP name and UID context, both callers, and native Property FTS', async (ctx) => { + const temp = await createTempDir(); + let backend: LocalBackend | undefined; + try { + const root = await fs.realpath(temp.dbPath); + const main = path.join(root, 'main'); + const worktree = path.join(root, 'linked'); + const nativeHome = path.join(root, 'user-home'); + await fs.mkdir(nativeHome); + vi.stubEnv('HOME', nativeHome); + vi.stubEnv('USERPROFILE', nativeHome); + vi.stubEnv('GITNEXUS_HOME', path.join(root, 'home')); + vi.stubEnv('GITNEXUS_STORAGE_PATH', undefined); + vi.stubEnv('GITNEXUS_STORAGE_ROOT', undefined); + // Keep a real linked checkout, but avoid the shared-store seed/full + // rebuild path: the behavior under test is selective row replacement. + vi.stubEnv('GITNEXUS_SHARED_STORE', 'off'); + vi.stubEnv('GITNEXUS_CONTENT_RETENTION', 'full'); + vi.stubEnv('GITNEXUS_WAL_MANUAL_CHECKPOINT', '1'); + vi.stubEnv('GITNEXUS_LBUG_EXTENSION_INSTALL', 'load-only'); + + await adapter.initLbug(path.join(root, 'fts-probe')); + // Require an existing extension; a network install would make this + // isolated fixture depend on the developer's machine or connectivity. + if (!(await adapter.loadFTSExtension(undefined, { policy: 'load-only' }))) { + if (process.env.GITNEXUS_REQUIRE_FTS === '1') { + throw new Error( + 'FTS is required (GITNEXUS_REQUIRE_FTS=1) but could not load in the isolated fixture.', + ); + } + ctx.skip('FTS extension unavailable under the load-only policy'); + } + await adapter.closeLbug(); + + await fs.mkdir(path.join(main, 'Sources'), { recursive: true }); + await fs.mkdir(path.join(main, 'checks'), { recursive: true }); + await fs.mkdir(path.join(main, 'retained'), { recursive: true }); + await fs.writeFile(path.join(main, '.gitignore'), '.gitnexus/\n'); + await fs.writeFile(path.join(main, SWIFT_PATH), SWIFT_SOURCE); + await fs.writeFile(path.join(main, 'checks/__init__.py'), ''); + await fs.writeFile(path.join(main, TEST_PATH), 'def fixture_number():\n return 7\n'); + await fs.writeFile( + path.join(main, IMPORTER_PATH), + 'from checks.test_numbers import fixture_number\n\ndef run_checks():\n return fixture_number()\n', + ); + // Keep the two-file write set below the 50-file / 50% escalation gate. + for (let i = 0; i < 55; i++) { + await fs.writeFile( + path.join(main, `retained/value_${i}.py`), + `def retained_value_${i}():\n return ${i}\n`, + ); + } + initGitRepo(main); + git(main, 'config', 'commit.gpgsign', 'false'); + commitAll(main, 'baseline mixed-language fixture'); + git(main, 'worktree', 'add', '-q', '-b', 'lookup-refresh', worktree); + expect((await fs.stat(path.join(worktree, '.git'))).isFile()).toBe(true); + expect(git(worktree, 'rev-parse', 'HEAD')).toBe(git(main, 'rev-parse', 'HEAD')); + + const { storagePath, lbugPath } = getStoragePaths(worktree); + expect(storagePath).toBe(path.join(worktree, '.gitnexus')); + const baseline = await runFullAnalysis(worktree, options, callbacks); + const baselineMeta = await loadMeta(storagePath); + expect(baselineMeta?.runnerIdentity).toBeDefined(); + const swiftFunctions = [...baseline.pipelineResult.graph.iterNodes()].filter( + (node) => node.label === 'Function' && node.properties.filePath === SWIFT_PATH, + ); + function symbol(name: string): SymbolRef { + const matches = swiftFunctions.filter((node) => node.properties.name === name); + expect(matches, name).toHaveLength(1); + return { uid: matches[0].id, name, filePath: SWIFT_PATH }; + } + const target = symbol(TARGET_NAME); + const callers = CALLER_NAMES.map(symbol); + expect(new Set(callers.map(({ uid }) => uid)).size).toBe(2); + + // runFullAnalysis is imported from source. Use the same source build + // for status so it actually measures content drift instead of stopping + // at a source/dist runner-identity mismatch. Context may use either. + const statusPrefix = [ + '--import', + tsxLoaderUrl(), + fileURLToPath(new URL('../../src/cli/index.ts', import.meta.url)), + ]; + const readStatus = (): StatusResult => + runCli(worktree, ['status', '--json'], statusPrefix); + const expectCurrentStatus = (): void => { + const status = readStatus(); + expect(status.repository).toBe(worktree); + expect(status.index.runnerIdentityStatus).toBe('current'); + expect(status.contentDrift.status).toBe('current'); + expect(status.contentDrift.coveredFiles).toBeGreaterThan(50); + expect(status.index.incompleteReasons).toEqual([]); + expect(status.status).toBe('up-to-date'); + }; + const readPublicContexts = async (): Promise => { + // CLI and MCP own separate native readers, released before writes. + expectContext(runCli(worktree, ['context', TARGET_NAME]), target, callers); + expectContext(runCli(worktree, ['context', '--uid', target.uid]), target, callers); + backend = new LocalBackend(); + try { + expect(await backend.init()).toBe(true); + for (let repeat = 0; repeat < 2; repeat++) { + expectContext( + await backend.callTool('context', { name: TARGET_NAME }), + target, + callers, + ); + expectContext( + await backend.callTool('context', { uid: target.uid }), + target, + callers, + ); + } + } finally { + await backend.dispose(); + backend = undefined; + } + }; + const readPropertyFts = async (): Promise => { + await adapter.initLbug(lbugPath); + try { + const rows = await adapter.executeQuery( + `CALL QUERY_FTS_INDEX('Property', 'property_fts', '${PROPERTY_NAME}') + RETURN node.id AS uid, node.name AS name, node.filePath AS filePath`, + ); + expect(rows).toEqual([ + { uid: expect.any(String), name: PROPERTY_NAME, filePath: SWIFT_PATH }, + ]); + return rows; + } finally { + await adapter.closeLbug(); + } + }; + + await readPublicContexts(); + expectCurrentStatus(); + const propertyBefore = await readPropertyFts(); + + await fs.writeFile(path.join(worktree, TEST_PATH), 'def fixture_number():\n return 8\n'); + expect(git(worktree, 'diff', '--name-only')).toBe(TEST_PATH); + const dirtyStatus = readStatus(); + expect(dirtyStatus.contentDrift.status).toBe('drifted'); + expect(dirtyStatus.index.incompleteReasons).toEqual([]); + + // A call-through spy observes the exact native deletion boundary; it + // neither supplies graph rows nor substitutes for the incremental write. + const deleteNodes = vi.spyOn(adapter, 'deleteNodesForFiles'); + const refreshed = await runFullAnalysis(worktree, options, callbacks); + expect(refreshed.incrementalStats).toMatchObject({ + changedFiles: 1, + writeMode: 'incremental', + }); + expect(refreshed.incrementalStats?.affectedDependents).toBeGreaterThan(0); + expect(deleteNodes).toHaveBeenCalledTimes(1); + const writePaths = [...deleteNodes.mock.calls[0][0]].sort(); + // Effective write-set expansion also refreshes the containing Folder + // row through graph edges; no other source file enters the write set. + expect(writePaths).toEqual(['checks', TEST_PATH, IMPORTER_PATH].sort()); + expect(writePaths.filter((filePath) => filePath.endsWith('.py'))).toEqual( + [TEST_PATH, IMPORTER_PATH].sort(), + ); + expect(writePaths).not.toContain(SWIFT_PATH); + expect((await loadMeta(storagePath))?.runnerIdentity).toEqual(baselineMeta?.runnerIdentity); + expect(await fs.readFile(path.join(worktree, SWIFT_PATH), 'utf8')).toBe(SWIFT_SOURCE); + await readPublicContexts(); + expectCurrentStatus(); + expect(await readPropertyFts()).toEqual(propertyBefore); + + deleteNodes.mockClear(); + const noop = await runFullAnalysis(worktree, options, callbacks); + // The edit remains uncommitted, so clean-porcelain early return is + // unavailable. Identical content instead takes a zero-file refresh. + expect(noop.incrementalStats).toMatchObject({ + changedFiles: 0, + affectedDependents: 0, + deletedFiles: 0, + writeMode: 'incremental', + }); + expect(deleteNodes.mock.calls.flatMap(([filePaths]) => filePaths)).toEqual([]); + expect((await loadMeta(storagePath))?.runnerIdentity).toEqual(baselineMeta?.runnerIdentity); + await readPublicContexts(); + expectCurrentStatus(); + expect(await readPropertyFts()).toEqual(propertyBefore); + } finally { + await backend?.dispose(); + await adapter.closeLbug(); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await temp.cleanup(); + } + }, 180_000); + }, +); diff --git a/gitnexus/test/unit/bm25-search.test.ts b/gitnexus/test/unit/bm25-search.test.ts index 08b2fc46a..87b750fb7 100644 --- a/gitnexus/test/unit/bm25-search.test.ts +++ b/gitnexus/test/unit/bm25-search.test.ts @@ -45,7 +45,9 @@ describe('BM25 search', () => { // One SHOW_INDEXES call returns a catalog row per configured index, each // covering exactly its expected properties. const showIndexesRows = FTS_INDEXES.map((i) => ({ + table_name: i.table, index_name: i.indexName, + index_type: 'FTS', property_names: [...i.properties], })); const executeQuery = vi.fn().mockResolvedValue(showIndexesRows); @@ -62,7 +64,9 @@ describe('BM25 search', () => { // missing `description`. Every other index covers its columns. const staleIndex = 'function_fts'; const showIndexesRows = FTS_INDEXES.map((i) => ({ + table_name: i.table, index_name: i.indexName, + index_type: 'FTS', property_names: i.indexName === staleIndex ? ['name', 'content'] : [...i.properties], })); const executeQuery = vi.fn().mockResolvedValue(showIndexesRows); @@ -77,7 +81,9 @@ describe('BM25 search', () => { // Every configured index present and covering, except const_fts is missing. const absentIndex = 'const_fts'; const showIndexesRows = FTS_INDEXES.filter((i) => i.indexName !== absentIndex).map((i) => ({ + table_name: i.table, index_name: i.indexName, + index_type: 'FTS', property_names: [...i.properties], })); const executeQuery = vi.fn().mockResolvedValue(showIndexesRows); diff --git a/gitnexus/test/unit/fts-indexes.test.ts b/gitnexus/test/unit/fts-indexes.test.ts index 84a9dd0ee..37026dede 100644 --- a/gitnexus/test/unit/fts-indexes.test.ts +++ b/gitnexus/test/unit/fts-indexes.test.ts @@ -34,13 +34,19 @@ const { getSearchFTSStemmer, initialiseSearchFTSStemmer, missingSearchFTSIndexTables, + verifySearchFTSIndexes, } = await import('../../src/core/search/fts-indexes.js'); const { FTS_INDEXES, getFtsIndexes } = await import('../../src/core/search/fts-schema.js'); const { createFTSIndex } = await import('../../src/core/lbug/lbug-adapter.js'); /** SHOW_INDEXES rows covering every configured FTS index's expected properties. */ const fullCoverageRows = () => - FTS_INDEXES.map((i) => ({ index_name: i.indexName, property_names: [...i.properties] })); + FTS_INDEXES.map((i) => ({ + table_name: i.table, + index_name: i.indexName, + index_type: 'FTS', + property_names: [...i.properties], + })); /** The row-level tokenizer error of #2544/#2546/#2889, verbatim. */ const POISON = 'Runtime exception: Failed calling LOWER: Invalid UTF-8.'; @@ -141,6 +147,91 @@ describe('createSearchFTSIndexes', () => { }); }); +describe('verifySearchFTSIndexes', () => { + const propertyIndex = FTS_INDEXES.find((index) => index.table === 'Property')!; + const propertyRow = () => fullCoverageRows().find((row) => row.table_name === 'Property')!; + + it('accepts exact identity/type with additional properties, using one catalog read', async () => { + const executeQuery = vi.fn(async () => [ + { ...propertyRow(), property_names: [...propertyIndex.properties, 'extra'] }, + ]); + const onMismatch = vi.fn(); + + expect(await verifySearchFTSIndexes(executeQuery, [propertyIndex], onMismatch)).toEqual([]); + expect(executeQuery).toHaveBeenCalledExactlyOnceWith('CALL SHOW_INDEXES() RETURN *'); + expect(onMismatch).not.toHaveBeenCalled(); + }); + + it.each([ + ['wrong table', { table_name: 'Function' }], + ['wrong type', { index_type: 'HASH' }], + ['missing type', { index_type: undefined }], + ['missing properties', { property_names: undefined }], + ['non-array properties', { property_names: 'name,content,description' }], + ['malformed properties', { property_names: ['name', 'content', 'description', 17] }], + ['incomplete properties', { property_names: ['name', 'content'] }], + ])('rejects %s without accepting the matching index name', async (_label, mismatch) => { + const executeQuery = vi.fn(async () => [{ ...propertyRow(), ...mismatch }]); + const onMismatch = vi.fn(); + + expect(await verifySearchFTSIndexes(executeQuery, [propertyIndex], onMismatch)).toEqual([ + 'Property.property_fts', + ]); + expect(onMismatch).toHaveBeenCalledExactlyOnceWith({ + table: 'Property', + indexName: 'property_fts', + message: expect.stringContaining('Property.property_fts'), + }); + expect(onMismatch.mock.calls[0][0].message).toContain('observed'); + }); + + it('accepts a valid row even when another table has the same index name', async () => { + const executeQuery = vi.fn(async () => [ + propertyRow(), + { ...propertyRow(), table_name: 'Function', property_names: ['name'] }, + ]); + + expect(await verifySearchFTSIndexes(executeQuery, [propertyIndex])).toEqual([]); + }); + + it('bounds diagnostic evidence and excludes source and index definitions', async () => { + const privateSource = 'PRIVATE_SOURCE_MUST_NOT_APPEAR'; + const executeQuery = vi.fn(async () => + Array.from({ length: 50 }, (_, i) => ({ + ...propertyRow(), + table_name: `Wrong${i}${'x'.repeat(10_000)}`, + property_names: [{ source: privateSource }, ...Array(100).fill('x'.repeat(10_000))], + content: privateSource, + index_definition: privateSource, + })), + ); + const onMismatch = vi.fn(); + + expect(await verifySearchFTSIndexes(executeQuery, [propertyIndex], onMismatch)).toEqual([ + 'Property.property_fts', + ]); + const message = onMismatch.mock.calls[0][0].message as string; + expect(message).toContain('Wrong0'); + expect(message).toContain('truncated'); + expect(message.length).toBeLessThanOrEqual(2048); + expect(message).not.toContain(privateSource); + expect(message).not.toContain('index_definition'); + expect(executeQuery).toHaveBeenCalledTimes(1); + }); + + it('preserves unknown catalog state as an exception, without reporting absence', async () => { + const executeQuery = vi.fn(async () => { + throw new Error('catalog unavailable'); + }); + const onMismatch = vi.fn(); + + await expect(verifySearchFTSIndexes(executeQuery, [propertyIndex], onMismatch)).rejects.toThrow( + 'catalog unavailable', + ); + expect(onMismatch).not.toHaveBeenCalled(); + }); +}); + describe('buildSearchIndexesOrDegrade', () => { it('returns ok:true when every index builds and verifies (#2544/#2546)', async () => { const executeQuery = vi.fn(async () => fullCoverageRows()); @@ -215,6 +306,69 @@ describe('buildSearchIndexesOrDegrade', () => { expect(result.error).toContain(`${failed} (${POISON})`); expect(result.error).not.toContain('missing indexes'); }); + + it('explains missing catalog coverage when CREATE returned no build error', async () => { + const executeQuery = vi.fn(async () => + fullCoverageRows().filter((row) => row.table_name !== 'Property'), + ); + + const result = await buildSearchIndexesOrDegrade(executeQuery); + + expect(result.ok).toBe(false); + expect(result.failureClass).toBe('capability'); + expect(result.error).toContain('Property.property_fts'); + expect(result.error).toContain('no build error was returned'); + expect(result.error).toContain('observed'); + expect(result.error).toContain('no matching'); + expect(executeQuery).toHaveBeenCalledTimes(1); + }); + + it.each([ + ['integrity', 'IO exception: checkpoint failed', 'integrity'], + ['capability', POISON, 'capability'], + ['unknown', 'catalog unavailable', 'capability'], + ])( + 'classifies a catalog-only %s error after every index builds', + async (_label, error, failureClass) => { + const onIndexReady = vi.fn(); + const executeQuery = vi.fn(async () => { + throw new Error(error); + }); + + const result = await buildSearchIndexesOrDegrade(executeQuery, { onIndexReady }); + + expect(createFTSIndex).toHaveBeenCalledTimes(FTS_INDEXES.length); + expect(onIndexReady.mock.calls).toEqual( + FTS_INDEXES.map(({ table, indexName }) => [table, indexName]), + ); + expect(executeQuery).toHaveBeenCalledExactlyOnceWith('CALL SHOW_INDEXES() RETURN *'); + expect(result).toEqual({ + ok: false, + error: `FTS catalog verification failed: ${error}`, + failureClass, + }); + }, + ); + + it('retains integrity and tokenizer causes when catalog verification also throws', async () => { + const integrityError = 'IO exception: checkpoint failed'; + vi.mocked(createFTSIndex) + .mockRejectedValueOnce(new Error(integrityError)) + .mockRejectedValueOnce(new Error(POISON)); + const executeQuery = vi.fn(async () => { + throw new Error('catalog unavailable'); + }); + + const result = await buildSearchIndexesOrDegrade(executeQuery); + + expect(result.ok).toBe(false); + expect(result.failureClass).toBe('integrity'); + expect(result.error).toContain(integrityError); + expect(result.error).toContain(POISON); + expect(result.error).toContain('catalog unavailable'); + expect(result.error).not.toContain('missing indexes'); + expect(executeQuery).toHaveBeenCalledTimes(1); + }); }); describe('missingSearchFTSIndexTables (#3016)', () => { diff --git a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts index 8592546dc..aaa5c58a8 100644 --- a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts +++ b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts @@ -730,6 +730,68 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { await adapter.closeLbug(); }); + it.each([false, true])( + 'drains every FTS CREATE result and preserves a deferred failure (failure=%s)', + async (fails) => { + vi.resetModules(); + const nativeError = new Error('Runtime exception: Failed calling LOWER: Invalid UTF-8.'); + const events: string[] = []; + const results = ['first', 'second', 'third'].map((name) => ({ + getAll: vi.fn(async () => { + events.push(`${name}:getAll`); + if (fails && name === 'second') throw nativeError; + return []; + }), + close: vi.fn(() => { + events.push(`${name}:close`); + }), + })); + const genericResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const conn = makeConn(async (sql: string) => + sql.startsWith('CALL CREATE_FTS_INDEX') ? results : genericResult, + ); + const db = { close: vi.fn(async () => {}) }; + const dbPath = '/tmp/gitnexus-lbug-fts-create-drain/lbug'; + mockFsForInit(dbPath); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: vi.fn(async () => ({ db, conn })), + closeLbugConnection: vi.fn(async () => {}), + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + waitForWindowsHandleRelease: vi.fn(async () => true), + isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + STORAGE_VERSION_MISMATCH_SUGGESTION: '', + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(dbPath); + try { + const create = adapter.createFTSIndex('Property', 'property_fts', ['name']); + if (fails) await expect(create).rejects.toBe(nativeError); + else await expect(create).resolves.toBeUndefined(); + expect(events).toEqual([ + 'first:getAll', + 'first:close', + 'second:getAll', + 'second:close', + 'third:getAll', + 'third:close', + ]); + for (const result of results) expect(result.close).toHaveBeenCalledOnce(); + } finally { + await adapter.closeLbug(); + } + }, + ); + it('closes non-first stream query results when LadybugDB returns an array', async () => { vi.resetModules(); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 452166590..b6dda595c 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -295,6 +295,123 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { ...overrides, }); + it.each([ + { label: 'old catalog entry survives a failed DROP', buildError: true, catalog: 'complete' }, + { label: 'CREATE succeeds without a catalog entry', buildError: false, catalog: 'missing' }, + { label: 'build and catalog verification both fail', buildError: true, catalog: 'missing' }, + { label: 'catalog read throws after a build failure', buildError: true, catalog: 'throws' }, + { label: 'catalog read alone throws', buildError: false, catalog: 'throws' }, + ])('keeps repair incomplete when $label', async ({ buildError, catalog }) => { + const closeLbug = vi.fn(async () => undefined); + vi.doMock('../../src/core/lbug/lbug-adapter.js', () => + mockRepairSuccessLbugAdapter({ closeLbug }), + ); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + createSearchFTSIndexes: vi.fn(async () => + buildError + ? [ + { + table: 'Property', + indexName: 'property_fts', + error: 'DROP failed: native I/O error', + }, + ] + : [], + ), + verifySearchFTSIndexes: vi.fn(async (_query, _indexes, onMismatch) => { + if (catalog === 'throws') throw new Error('catalog read unavailable'); + if (catalog === 'missing') { + onMismatch?.({ + table: 'Property', + indexName: 'property_fts', + message: 'Property.property_fts: no catalog row returned', + }); + return ['Property.property_fts']; + } + return []; + }), + })); + + const tmpRepo = await createTempDir('gitnexus-repair-incomplete-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + const seeded: RepoMeta = { + repoPath: tmpRepo.dbPath, + lastCommit: 'healthy-commit', + indexedAt: '2026-01-01T00:00:00.000Z', + stats: { files: 7, nodes: 42, edges: 10, embeddings: 3 }, + runnerIdentity: { + schemaVersion: 4, + runtime: { + executablePath: '/usr/bin/node', + version: 'v24.11.0', + platform: 'linux', + architecture: 'x64', + modulesAbi: '137', + libc: 'glibc', + }, + cliVersion: '1.0.0', + invokedArtifact: { path: '/x/cli/index.ts', digest: 'src-digest' }, + build: { + kind: 'source' as const, + rootPath: '/x', + canonicalization: 'gitnexus-analyzer-build-v2', + digest: 'build-digest', + }, + dependencyRuntime: { + manifestPath: '/x/package.json', + lockfilePath: null, + canonicalization: 'gitnexus-analyzer-dependency-runtime-v4', + packageCount: 1, + artifactCount: 1, + digest: 'dep-digest', + }, + }, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'degraded' }, + vectorSearch: { provider: 'exact-scan', status: 'unavailable', exactScanLimit: 500 }, + }, + }; + await saveMeta(storagePath, seeded); + await createPlaceholderGraphStore(lbugPath); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const error = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { onProgress: () => {} }, + ).then( + () => null, + (cause: unknown) => cause, + ); + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain('FTS repair failed'); + if (buildError) expect(message).toContain('DROP failed: native I/O error'); + if (catalog === 'missing') { + expect(message).toContain('Property.property_fts: no catalog row returned'); + if (!buildError) expect(message).toContain('no build error was returned'); + } + if (catalog === 'throws') { + expect(message).toContain('catalog read unavailable'); + expect(message).not.toContain('missing indexes'); + } + const after = await loadMeta(storagePath); + expect(after?.incrementalInProgress).toMatchObject({ phase: 'fts' }); + expect(after?.capabilities).toEqual(seeded.capabilities); + expect(after?.indexedAt).toBe(seeded.indexedAt); + expect(after?.lastCommit).toBe(seeded.lastCommit); + expect(after?.stats).toEqual(seeded.stats); + expect(after?.runnerIdentity).toEqual(seeded.runnerIdentity); + expect(closeLbug).toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }); + it('--repair-fts stamps capabilities.fts.status while leaving indexedAt/lastCommit/runnerIdentity/stats byte-identical (#2767)', async () => { vi.doMock('../../src/core/lbug/lbug-adapter.js', () => mockRepairSuccessLbugAdapter()); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 53e9f0ba1..2b1d43c38 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -69,6 +69,7 @@ export default defineConfig({ 'test/integration/skip-fts.test.ts', 'test/integration/impact-callable-value-references.test.ts', 'test/integration/impact-context-integrity.test.ts', + 'test/integration/incremental-worktree-context.test.ts', 'test/integration/impact-epistemic-lower-bound.test.ts', 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts', @@ -165,6 +166,7 @@ export default defineConfig({ 'test/integration/skip-fts.test.ts', 'test/integration/impact-callable-value-references.test.ts', 'test/integration/impact-context-integrity.test.ts', + 'test/integration/incremental-worktree-context.test.ts', 'test/integration/impact-epistemic-lower-bound.test.ts', 'test/integration/impact-scope-omission-persistence.test.ts', 'test/integration/lbug-core-adapter.test.ts',