From 4d7a0a69edcc87ee2b2046aba7fbe8040c0de3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 18 Jul 2026 08:44:09 +0100 Subject: [PATCH 01/15] fix(analyze): degrade FTS search instead of aborting analyze on index-build failure (#2548) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(analyze): degrade FTS search instead of aborting analyze on index-build failure createSearchFTSIndexes re-tokenizes every stored row on every analyze run (full or incremental). A native LadybugDB tokenizer error on a single pre-existing row (e.g. "Failed calling LOWER: Invalid UTF-8") previously propagated uncaught out of run-analyze.ts's main FTS phase, discarding an otherwise-successful run's graph/embeddings work every time analyze ran thereafter. Add buildSearchIndexesOrDegrade(), which catches build/verify failures and lets analyze finish with keyword search degraded for that run instead — mirroring the existing sibling degrade path for a missing FTS extension. The dedicated --repair-fts path is untouched and still fails loudly. Fixes #2544, #2546. * fix(analyze): keep capabilities.fts/ftsSkipped honest when index build degrades ftsSkipped and capabilities.fts.status were keyed only on ftsAvailable (extension loaded), which the new degrade path leaves true even when the index build itself failed. Track that outcome in ftsReady and use it for both, and update run-analyze-fts-repair.test.ts's coverage of this path from asserting the old throw to asserting the new degrade contract (ftsSkipped, log message, meta.json capabilities.fts.status). --- gitnexus/src/core/run-analyze.ts | 30 +++++++++---- gitnexus/src/core/search/fts-indexes.ts | 31 +++++++++++++ gitnexus/test/unit/fts-indexes.test.ts | 44 ++++++++++++++++++- .../test/unit/run-analyze-fts-repair.test.ts | 40 +++++++++++------ 4 files changed, 120 insertions(+), 25 deletions(-) diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 18f9f37aa..bfbcb152f 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -36,6 +36,7 @@ import { } from './lbug/lbug-adapter.js'; import { escapeCypherString } from './lbug/cypher-escape.js'; import { + buildSearchIndexesOrDegrade, createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, @@ -1623,8 +1624,17 @@ export async function runFullAnalysis( const ftsAvailable = await loadFTSExtension(undefined, { policy: resolveAnalyzeInstallPolicy(), }); + // Tracks whether search indexes actually ended up usable this run — starts + // as ftsAvailable (extension loaded) but flips to false below when the + // build/verify step itself fails, so capabilities.fts.status / ftsSkipped + // stay honest even though that failure no longer aborts the whole analyze. + let ftsReady = ftsAvailable; if (ftsAvailable) { - await createSearchFTSIndexes({ + // Degrade rather than throw: createSearchFTSIndexes re-tokenizes every + // stored row on every run, so a native tokenizer error on a single + // pre-existing row (#2544/#2546) must not discard this run's otherwise- + // successful graph/embeddings work — only keyword search degrades. + const ftsResult = await buildSearchIndexesOrDegrade(executeQuery, { onIndexStart: options.verbose ? (table, indexName) => log(`FTS: creating ${table}.${indexName}`) : undefined, @@ -1632,14 +1642,16 @@ export async function runFullAnalysis( ? (table, indexName) => log(`FTS: ready ${table}.${indexName}`) : undefined, }); - const missingIndexNames = await verifySearchFTSIndexes(executeQuery); - if (missingIndexNames.length > 0) { - throw new Error( - `FTS verification failed - missing indexes after analyze: ${missingIndexNames.join(', ')}. ` + - 'Check FTS extension availability, then retry `gitnexus analyze --force` for a full rebuild.', + if (ftsResult.ok) { + progress('fts', 90, 'Search indexes ready'); + } else { + ftsReady = false; + log( + `FTS index build failed (${ftsResult.error}) — keyword search degraded this run. ` + + 'Graph and embeddings analysis completed successfully. Run `gitnexus analyze --repair-fts` to retry.', ); + progress('fts', 90, 'Search indexes skipped (build failed)'); } - progress('fts', 90, 'Search indexes ready'); } else { // For a missing runtime dependency (#2374) the file is present, so the // generic "install it with network access" tail in FTS_UNAVAILABLE_MESSAGE @@ -2036,7 +2048,7 @@ export async function runFullAnalysis( // meta.json / `gitnexus doctor` honest about degraded search. fts: { provider: 'ladybugdb-fts', - status: ftsAvailable ? runtimeCapabilities.fts : 'unavailable', + status: ftsReady ? runtimeCapabilities.fts : 'unavailable', }, vectorSearch: { provider: effectiveSemanticMode === 'vector-index' ? 'ladybugdb-vector' : 'exact-scan', @@ -2216,7 +2228,7 @@ export async function runFullAnalysis( repoPath, stats: meta.stats, pipelineResult, - ftsSkipped: !ftsAvailable, + ftsSkipped: !ftsReady, isPrimaryBranch: !placement.branch, }; } catch (err) { diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index cf0de50f9..dfc4f2eeb 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -178,3 +178,34 @@ export async function verifySearchFTSIndexes( } return missing; } + +export interface BuildSearchIndexesResult { + ok: boolean; + error?: string; +} + +/** + * Build + verify FTS indexes, catching any failure instead of letting it + * propagate. `createSearchFTSIndexes` re-tokenizes every stored row on every + * analyze run (see the `ponytail:` comment above) — a native LadybugDB + * tokenizer error on a single pre-existing row (e.g. a "Failed calling + * LOWER: Invalid UTF-8", #2544/#2546) must not discard an otherwise- + * successful analyze's graph/embeddings work. The caller degrades keyword + * search for this run instead, mirroring the existing FTS-extension- + * unavailable degrade path in `run-analyze.ts`. + */ +export async function buildSearchIndexesOrDegrade( + executeQuery: (cypher: string) => Promise, + options?: CreateSearchFTSIndexesOptions, +): Promise { + try { + await createSearchFTSIndexes(options); + const missing = await verifySearchFTSIndexes(executeQuery); + if (missing.length > 0) { + return { ok: false, error: `missing indexes after build: ${missing.join(', ')}` }; + } + return { ok: true }; + } catch (e) { + return { ok: false, error: e instanceof Error ? e.message : String(e) }; + } +} diff --git a/gitnexus/test/unit/fts-indexes.test.ts b/gitnexus/test/unit/fts-indexes.test.ts index c104d742b..259c745c6 100644 --- a/gitnexus/test/unit/fts-indexes.test.ts +++ b/gitnexus/test/unit/fts-indexes.test.ts @@ -15,9 +15,18 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ ), })); -const { createSearchFTSIndexes, getSearchFTSStemmer, initialiseSearchFTSStemmer } = - await import('../../src/core/search/fts-indexes.js'); +const { + buildSearchIndexesOrDegrade, + createSearchFTSIndexes, + getSearchFTSStemmer, + initialiseSearchFTSStemmer, +} = await import('../../src/core/search/fts-indexes.js'); const { FTS_INDEXES } = await import('../../src/core/search/fts-schema.js'); +const { createFTSIndex } = await import('../../src/core/lbug/lbug-adapter.js'); + +/** SHOW_INDEXES rows covering every configured FTS index's expected properties. */ +const fullCoverageRows = () => + FTS_INDEXES.map((i) => ({ index_name: i.indexName, property_names: [...i.properties] })); afterEach(() => { calls.length = 0; @@ -65,6 +74,37 @@ describe('createSearchFTSIndexes', () => { }); }); +describe('buildSearchIndexesOrDegrade', () => { + it('returns ok:true when every index builds and verifies (#2544/#2546)', async () => { + const executeQuery = vi.fn(async () => fullCoverageRows()); + + const result = await buildSearchIndexesOrDegrade(executeQuery); + + expect(result).toEqual({ ok: true }); + }); + + it('returns ok:false instead of throwing when a single index build rejects (#2544/#2546)', async () => { + vi.mocked(createFTSIndex).mockRejectedValueOnce( + new Error('Runtime exception: Failed calling LOWER: Invalid UTF-8.'), + ); + const executeQuery = vi.fn(async () => fullCoverageRows()); + + const result = await buildSearchIndexesOrDegrade(executeQuery); + + expect(result.ok).toBe(false); + expect(result.error).toContain('Invalid UTF-8'); + }); + + it('returns ok:false when verification finds a missing index, without throwing', async () => { + const executeQuery = vi.fn(async () => fullCoverageRows().slice(1)); + + const result = await buildSearchIndexesOrDegrade(executeQuery); + + expect(result.ok).toBe(false); + expect(result.error).toContain('missing indexes'); + }); +}); + describe('getSearchFTSStemmer', () => { it('defaults to porter when unset', () => { expect(getSearchFTSStemmer()).toBe('porter'); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index b0c0b4c9d..eb524305f 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -459,7 +459,11 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { } }); - it('fails full analyze when FTS verification reports missing indexes after creation', async () => { + it('degrades gracefully (no throw, warns, ftsSkipped) when FTS verification reports missing indexes after creation (#2544/#2546)', async () => { + // A native tokenizer error on one pre-existing row (the #2544/#2546 + // failure mode) must not abort an otherwise-successful full analyze — + // it degrades keyword search for this run instead, same contract as the + // FTS-extension-unavailable sibling test below. vi.doMock('../../src/core/lbug/lbug-adapter.js', () => ({ initLbug: vi.fn(async () => undefined), loadGraphToLbug: vi.fn(async () => undefined), @@ -480,34 +484,42 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), queryImporters: vi.fn(async () => []), queryImportersBatch: vi.fn(async () => []), - // FTS extension loads → analyze proceeds to create + verify indexes. + // FTS extension loads → analyze proceeds to build + verify indexes. loadFTSExtension: vi.fn(async () => true), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ initialiseSearchFTSStemmer: vi.fn(() => 'porter'), - createSearchFTSIndexes: vi.fn(async () => undefined), - verifySearchFTSIndexes: vi.fn(async () => ['Function.function_fts']), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ + ok: false, + error: 'missing indexes after build: Function.function_fts', + })), })); vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ repoPath, - // Full-analyze path only needs `forEachNode` before the FTS verify guard. + // Full-analyze path only needs `forEachNode` before the FTS phase. graph: { forEachNode: () => undefined }, })), })); const tmpRepo = await createTempDir('gitnexus-run-analyze-full-verify-fail-'); try { + const logs: string[] = []; const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); - await expect( - runFullAnalysis( - tmpRepo.dbPath, - { force: true }, - { - onProgress: () => {}, - }, - ), - ).rejects.toThrow(/FTS verification failed - missing indexes after analyze/i); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { onProgress: () => {}, onLog: (msg: string) => logs.push(msg) }, + ); + + expect(result.ftsSkipped).toBe(true); + expect(logs.join('\n')).toMatch( + /FTS index build failed.*missing indexes after build.*keyword search degraded this run/i, + ); + + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + const meta = JSON.parse(await fs.readFile(`${storagePath}/meta.json`, 'utf-8')); + expect(meta.capabilities.fts.status).toBe('unavailable'); } finally { await tmpRepo.cleanup(); } From 1abcac9c1630b66038fa611005cbd37266a7a79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 18 Jul 2026 14:30:37 +0100 Subject: [PATCH 02/15] fix(scope-resolution): stop platform builtins resolving to unrelated same-file symbols (#2549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(scope-resolution): stop platform builtins resolving to unrelated same-file symbols (#2545) An unqualified call to a platform/language builtin (e.g. TypeScript's global fetch()) could resolve to an unrelated same-file declaration sharing that name, most visibly a Cloudflare Worker's `export default { async fetch(req) {...} }` handler. Two contributing gaps, both fixed: - Object literals had no scope boundary in the TS/JS grammar queries, so a method's/property-arrow's name auto-hoisted past the literal into whatever lexically enclosed it (scope-extractor.ts's auto-hoist logic had nowhere to stop). Give object literals a Block scope, like 6 other languages already do for lexical blocks. - Independently, finalize's per-file bindings bucket (materializeBindings in gitnexus-shared) flattens every local declaration in a file onto its module scope for cross-file import resolution, regardless of true nesting -- so free-call-fallback's scope-chain walk could still hit the leaked binding at module scope. Guard free-call resolution: when a match for a known builtin name (LanguageProvider.isBuiltInName, already populated for TS/JS but never consulted by this pass) has no binding reachable via the true lexical scope chain, leave the call unresolved instead of emitting a false CALLS edge. Verified against the full TS/JS resolver suites plus every other language populating builtInNames (Python, Go, C/C++, C#, Dart, Kotlin, PHP, Ruby, Rust, Swift, Vue) -- 2333 tests, no regressions. Co-Authored-By: Claude Sonnet 5 * fix(scope-resolution): extend the #2545 scope-leak fix to Kotlin and Java Anonymous object-expressions (Kotlin `object { ... }`) and anonymous class bodies (Java `new Runnable() { ... }`) have the same missing scope-boundary gap that caused #2545 in TypeScript/JavaScript: a method declared inside has no scope of its own to stop the auto-hoist at, so its name leaks past the container into the enclosing scope. - Kotlin: `(object_literal) @scope.class` (distinct from the already- scoped named `object_declaration`/`companion_object`). Kotlin already populates `builtInNames`, so free-call-fallback's isBuiltInName guard (added for #2545) fully closes the equivalent leak here too -- verified with a `println`-shadowing regression test. - Java: `(object_creation_expression (class_body) @scope.class)`, matching PHP's existing `anonymous_class` handling. Java has no `builtInNames` list, so the isBuiltInName guard doesn't engage -- the scope-tree fix is still correct and necessary (the anonymous class's own methods are now owned by the right scope), but an unqualified call to an unrelated same-file method sharing the anonymous class's method name can still resolve via finalize's per-file module-scope bucket (materializeBindings, shared/ language-agnostic, intentionally not touched by this PR). Documented in the test as a known residual gap, same as TS/JS/Kotlin's own non-builtin-name collisions. Audited every other language for the same shape (a value/container node with no @scope.* capture hosting a would-be-auto-hoisted named declaration): PHP and Vue already handle it correctly (PHP scopes anonymous_class; Vue's