From 62b519fc5c1e688f8d38cad20f07d31863a18bf8 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Mon, 7 Sep 2026 18:15:06 +0100 Subject: [PATCH 1/5] feat(indexing): index XAML declarations for search --- ARCHITECTURE.md | 12 +- README.md | 2 + gitnexus/scripts/cross-platform-tests.ts | 1 + .../src/core/ingestion/documents/registry.ts | 6 + .../src/core/ingestion/documents/types.ts | 10 + gitnexus/src/core/ingestion/documents/xaml.ts | 88 ++++ .../ingestion/pipeline-phases/documents.ts | 57 +++ .../core/ingestion/pipeline-phases/index.ts | 1 + .../core/ingestion/pipeline-phases/parse.ts | 4 +- gitnexus/src/core/ingestion/pipeline.ts | 4 +- gitnexus/src/core/run-analyze.ts | 17 +- gitnexus/src/core/search/fts-schema.ts | 2 + gitnexus/src/mcp/local/local-backend.ts | 4 +- .../test/integration/xaml-documents.test.ts | 68 +++ gitnexus/test/integration/xaml-search.test.ts | 443 ++++++++++++++++++ gitnexus/test/unit/fts-schema.test.ts | 8 + .../ingestion/pipeline-phase-registry.test.ts | 1 + gitnexus/test/unit/xaml-declarations.test.ts | 84 ++++ gitnexus/vitest.config.ts | 2 + 19 files changed, 803 insertions(+), 11 deletions(-) create mode 100644 gitnexus/src/core/ingestion/documents/registry.ts create mode 100644 gitnexus/src/core/ingestion/documents/types.ts create mode 100644 gitnexus/src/core/ingestion/documents/xaml.ts create mode 100644 gitnexus/src/core/ingestion/pipeline-phases/documents.ts create mode 100644 gitnexus/test/integration/xaml-documents.test.ts create mode 100644 gitnexus/test/integration/xaml-search.test.ts create mode 100644 gitnexus/test/unit/xaml-declarations.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b1076d1d5..10c511697 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -15,7 +15,7 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## End-to-end flow: index → graph → tools -1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). The default DAG of 19 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. +1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). The default DAG of 21 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. 2. **Persistence** — `repo-manager.ts` (paths, registry, LadybugDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches). @@ -82,11 +82,11 @@ Group-mode `trace` (`gitnexus/src/core/group/cross-trace.ts`) stitches a path th ## Pipeline Phase DAG -19 default phases are defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. `--pdg` adds `taintSummaries` and `callSummaries` (21 total). +21 default phases are defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. `--pdg` adds `taintSummaries` and `callSummaries` (23 total). ``` -scan → structure → [springConfig, markdown, cobol] → parse → [routes, tools, orm] - → crossFile → scopeResolution → [springAutoConfiguration, springAop] +scan → structure → [springConfig, markdown, documents, cobol] → parse → [routes, tools, orm] + → crossFile → scopeResolution → [springAutoConfiguration, springAop, springDestinations] → pruneLocalSymbols → mro → springAopInheritance → di → communities → processes ``` @@ -97,7 +97,8 @@ scan → structure → [springConfig, markdown, cobol] → parse → [routes, to | `springConfig` | `spring-config.ts` | `structure` | Spring configuration-property nodes and metadata | | `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx | | `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) | -| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | +| `documents` | `documents.ts` | `structure` | Section nodes and containment edges for static XAML declarations; no runtime call or inheritance edges | +| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `documents`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | | `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators, and JS/TS static route sources — see below) | | `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | | `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | @@ -105,6 +106,7 @@ scan → structure → [springConfig, markdown, cobol] → parse → [routes, to | `scopeResolution` | `scope-resolution/pipeline/phase.ts` | `parse`, `crossFile`, `structure` | Binding/reference + inheritance edges; disposes BindingAccumulator | | `springAutoConfiguration` | `spring-auto-configuration.ts` | `structure`, `scopeResolution` | DECLARES and CONDITIONAL_ON metadata for Spring configuration candidates | | `springAop` | `spring-aop.ts` | `scopeResolution` | Direct declarative/advice ADVISED_BY edges and pointcut evidence | +| `springDestinations` | `spring-destinations.ts` | `parse`, `scopeResolution`, `springConfig` | Destination nodes and publish/consume edges for Spring messaging | | `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution | | `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | | `springAopInheritance` | `spring-aop.ts` | `springAop`, `mro` | Propagates declarative behavior through class/interface inheritance decisions | diff --git a/README.md b/README.md index f2714bf89..38bce3ec5 100644 --- a/README.md +++ b/README.md @@ -642,6 +642,8 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas ### Supported Languages +XAML files are indexed as documents. Literal `x:Name`, `x:Key`, and `x:Class` declarations become searchable `Section` nodes with source ranges and file containment. Namespace aliases are supported. Runtime bindings, event-handler resolution, markup-extension keys, entity-containing declaration values, DTDs, and documents deeper than 100 elements are not interpreted; the file remains indexed when declaration extraction is unavailable. Run `gitnexus analyze --force` to add declaration nodes and their search indexes to an existing index. + | Language | Imports | Named Bindings | Exports | Heritage | Type Annotations | Constructor Inference | Config | Frameworks | Entry Points | | ---------- | ------- | -------------- | ------- | -------- | ---------------- | --------------------- | ------ | ---------- | ------------ | | TypeScript | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 79169214b..44b60626b 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -130,6 +130,7 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/xaml-search.test.ts', 'test/integration/lbug-core-adapter.test.ts', 'test/integration/lbug-vector-extension.test.ts', 'test/integration/lbug-pool.test.ts', diff --git a/gitnexus/src/core/ingestion/documents/registry.ts b/gitnexus/src/core/ingestion/documents/registry.ts new file mode 100644 index 000000000..7c6d68e30 --- /dev/null +++ b/gitnexus/src/core/ingestion/documents/registry.ts @@ -0,0 +1,6 @@ +import type { DocumentParser } from './types.js'; +import { extractXamlDeclarations } from './xaml.js'; + +export const DOCUMENT_PARSERS: ReadonlyMap = new Map([ + ['.xaml', extractXamlDeclarations], +]); diff --git a/gitnexus/src/core/ingestion/documents/types.ts b/gitnexus/src/core/ingestion/documents/types.ts new file mode 100644 index 000000000..b2818be26 --- /dev/null +++ b/gitnexus/src/core/ingestion/documents/types.ts @@ -0,0 +1,10 @@ +export interface DocumentDeclaration { + name: string; + description: string; + startIndex: number; + startLine: number; + endLine: number; + level: number; +} + +export type DocumentParser = (source: string) => readonly DocumentDeclaration[]; diff --git a/gitnexus/src/core/ingestion/documents/xaml.ts b/gitnexus/src/core/ingestion/documents/xaml.ts new file mode 100644 index 000000000..828a1b9b3 --- /dev/null +++ b/gitnexus/src/core/ingestion/documents/xaml.ts @@ -0,0 +1,88 @@ +import { XMLParser, XMLValidator, type XMLMetaData } from 'fast-xml-parser'; +import type { DocumentDeclaration } from './types.js'; + +const XAML_NAMESPACE = 'http://schemas.microsoft.com/winfx/2006/xaml'; +const DIRECTIVES = new Set(['Name', 'Key', 'Class']); +const META = XMLParser.getMetaDataSymbol() as symbol; +type XmlNode = Record & { [key: symbol]: XMLMetaData | undefined }; + +export function extractXamlDeclarations(source: string): DocumentDeclaration[] { + // fast-xml-parser normalizes newlines before computing its metadata offsets. + source = source.replace(/\r\n?/g, '\n'); + // Repository documents are untrusted. Never resolve DTDs or expand entities. + if (/ { + let lo = 0; + let hi = lineStarts.length; + while (lo + 1 < hi) { + const mid = (lo + hi) >>> 1; + if (lineStarts[mid] <= offset) lo = mid; + else hi = mid; + } + return lo; + }; + const declarations: DocumentDeclaration[] = []; + const stack = roots + .map((node) => ({ node, namespaces: new Map(), level: 1 })) + .reverse(); + while (stack.length) { + const { node, namespaces: inherited, level } = stack.pop()!; + const tag = Object.keys(node).find((key) => key !== ':@' && !key.startsWith('#')); + if (!tag || !Array.isArray(node[tag])) continue; + if (level > 100) throw new Error('Maximum document depth exceeded'); + const attributes = Object.entries((node[':@'] ?? {}) as Record); + const namespaceAttributes = attributes.filter(([key]) => key.startsWith('@_xmlns:')); + const namespaces = namespaceAttributes.length ? new Map(inherited) : inherited; + for (const [key, value] of namespaceAttributes) namespaces.set(key.slice(8), value); + const span = node[META]; + for (const [attribute, value] of attributes) { + const parts = attribute.slice(2).split(':'); + if ( + parts.length !== 2 || + namespaces.get(parts[0]) !== XAML_NAMESPACE || + !DIRECTIVES.has(parts[1]) + ) + continue; + if (typeof value !== 'string' || !value.trim() || /[{}&]/.test(value)) continue; + if (span?.startIndex === undefined || span.endIndex === undefined) continue; + declarations.push({ + name: value, + description: `${tag} x:${parts[1]} declaration`, + startIndex: span.startIndex, + startLine: lineAt(span.startIndex), + endLine: lineAt(Math.max(span.startIndex, span.endIndex - 1)), + level, + }); + } + const children = node[tag] as XmlNode[]; + for (let i = children.length - 1; i >= 0; i--) { + stack.push({ node: children[i], namespaces, level: level + 1 }); + } + } + return declarations; +} diff --git a/gitnexus/src/core/ingestion/pipeline-phases/documents.ts b/gitnexus/src/core/ingestion/pipeline-phases/documents.ts new file mode 100644 index 000000000..f0783b966 --- /dev/null +++ b/gitnexus/src/core/ingestion/pipeline-phases/documents.ts @@ -0,0 +1,57 @@ +import path from 'node:path'; +import { generateId } from '../../../lib/utils.js'; +import { logger } from '../../logger.js'; +import { DOCUMENT_PARSERS } from '../documents/registry.js'; +import { readFileContents } from '../filesystem-walker.js'; +import { getPhaseOutput, type PipelinePhase } from './types.js'; +import type { StructureOutput } from './structure.js'; + +export const documentsPhase: PipelinePhase<{ sections: number; failures: number }> = { + name: 'documents', + deps: ['structure'], + async execute(ctx, deps) { + const { scannedFiles } = getPhaseOutput(deps, 'structure'); + const files = scannedFiles.filter((file) => + DOCUMENT_PARSERS.has(path.extname(file.path).toLowerCase()), + ); + let sections = 0; + let failures = 0; + for (const file of files) { + const fileId = generateId('File', file.path); + if (!ctx.graph.getNode(fileId)) continue; + const content = (await readFileContents(ctx.repoPath, [file.path])).get(file.path); + if (content === undefined) continue; + const parser = DOCUMENT_PARSERS.get(path.extname(file.path).toLowerCase())!; + let declarations; + try { + declarations = parser(content); + } catch { + failures++; + logger.warn( + { filePath: file.path }, + 'Document declarations omitted: invalid or unsupported document', + ); + continue; + } + for (const declaration of declarations) { + const { startIndex, ...properties } = declaration; + const id = generateId('Section', `${file.path}:${startIndex}:${declaration.description}`); + ctx.graph.addNode({ + id, + label: 'Section', + properties: { ...properties, filePath: file.path }, + }); + ctx.graph.addRelationship({ + id: generateId('CONTAINS', `${fileId}->${id}`), + type: 'CONTAINS', + sourceId: fileId, + targetId: id, + confidence: 1, + reason: 'document-declaration', + }); + sections++; + } + } + return { sections, failures }; + }, +}; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/index.ts b/gitnexus/src/core/ingestion/pipeline-phases/index.ts index 99b7cc253..0b471bc73 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/index.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/index.ts @@ -10,6 +10,7 @@ export { scanPhase, type ScanOutput } from './scan.js'; export { structurePhase, type StructureOutput } from './structure.js'; export { markdownPhase, type MarkdownOutput } from './markdown.js'; +export { documentsPhase } from './documents.js'; export { cobolPhase, type CobolOutput } from './cobol.js'; export { parsePhase, type ParseOutput } from './parse.js'; export { routesPhase, type RoutesOutput, type RouteEntry } from './routes.js'; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index b47ceafdf..e974aab94 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -9,7 +9,7 @@ * from the original pipeline. The chunk loop is a memory optimization * internal to this phase, not a phase boundary. * - * @deps structure, markdown, cobol + * @deps structure, markdown, documents, cobol * @reads scannedFiles, allPaths, totalFiles (from structure) * @writes graph (Symbol nodes, IMPORTS/CALLS/EXTENDS/IMPLEMENTS/ACCESSES edges) * @output exportedTypeMap, allFetchCalls, allExtractedRoutes, allDecoratorRoutes, @@ -107,7 +107,7 @@ export interface ParseOutput { export const parsePhase: PipelinePhase = { name: 'parse', - deps: ['structure', 'markdown', 'cobol'], + deps: ['structure', 'markdown', 'documents', 'cobol'], async execute( ctx: PipelineContext, diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 8ee94e265..9ebd1f90b 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -25,6 +25,7 @@ import { scanPhase, structurePhase, markdownPhase, + documentsPhase, cobolPhase, parsePhase, routesPhase, @@ -301,7 +302,7 @@ export interface PipelineOptions { * * Phase dependency graph: * - * scan → structure → [springConfig, markdown, cobol] → parse → [routes, tools, orm] + * scan → structure → [springConfig, markdown, documents, cobol] → parse → [routes, tools, orm] * → crossFile → scopeResolution → [springAutoConfiguration, springAop, * springDestinations] → pruneLocalSymbols * → mro → springAopInheritance → di → communities → processes @@ -323,6 +324,7 @@ export function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { .register(structurePhase) .register(springConfigPhase) .register(markdownPhase) + .register(documentsPhase) .register(cobolPhase) .register(parsePhase) .register(routesPhase) diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 8b70ef10e..24753b305 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1791,6 +1791,19 @@ async function runFullAnalysisInner( // fast path because the previous analyze just wrote them // (regression vs PR #1233 behavior). const dirty = isWorkingTreeDirty(repoPath); + // Git can report clean after reverting/removing an indexed dirty file. + // Hidden Git flags also persist in dirtyPaths, so compare content rather + // than repeatedly reanalyzing unchanged candidates. + const indexedDirtyPaths = existingMeta.indexCoverage?.dirtyPaths ?? []; + let indexedContentChanged = false; + if (!dirty && indexedDirtyPaths.length > 0) { + const currentHashes = await computeFileHashes(repoPath, indexedDirtyPaths); + indexedContentChanged = indexedDirtyPaths.some( + (filePath) => + !currentHashes.has(filePath) || + currentHashes.get(filePath) !== existingMeta.fileHashes?.[filePath], + ); + } // Registration wrinkle around the fast path (#2264). A prior // `analyze --name X` that hit a name collision writes meta.json (meta-save // runs before registerRepo) then fails before registering, leaving the @@ -1820,7 +1833,7 @@ async function runFullAnalysisInner( // re-analysis whenever an index authored where FTS was unavailable was // later read on a host where it loads — which is a legitimate, common // state, and the invariant `analyzer-identity-cli.test.ts` pins. - if (!dirty && !healUnregistered) { + if (!dirty && !indexedContentChanged && !healUnregistered) { if (options.registryName) { await registerRepo(repoPath, existingMeta, { name: options.registryName, @@ -2984,6 +2997,8 @@ async function runFullAnalysisInner( ? await snapshotDerivedRelsForFiles(filesToDelete, [...tablesWithRows]) : []; await dropSearchFTSIndexes(indexCatalogRows, incrementalFtsRebuildTables); + // Release dropped search-index storage before the next bulk COPY. + await checkpointOnce(); // 1b. Remove the write set's existing rows — batched (#2409): one // DETACH DELETE per table per 200-file chunk. The former per-file // loop issued a count + delete per table per FILE — ~13k diff --git a/gitnexus/src/core/search/fts-schema.ts b/gitnexus/src/core/search/fts-schema.ts index eca162448..6e64ecd3f 100644 --- a/gitnexus/src/core/search/fts-schema.ts +++ b/gitnexus/src/core/search/fts-schema.ts @@ -44,4 +44,6 @@ export const FTS_INDEXES: readonly FTSIndexDefinition[] = [ { table: 'Union', indexName: 'union_fts', properties: FTS_PROPERTIES }, { table: 'Static', indexName: 'static_fts', properties: FTS_PROPERTIES }, { table: 'Variable', indexName: 'variable_fts', properties: FTS_PROPERTIES }, + // Markdown headings and static document declarations must be discoverable by name. + { table: 'Section', indexName: 'section_fts', properties: FTS_PROPERTIES }, ]; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 52eaccea1..84669fc00 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -3242,7 +3242,7 @@ export class LocalBackend { ` MATCH (n) WHERE n.id IN $nodeIds - RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine + RETURN n.id AS id, n.name AS name, labels(n) AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine ORDER BY startLine, id `, { nodeIds }, @@ -3257,7 +3257,7 @@ export class LocalBackend { MATCH (n) WHERE n.filePath = $filePath AND NOT n.id STARTS WITH 'BasicBlock:' - RETURN n.id AS id, n.name AS name, labels(n)[0] AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine + RETURN n.id AS id, n.name AS name, labels(n) AS type, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine ORDER BY startLine, id LIMIT 3 `, diff --git a/gitnexus/test/integration/xaml-documents.test.ts b/gitnexus/test/integration/xaml-documents.test.ts new file mode 100644 index 000000000..904efc502 --- /dev/null +++ b/gitnexus/test/integration/xaml-documents.test.ts @@ -0,0 +1,68 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createTempDir } from '../helpers/test-db.js'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; +import { generateId } from '../../src/lib/utils.js'; + +const NS = 'http://schemas.microsoft.com/winfx/2006/xaml'; +const fixtures: Awaited>[] = []; +afterEach(async () => { + for (const fixture of fixtures.splice(0)) await fixture.cleanup(); +}); + +describe('XAML pipeline documents (#3202)', () => { + it('indexes literal names and resource keys, with no invented runtime edges', async () => { + const repo = await createTempDir(); + fixtures.push(repo); + await fs.writeFile( + path.join(repo.dbPath, 'Home.XAML'), + `\n