mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-09 03:17:54 +00:00
refactor(ingestion): remove dead worker-side ParsedFile extraction
After #2038 gated worker `ParsedFile` emission behind `!isScopeResolutionLanguage(language)`, and with all 16 SupportedLanguages registered in SCOPE_RESOLVERS, that gate was structurally always true — the worker already produced no ParsedFiles and scope-resolution re-extracts each file from source on the main thread (run.ts). Remove the now-dead machinery: - Drop both worker `extractParsedFile` call-sites (tree-sitter processFileGroup + the standalone-provider branch) and the `result.parsedFiles.push`. The standalone branch keeps fileCount/onFileProcessed per file. `result.parsedFiles` stays declared but empty (field removal deferred). - Remove the now-orphaned `scopeSourceKind` var + `ScopeCaptureSourceKind`/`extractParsedFile`/`isScopeResolutionLanguage` imports. - Delete the consumerless `migrated-languages.ts` (isScopeResolutionLanguage + SCOPE_RESOLUTION_LANGUAGES) and its drift-guard test — parse-worker was their only importer. Also improves AGENTS.md "shared ingestion code must not name languages" compliance. `extractParsedFile` and the scope-extractor-bridge stay (scope-resolution/run.ts + Vue resolver use them). Behavior-preserving: worker-sequential-parity passes before and after; tsc/eslint clean; no baseline/golden drift. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
9b6650822f
commit
30abb2f969
3 changed files with 12 additions and 117 deletions
|
|
@ -1,32 +0,0 @@
|
|||
/**
|
||||
* Languages that resolve via the scope-resolution pipeline (RFC #909 Ring 3).
|
||||
*
|
||||
* Kept free of `ScopeResolver` imports so worker threads can gate
|
||||
* `ParsedFile` emission without pulling in resolver implementations.
|
||||
* Keep in sync with `SCOPE_RESOLVERS` in `registry.ts`.
|
||||
*/
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
|
||||
export const SCOPE_RESOLUTION_LANGUAGES: ReadonlySet<SupportedLanguages> = new Set([
|
||||
SupportedLanguages.Python,
|
||||
SupportedLanguages.CSharp,
|
||||
SupportedLanguages.TypeScript,
|
||||
SupportedLanguages.Go,
|
||||
SupportedLanguages.Java,
|
||||
SupportedLanguages.C,
|
||||
SupportedLanguages.CPlusPlus,
|
||||
SupportedLanguages.PHP,
|
||||
SupportedLanguages.Rust,
|
||||
SupportedLanguages.JavaScript,
|
||||
SupportedLanguages.Kotlin,
|
||||
SupportedLanguages.Ruby,
|
||||
SupportedLanguages.Cobol,
|
||||
SupportedLanguages.Swift,
|
||||
SupportedLanguages.Dart,
|
||||
SupportedLanguages.Vue,
|
||||
]);
|
||||
|
||||
export const isScopeResolutionLanguage = (
|
||||
lang: SupportedLanguages | null,
|
||||
): lang is SupportedLanguages => lang !== null && SCOPE_RESOLUTION_LANGUAGES.has(lang);
|
||||
|
|
@ -96,8 +96,6 @@ import {
|
|||
import { extractTemplateArguments, templateArgumentsIdTag } from '../utils/template-arguments.js';
|
||||
import type { LanguageProvider } from '../language-provider.js';
|
||||
import type { ParsedFile } from 'gitnexus-shared';
|
||||
import { extractParsedFile, type ScopeCaptureSourceKind } from '../scope-extractor-bridge.js';
|
||||
import { isScopeResolutionLanguage } from '../scope-resolution/pipeline/migrated-languages.js';
|
||||
import { extractLaravelRoutes, type ExtractedRoute } from '../route-extractors/laravel.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
|
|
@ -835,38 +833,14 @@ const processBatch = (
|
|||
const queryString = provider.treeSitterQueries;
|
||||
if (!queryString) {
|
||||
// Standalone providers (regex-based, no tree-sitter) that implement
|
||||
// emitScopeCaptures feed into the scope-resolution pipeline via
|
||||
// extractParsedFile directly — no tree-sitter involved.
|
||||
// emitScopeCaptures resolve via the scope-resolution pipeline, which
|
||||
// re-extracts from source on the main thread.
|
||||
if (provider.emitScopeCaptures) {
|
||||
for (const file of langFiles) {
|
||||
// #1983: skip building the ParsedFile for registry-primary
|
||||
// (scope-resolution) languages — the scope-resolution phase
|
||||
// re-extracts from source on the main thread, so the worker copy is
|
||||
// unused work + retained RAM. COBOL is a standalone provider that is
|
||||
// in SCOPE_RESOLUTION_LANGUAGES; its graph nodes come from cobolPhase,
|
||||
// not from this ParsedFile, so gating here is safe.
|
||||
const parsedFile = isScopeResolutionLanguage(language)
|
||||
? undefined
|
||||
: extractParsedFile(
|
||||
provider,
|
||||
file.content,
|
||||
file.path,
|
||||
(message) => {
|
||||
if (parentPort) {
|
||||
parentPort.postMessage({ type: 'warning', message });
|
||||
} else {
|
||||
logger.warn(message);
|
||||
}
|
||||
},
|
||||
undefined, // no cachedTree for standalone providers
|
||||
);
|
||||
if (parsedFile !== undefined) {
|
||||
result.parsedFiles.push(parsedFile);
|
||||
}
|
||||
// fileCount / progress fire per file regardless of whether a
|
||||
// ParsedFile was produced — matching the tree-sitter branch (which
|
||||
// increments fileCount outside the parsedFile gate). Otherwise gated
|
||||
// COBOL files would vanish from worker progress counts.
|
||||
// The worker no longer builds `ParsedFile`s for standalone providers
|
||||
// either — scope-resolution re-extracts on the main thread, and for
|
||||
// standalone COBOL the graph nodes come from cobolPhase, not this
|
||||
// artifact (#1983). Count one unit of progress per file, as before.
|
||||
for (let i = 0; i < langFiles.length; i++) {
|
||||
result.fileCount++;
|
||||
onFileProcessed?.();
|
||||
}
|
||||
|
|
@ -1107,14 +1081,12 @@ const processFileGroup = (
|
|||
|
||||
// Vue SFC preprocessing: extract <script> block content
|
||||
let parseContent = file.content;
|
||||
let scopeSourceKind: ScopeCaptureSourceKind = 'full-file';
|
||||
let lineOffset = 0;
|
||||
let isVueSetup = false;
|
||||
if (language === SupportedLanguages.Vue) {
|
||||
const extracted = extractVueScript(file.content);
|
||||
if (!extracted) continue; // skip .vue files with no script block
|
||||
parseContent = extracted.scriptContent;
|
||||
scopeSourceKind = 'pre-extracted-script';
|
||||
lineOffset = extracted.lineOffset;
|
||||
isVueSetup = extracted.isSetup;
|
||||
}
|
||||
|
|
@ -1154,28 +1126,11 @@ const processFileGroup = (
|
|||
|
||||
const provider = getProvider(language);
|
||||
|
||||
// RFC #909 Ring 2: produce a `ParsedFile` for the new scope-based
|
||||
// resolution pipeline. Skipped for registry-primary languages — the
|
||||
// scope-resolution phase re-extracts from source on the main thread,
|
||||
// which avoids retaining ~2× semantic model in RAM on huge repos (#1983).
|
||||
let parsedFile: import('gitnexus-shared').ParsedFile | undefined;
|
||||
if (!isScopeResolutionLanguage(language)) {
|
||||
parsedFile = extractParsedFile(
|
||||
provider,
|
||||
parseContent,
|
||||
file.path,
|
||||
(message) => {
|
||||
if (parentPort) {
|
||||
parentPort.postMessage({ type: 'warning', message });
|
||||
} else {
|
||||
logger.warn(message);
|
||||
}
|
||||
},
|
||||
tree,
|
||||
scopeSourceKind,
|
||||
);
|
||||
}
|
||||
if (parsedFile !== undefined) result.parsedFiles.push(parsedFile);
|
||||
// The worker no longer builds a `ParsedFile` here. Scope-resolution
|
||||
// re-extracts each file from source on the main thread (scope-resolution/
|
||||
// pipeline/run.ts); the worker copy retained ~2× the semantic model in RAM
|
||||
// on huge repos (#1983) and, with every language on the scope-resolution
|
||||
// path, was consumed by no one. `result.parsedFiles` stays empty.
|
||||
|
||||
// Build per-file type environment + constructor bindings in a single AST walk.
|
||||
// The legacy heritage pre-pass that seeded a file-local parentMap for
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js';
|
||||
import {
|
||||
SCOPE_RESOLUTION_LANGUAGES,
|
||||
isScopeResolutionLanguage,
|
||||
} from '../../../src/core/ingestion/scope-resolution/pipeline/migrated-languages.js';
|
||||
|
||||
describe('SCOPE_RESOLUTION_LANGUAGES drift guard', () => {
|
||||
// The parse worker gates ParsedFile emission on `isScopeResolutionLanguage`,
|
||||
// which is derived from SCOPE_RESOLUTION_LANGUAGES — a hand-maintained
|
||||
// duplicate of the SCOPE_RESOLVERS key set (kept resolver-import-free so the
|
||||
// worker bundle stays light). The dangerous drift is asymmetric: a language
|
||||
// in this Set but missing from SCOPE_RESOLVERS would have its ParsedFile
|
||||
// skipped in the worker AND never re-extracted by scope resolution →
|
||||
// permanent loss. This test fails if the two ever diverge (#1983).
|
||||
it('covers exactly the languages registered in SCOPE_RESOLVERS', () => {
|
||||
const resolverLangs = [...SCOPE_RESOLVERS.keys()].sort();
|
||||
const skipSetLangs = [...SCOPE_RESOLUTION_LANGUAGES].sort();
|
||||
expect(skipSetLangs).toEqual(resolverLangs);
|
||||
});
|
||||
|
||||
it('isScopeResolutionLanguage returns true for every registered resolver language and false for null', () => {
|
||||
for (const lang of SCOPE_RESOLVERS.keys()) {
|
||||
expect(isScopeResolutionLanguage(lang)).toBe(true);
|
||||
}
|
||||
expect(isScopeResolutionLanguage(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue