mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-09 03:17:54 +00:00
Merge branch 'main' into docs/kilo-code-mcp
This commit is contained in:
commit
a3f8cd8c87
19 changed files with 1081 additions and 16 deletions
|
|
@ -303,6 +303,7 @@ Each language implements `LanguageProvider` (`language-provider.ts`). Key fields
|
|||
| `exportChecker` | Public/exported symbol detection |
|
||||
| `typeConfig` | Type annotation extraction rules |
|
||||
| `mroStrategy` | `first-wins` / `c3` / `none` |
|
||||
| `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) |
|
||||
|
||||
16 providers in `languages/index.ts` via `satisfies Record<SupportedLanguages, LanguageProvider>` — missing a language is a compile error.
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ const FUNCTION_DECLARATION_TYPES = new Set([
|
|||
'function_item',
|
||||
]);
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import type { NodeLabel } from 'gitnexus-shared';
|
||||
import type { LanguageProvider } from '../language-provider.js';
|
||||
import { createFieldExtractor } from '../field-extractors/generic.js';
|
||||
|
|
@ -397,6 +398,8 @@ export const cProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(cVariableConfig),
|
||||
classExtractor: cClassExtractor,
|
||||
// ── Doxygen doc comment → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
labelOverride: cppLabelOverride,
|
||||
builtInNames: C_BUILT_INS,
|
||||
|
||||
|
|
@ -482,6 +485,8 @@ export const cppProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(cppVariableConfig),
|
||||
classExtractor: cppClassExtractor,
|
||||
// ── Doxygen doc comment → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
labelOverride: cppLabelOverride,
|
||||
builtInNames: C_BUILT_INS,
|
||||
extractTemplateConstraints: extractCppTemplateConstraintsForProvider,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { csharpExportChecker } from '../export-detection.js';
|
|||
import { createImportResolver } from '../import-resolvers/resolver-factory.js';
|
||||
import { csharpImportConfig } from '../import-resolvers/configs/csharp.js';
|
||||
import { CSHARP_QUERIES } from '../tree-sitter-queries.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import type { AstFrameworkPatternConfig } from '../language-provider.js';
|
||||
import { createCallExtractor } from '../call-extractors/generic.js';
|
||||
import { csharpCallConfig } from '../call-extractors/configs/csharp.js';
|
||||
|
|
@ -194,6 +195,8 @@ export const csharpProvider = defineLanguage({
|
|||
methodExtractor: createMethodExtractor(csharpMethodConfig),
|
||||
variableExtractor: createVariableExtractor(csharpVariableConfig),
|
||||
classExtractor: createClassExtractor(csharpClassConfig),
|
||||
// ── XML doc comments (`///`) → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
builtInNames: BUILT_INS,
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ──────────
|
||||
|
|
|
|||
|
|
@ -9,9 +9,12 @@
|
|||
* The hook resolves the enclosing function by inspecting the previous sibling.
|
||||
*/
|
||||
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import {
|
||||
createLeadingDocDescriptionExtractor,
|
||||
FUNCTION_NODE_TYPES,
|
||||
type SyntaxNode,
|
||||
} from '../utils/ast-helpers.js';
|
||||
import type { NodeLabel } from 'gitnexus-shared';
|
||||
import { FUNCTION_NODE_TYPES } from '../utils/ast-helpers.js';
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import { createClassExtractor } from '../class-extractors/generic.js';
|
||||
import { dartClassConfig } from '../class-extractors/configs/dart.js';
|
||||
|
|
@ -124,6 +127,8 @@ export const dartProvider = defineLanguage({
|
|||
methodExtractor: createMethodExtractor(dartMethodConfig),
|
||||
variableExtractor: createVariableExtractor(dartVariableConfig),
|
||||
classExtractor: createClassExtractor(dartClassConfig),
|
||||
// ── Dartdoc (`///`) → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
enclosingFunctionFinder: dartEnclosingFunctionFinder,
|
||||
builtInNames: DART_BUILT_INS,
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import { createClassExtractor } from '../class-extractors/generic.js';
|
||||
import { goClassConfig } from '../class-extractors/configs/go.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import { createGoCfgVisitor } from '../cfg/visitors/go.js';
|
||||
import { defineLanguage } from '../language-provider.js';
|
||||
import { typeConfig as goConfig } from '../type-extractors/go.js';
|
||||
|
|
@ -138,6 +139,12 @@ export const goProvider = defineLanguage({
|
|||
methodExtractor: createMethodExtractor(goMethodConfig),
|
||||
variableExtractor: createVariableExtractor(goVariableConfig),
|
||||
classExtractor: createClassExtractor(goClassConfig),
|
||||
// ── godoc (`//` leading comments) → description (issue #2270). Build/tool
|
||||
// directives (//go:…, // +build, //nolint, //line) are not documentation. ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor({
|
||||
lineCommentPrefixes: ['//'],
|
||||
lineDirectivePrefixes: ['//go:', '// +build', '//nolint', '//line'],
|
||||
}),
|
||||
builtInNames: GO_BUILT_INS,
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks ──────────
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { createClassExtractor } from '../class-extractors/generic.js';
|
|||
import { javaClassConfig } from '../class-extractors/configs/jvm.js';
|
||||
import { defineLanguage } from '../language-provider.js';
|
||||
import type { AstFrameworkPatternConfig } from '../language-provider.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import { javaTypeConfig } from '../type-extractors/jvm.js';
|
||||
import { extractSpringRoutes } from '../route-extractors/spring.js';
|
||||
import { javaExportChecker } from '../export-detection.js';
|
||||
|
|
@ -117,6 +118,9 @@ export const javaProvider = defineLanguage({
|
|||
variableExtractor: createVariableExtractor(javaVariableConfig),
|
||||
classExtractor: createClassExtractor(javaClassConfig),
|
||||
|
||||
// ── Javadoc → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks ──
|
||||
emitScopeCaptures: emitJavaScopeCaptures,
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { SupportedLanguages } from 'gitnexus-shared';
|
|||
import { createClassExtractor } from '../class-extractors/generic.js';
|
||||
import { kotlinClassConfig } from '../class-extractors/configs/jvm.js';
|
||||
import { defineLanguage } from '../language-provider.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import { assertCloneable } from '../workers/clone-safety.js';
|
||||
import { kotlinTypeConfig } from '../type-extractors/jvm.js';
|
||||
import { kotlinExportChecker } from '../export-detection.js';
|
||||
|
|
@ -170,6 +171,10 @@ export const kotlinProvider = defineLanguage({
|
|||
variableExtractor: createVariableExtractor(kotlinVariableConfig),
|
||||
classExtractor: createClassExtractor(kotlinClassConfig),
|
||||
builtInNames: BUILT_INS,
|
||||
|
||||
// ── KDoc → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
|
||||
labelOverride: (functionNode, defaultLabel) => {
|
||||
if (defaultLabel !== 'Function') return defaultLabel;
|
||||
if (isKotlinClassMethod(functionNode)) return 'Method';
|
||||
|
|
|
|||
|
|
@ -21,13 +21,22 @@ import { SupportedLanguages } from 'gitnexus-shared';
|
|||
import { createClassExtractor } from '../class-extractors/generic.js';
|
||||
import { phpClassConfig } from '../class-extractors/configs/php.js';
|
||||
import { createPhpCfgVisitor } from '../cfg/visitors/php.js';
|
||||
import { defineLanguage, type AstFrameworkPatternConfig } from '../language-provider.js';
|
||||
import {
|
||||
defineLanguage,
|
||||
type AstFrameworkPatternConfig,
|
||||
type CaptureMap,
|
||||
} from '../language-provider.js';
|
||||
import { typeConfig as phpConfig } from '../type-extractors/php.js';
|
||||
import { phpExportChecker } from '../export-detection.js';
|
||||
import { createImportResolver } from '../import-resolvers/resolver-factory.js';
|
||||
import { phpImportConfig } from '../import-resolvers/configs/php.js';
|
||||
import { PHP_QUERIES } from '../tree-sitter-queries.js';
|
||||
import { findDescendant, extractStringContent, type SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import {
|
||||
findDescendant,
|
||||
extractStringContent,
|
||||
createLeadingDocDescriptionExtractor,
|
||||
type SyntaxNode,
|
||||
} from '../utils/ast-helpers.js';
|
||||
import type { NodeLabel } from 'gitnexus-shared';
|
||||
import { createFieldExtractor } from '../field-extractors/generic.js';
|
||||
import { phpConfig as phpFieldConfig } from '../field-extractors/configs/php.js';
|
||||
|
|
@ -221,22 +230,32 @@ function extractEloquentRelationDescription(methodNode: SyntaxNode): string | nu
|
|||
return null;
|
||||
}
|
||||
|
||||
/** PHPDoc-docblock fallback, shared with the other leading-comment languages. */
|
||||
const phpLeadingDocFallback = createLeadingDocDescriptionExtractor();
|
||||
|
||||
/**
|
||||
* LanguageProvider.descriptionExtractor implementation for PHP.
|
||||
* Extracts Eloquent model property metadata and relationship descriptions.
|
||||
* Eloquent model property metadata and relationship descriptions take
|
||||
* precedence (they are richer than prose); otherwise documentable symbols fall
|
||||
* back to their leading PHPDoc docblock (issue #2270), mirroring the other
|
||||
* leading-comment languages.
|
||||
*/
|
||||
function phpDescriptionExtractor(
|
||||
nodeLabel: NodeLabel,
|
||||
nodeName: string,
|
||||
captureMap: Record<string, SyntaxNode>,
|
||||
captureMap: CaptureMap,
|
||||
): string | undefined {
|
||||
if (nodeLabel === 'Property' && captureMap['definition.property']) {
|
||||
return extractPhpPropertyDescription(nodeName, captureMap['definition.property']) ?? undefined;
|
||||
const propertyNode = captureMap['definition.property'];
|
||||
if (nodeLabel === 'Property' && propertyNode) {
|
||||
const eloquentProperty = extractPhpPropertyDescription(nodeName, propertyNode);
|
||||
if (eloquentProperty) return eloquentProperty;
|
||||
}
|
||||
if (nodeLabel === 'Method' && captureMap['definition.method']) {
|
||||
return extractEloquentRelationDescription(captureMap['definition.method']) ?? undefined;
|
||||
const methodNode = captureMap['definition.method'];
|
||||
if (nodeLabel === 'Method' && methodNode) {
|
||||
const eloquentRelation = extractEloquentRelationDescription(methodNode);
|
||||
if (eloquentRelation) return eloquentRelation;
|
||||
}
|
||||
return undefined;
|
||||
return phpLeadingDocFallback(nodeLabel, nodeName, captureMap);
|
||||
}
|
||||
|
||||
/** Detect Laravel route files by path convention. */
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import { createClassExtractor } from '../class-extractors/generic.js';
|
|||
import { rubyClassConfig } from '../class-extractors/configs/ruby.js';
|
||||
import { defineLanguage } from '../language-provider.js';
|
||||
import type { AstFrameworkPatternConfig } from '../language-provider.js';
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { createLeadingDocDescriptionExtractor, type SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { typeConfig as rubyConfig } from '../type-extractors/ruby.js';
|
||||
import { routeRubyCall } from '../call-routing.js';
|
||||
import { rubyExportChecker } from '../export-detection.js';
|
||||
|
|
@ -197,6 +197,20 @@ export const rubyProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(rubyVariableConfig),
|
||||
classExtractor: createClassExtractor(rubyClassConfig),
|
||||
// ── Leading `#` comments (RDoc/YARD) → description (issue #2270). Magic
|
||||
// comments and the shebang are not documentation. ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor({
|
||||
lineCommentPrefixes: ['#'],
|
||||
lineDirectivePrefixes: [
|
||||
'# frozen_string_literal:',
|
||||
'# encoding:',
|
||||
'# coding:',
|
||||
'# -*-',
|
||||
'#!',
|
||||
'# rubocop:',
|
||||
'# typed:',
|
||||
],
|
||||
}),
|
||||
labelOverride: rubyLabelOverride,
|
||||
// Ruby MRO is kind-aware: prepend providers beat the class's own method,
|
||||
// which in turn beats include providers. The graph-level MRO phase
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import type { NodeLabel } from 'gitnexus-shared';
|
|||
import { createClassExtractor } from '../class-extractors/generic.js';
|
||||
import { rustClassConfig } from '../class-extractors/configs/rust.js';
|
||||
import { defineLanguage } from '../language-provider.js';
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { createLeadingDocDescriptionExtractor, type SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { typeConfig as rustConfig } from '../type-extractors/rust.js';
|
||||
import { rustExportChecker } from '../export-detection.js';
|
||||
import { createImportResolver } from '../import-resolvers/resolver-factory.js';
|
||||
|
|
@ -176,6 +176,13 @@ export const rustProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(rustVariableConfig),
|
||||
classExtractor: createClassExtractor(rustClassConfig),
|
||||
// ── Rust outer doc comments (`///`, `/** */`) → description (issue #2270).
|
||||
// `//!` / `/*!` are INNER docs (document the enclosing item), so they must
|
||||
// not attach to the following item — opt out of both. ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor({
|
||||
lineCommentPrefixes: ['///'],
|
||||
blockDocPrefixes: ['/**'],
|
||||
}),
|
||||
builtInNames: BUILT_INS,
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks ──────────
|
||||
emitScopeCaptures: emitRustScopeCaptures,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { createImportResolver } from '../import-resolvers/resolver-factory.js';
|
|||
import { swiftImportConfig } from '../import-resolvers/configs/swift.js';
|
||||
import { SWIFT_QUERIES } from '../tree-sitter-queries.js';
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import { createFieldExtractor } from '../field-extractors/generic.js';
|
||||
import { swiftConfig as swiftFieldConfig } from '../field-extractors/configs/swift.js';
|
||||
import { createMethodExtractor } from '../method-extractors/generic.js';
|
||||
|
|
@ -243,6 +244,8 @@ export const swiftProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(swiftVariableConfig),
|
||||
classExtractor: createClassExtractor(swiftClassConfig),
|
||||
// ── Swift doc comments (`///`, `/** */`) → description (issue #2270) ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor(),
|
||||
orderSameNameTypeCandidates: orderSwiftSameNameTypeCandidates,
|
||||
builtInNames: BUILT_INS,
|
||||
// ── Scope-based resolution hooks (RFC #909 Ring 3, issue #937). See
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
javascriptClassConfig,
|
||||
} from '../class-extractors/configs/typescript-javascript.js';
|
||||
import type { SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js';
|
||||
import { createTypeScriptCfgVisitor } from '../cfg/visitors/typescript.js';
|
||||
import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js';
|
||||
import { tsExportChecker } from '../export-detection.js';
|
||||
|
|
@ -344,6 +345,11 @@ export const typescriptProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(typescriptVariableConfig),
|
||||
classExtractor: createClassExtractor(typescriptClassConfig),
|
||||
// ── JSDoc → description (issue #2270). An exported decl is captured as the
|
||||
// inner declaration; its JSDoc precedes the wrapping `export_statement`. ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor({
|
||||
wrapperNodeTypes: ['export_statement'],
|
||||
}),
|
||||
builtInNames: BUILT_INS,
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ──────────
|
||||
|
|
@ -406,6 +412,11 @@ export const javascriptProvider = defineLanguage({
|
|||
}),
|
||||
variableExtractor: createVariableExtractor(javascriptVariableConfig),
|
||||
classExtractor: createClassExtractor(javascriptClassConfig),
|
||||
// ── JSDoc → description (issue #2270). An exported decl is captured as the
|
||||
// inner declaration; its JSDoc precedes the wrapping `export_statement`. ──
|
||||
descriptionExtractor: createLeadingDocDescriptionExtractor({
|
||||
wrapperNodeTypes: ['export_statement'],
|
||||
}),
|
||||
builtInNames: BUILT_INS,
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ──────────
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ export const DEFINITION_CAPTURE_KEYS = [
|
|||
|
||||
/** Extract the definition node from a tree-sitter query capture map. */
|
||||
export const getDefinitionNodeFromCaptures = (
|
||||
captureMap: Record<string, SyntaxNode>,
|
||||
captureMap: Record<string, SyntaxNode | undefined>,
|
||||
): SyntaxNode | null => {
|
||||
for (const key of DEFINITION_CAPTURE_KEYS) {
|
||||
if (captureMap[key]) return captureMap[key];
|
||||
|
|
@ -310,7 +310,7 @@ export function findAncestorBeforeBoundary(
|
|||
* Returns null if the capture should be skipped (import, call, C/C++ duplicate, missing name).
|
||||
*/
|
||||
export function getLabelFromCaptures(
|
||||
captureMap: Record<string, SyntaxNode>,
|
||||
captureMap: Record<string, SyntaxNode | undefined>,
|
||||
provider: LanguageProvider,
|
||||
): NodeLabel | null {
|
||||
if (captureMap['import'] || captureMap['call']) return null;
|
||||
|
|
@ -926,6 +926,227 @@ export function findChild(node: SyntaxNode, type: string): SyntaxNode | null {
|
|||
return null;
|
||||
}
|
||||
|
||||
/** Remove bidi-override and zero-width control characters. Doc text is
|
||||
* attacker-influenced (any indexed repo) and is returned verbatim to MCP
|
||||
* clients, so strip Trojan-Source-style hidden controls from the description
|
||||
* before it leaves the extractor (#2286 review). Scoped to the doc-comment path
|
||||
* only — global `sanitizeUTF8` is intentionally untouched. */
|
||||
const stripBidiAndZeroWidth = (text: string): string =>
|
||||
Array.from(text)
|
||||
.filter((ch) => {
|
||||
const c = ch.codePointAt(0) ?? 0;
|
||||
// Bidi overrides/isolates (U+202A–202E, U+2066–2069), zero-width
|
||||
// space/joiners (U+200B–200D), and BOM/zero-width-no-break (U+FEFF).
|
||||
return !(
|
||||
(c >= 0x202a && c <= 0x202e) ||
|
||||
(c >= 0x2066 && c <= 0x2069) ||
|
||||
(c >= 0x200b && c <= 0x200d) ||
|
||||
c === 0xfeff
|
||||
);
|
||||
})
|
||||
.join('');
|
||||
|
||||
/** Normalize a block doc comment body: strip the opening (double-star or
|
||||
* bang) delimiter, the closing delimiter, and per-line gutter stars, then
|
||||
* collapse whitespace so tag content stays as searchable words. */
|
||||
const normalizeBlockDocComment = (text: string): string | undefined => {
|
||||
const inner = stripBidiAndZeroWidth(
|
||||
text
|
||||
.replace(/^\/\*[*!]/, '')
|
||||
// Close delimiter: tolerate the degenerate empty comment `/**/`, where the
|
||||
// opening strip already consumed the shared `*`, leaving a lone `/`.
|
||||
.replace(/\*?\/\s*$/, '')
|
||||
.replace(/^[ \t]*\*[ \t]?/gm, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim(),
|
||||
);
|
||||
return inner.length > 0 ? inner : undefined;
|
||||
};
|
||||
|
||||
/** Default line-comment prefixes treated as documentation: the universal
|
||||
* triple-slash / bang-slash doc markers (Rust, C#, Dart, Swift, Doxygen).
|
||||
* Go (`//`) and Ruby (`#`) opt into their conventional markers explicitly. */
|
||||
const DEFAULT_LINE_DOC_PREFIXES: readonly string[] = ['///', '//!'];
|
||||
|
||||
/** Default block-comment doc openers: Javadoc/JSDoc-style `/**` and Doxygen
|
||||
* `/*!`. Rust opts out of `/*!` (and `//!`) because those are *inner* docs that
|
||||
* document the enclosing item, not the following one. */
|
||||
const DEFAULT_BLOCK_DOC_PREFIXES: readonly string[] = ['/**', '/*!'];
|
||||
|
||||
/** A file-top `/** … *\/` license/copyright/file-overview block has no
|
||||
* package/import sibling to shield it, so it would otherwise be absorbed as the
|
||||
* first declaration's description (PR #2286 review). These markers identify such
|
||||
* headers; they are specific enough not to fire on an ordinary symbol doc that
|
||||
* merely mentions the word "copyright". `@file`/`@fileoverview` are explicitly
|
||||
* file-level JSDoc tags, so a block carrying them is not a symbol doc. */
|
||||
const FILE_HEADER_MARKER =
|
||||
/SPDX-License-Identifier|@licen[sc]e\b|@fileoverview\b|@file\b|Licen[sc]ed under|copyright\s*(\(c\)|©|\d{4})/i;
|
||||
|
||||
/**
|
||||
* Extract the normalized text of a leading doc comment immediately preceding a
|
||||
* definition node — covering both block doc comments (Javadoc / KDoc / JSDoc /
|
||||
* PHPDoc / Doxygen, opened by `/**` or `/*!`) and runs of line doc comments
|
||||
* (`///`, `//!`, or the caller-supplied prefixes such as Go's `//` or Ruby's
|
||||
* `#`). Returns `undefined` when there is no preceding doc comment or it is
|
||||
* empty.
|
||||
*
|
||||
* Grammar-agnostic by design: matches on the comment text prefix rather than a
|
||||
* grammar node type, because the comment node is named differently across
|
||||
* grammars (`block_comment`, `multiline_comment`, `comment`, `line_comment`).
|
||||
* Annotations and modifiers live inside the definition node, so the doc comment
|
||||
* remains the definition's `previousNamedSibling` even on annotated/decorated
|
||||
* declarations.
|
||||
*
|
||||
* Block comments are taken as the immediately-preceding sibling (intervening
|
||||
* package/import/code siblings already shield a file-level license block from
|
||||
* the first declaration). Line doc comments enforce row-adjacency: the first
|
||||
* comment must sit on the line directly above the definition, and each comment
|
||||
* walked further up must sit directly above the previous one — so a run stops
|
||||
* at a blank line. This matches godoc/RDoc/rustdoc convention and prevents an
|
||||
* unrelated comment block (a license header, a Ruby shebang + magic comment)
|
||||
* separated by a blank line from being absorbed. Adjacency is checked on
|
||||
* `startPosition.row` (reliable) rather than `endPosition.row`, since some
|
||||
* grammars fold the trailing newline into the comment node.
|
||||
*
|
||||
* Normalization mirrors Python docstring handling: strip the comment delimiters
|
||||
* / per-line markers, then collapse whitespace to single spaces so tag content
|
||||
* (`@param`, `@deprecated since 2.0, use computeBalanceV2`) survives.
|
||||
*
|
||||
* When the captured definition is an inner node and its own preceding sibling
|
||||
* carries no doc, the search retries from a wrapping node whose type is listed in
|
||||
* `opts.wrapperNodeTypes` (e.g. an `export_statement` wrapping an exported
|
||||
* function/class — the JSDoc precedes the wrapper, not the inner declaration).
|
||||
*/
|
||||
export interface LeadingDocCommentOptions {
|
||||
/** Line-comment doc prefixes (defaults to {@link DEFAULT_LINE_DOC_PREFIXES};
|
||||
* Go passes `['//']`, Ruby passes `['#']`). */
|
||||
lineCommentPrefixes?: readonly string[];
|
||||
/** Grammar node types that wrap a definition such that the doc comment is the
|
||||
* wrapper's preceding sibling rather than the definition's. TS/JS pass
|
||||
* `['export_statement']`. Empty by default → no wrapper retry. */
|
||||
wrapperNodeTypes?: readonly string[];
|
||||
/** Line-comment prefixes that are tool/build directives or magic comments
|
||||
* rather than documentation (Go passes `['//go:', '// +build', …]`, Ruby
|
||||
* passes `['# frozen_string_literal:', '#!', …]`). A matching line is skipped
|
||||
* in the doc run rather than absorbed. Empty by default. */
|
||||
lineDirectivePrefixes?: readonly string[];
|
||||
/** Block-comment doc openers (defaults to `['/**', '/*!']`). Rust passes
|
||||
* `['/**']` so its inner-doc `/*!` does not attach to the following item. */
|
||||
blockDocPrefixes?: readonly string[];
|
||||
}
|
||||
|
||||
export function extractLeadingDocComment(
|
||||
node: SyntaxNode,
|
||||
opts: LeadingDocCommentOptions = {},
|
||||
): string | undefined {
|
||||
const lineCommentPrefixes = opts.lineCommentPrefixes ?? DEFAULT_LINE_DOC_PREFIXES;
|
||||
const wrapperNodeTypes = opts.wrapperNodeTypes ?? [];
|
||||
const lineDirectivePrefixes = opts.lineDirectivePrefixes ?? [];
|
||||
const blockDocPrefixes = opts.blockDocPrefixes ?? DEFAULT_BLOCK_DOC_PREFIXES;
|
||||
|
||||
const fromNode = (anchor: SyntaxNode): string | undefined => {
|
||||
const prev = anchor.previousNamedSibling;
|
||||
if (!prev) return undefined;
|
||||
|
||||
// Block doc comment: /** ... */ or /*! ... */
|
||||
if (blockDocPrefixes.some((p) => prev.text.startsWith(p))) {
|
||||
// Skip a file-top license/copyright/overview header (no package/import
|
||||
// sibling shields it from the first declaration). A strict row-adjacency
|
||||
// check is unreliable here — some grammars fold the trailing newline into
|
||||
// the comment node — so match header markers instead.
|
||||
if (FILE_HEADER_MARKER.test(prev.text)) return undefined;
|
||||
return normalizeBlockDocComment(prev.text);
|
||||
}
|
||||
|
||||
// Run of row-adjacent preceding line doc comments (e.g. `///` or `//`).
|
||||
const matchedPrefix = (text: string): string | undefined =>
|
||||
lineCommentPrefixes.find((prefix) => text.trimStart().startsWith(prefix));
|
||||
const isDirective = (text: string): boolean =>
|
||||
lineDirectivePrefixes.some((prefix) => text.trimStart().startsWith(prefix));
|
||||
|
||||
const lines: string[] = [];
|
||||
let current: SyntaxNode | null = prev;
|
||||
let expectedRow = anchor.startPosition.row - 1;
|
||||
while (current) {
|
||||
const text = current.text;
|
||||
const prefix = matchedPrefix(text);
|
||||
if (prefix === undefined || current.startPosition.row !== expectedRow) break;
|
||||
// A build/tool directive or magic comment (e.g. `//go:build`,
|
||||
// `# frozen_string_literal:`) is not documentation: skip it but keep
|
||||
// walking the adjacent run, so a real doc above it is still collected.
|
||||
if (!isDirective(text)) lines.unshift(text.trimStart().slice(prefix.length));
|
||||
expectedRow = current.startPosition.row - 1;
|
||||
current = current.previousNamedSibling;
|
||||
}
|
||||
|
||||
const joined = stripBidiAndZeroWidth(lines.join(' ').replace(/\s+/g, ' ').trim());
|
||||
return joined.length > 0 ? joined : undefined;
|
||||
};
|
||||
|
||||
const direct = fromNode(node);
|
||||
if (direct !== undefined) return direct;
|
||||
|
||||
const parent = node.parent;
|
||||
if (parent && wrapperNodeTypes.includes(parent.type)) {
|
||||
return fromNode(parent);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Node labels that can carry a leading doc comment — callables and type-like
|
||||
* declarations. Field/property/variable/const doc is intentionally excluded
|
||||
* (issue #2270 scopes this to method/type documentation). Language-neutral:
|
||||
* a label a given grammar never emits simply never matches.
|
||||
*
|
||||
* Bounded to labels that are also in `embeddings/types.ts` `EMBEDDABLE_LABELS`:
|
||||
* the description is only useful once it reaches the embedding metadata header,
|
||||
* and the embedding pipeline only queries embeddable labels. Extracting docs
|
||||
* for a non-embeddable label is a wasted write that never becomes searchable.
|
||||
* A subset invariant in the unit tests guards against drift. Making currently-
|
||||
* non-embeddable doc-bearing labels (Module, Delegate, Annotation, and C++
|
||||
* `Template`) searchable is tracked as a follow-up — it needs an embedding-
|
||||
* pipeline/schema change beyond this fix. */
|
||||
export const DOC_BEARING_LABELS: ReadonlySet<NodeLabel> = new Set<NodeLabel>([
|
||||
'Function',
|
||||
'Method',
|
||||
'Constructor',
|
||||
'Class',
|
||||
'Interface',
|
||||
'Enum',
|
||||
'Struct',
|
||||
'Trait',
|
||||
'Record',
|
||||
'Union',
|
||||
'Namespace',
|
||||
'TypeAlias',
|
||||
'Macro',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Build a `LanguageProvider.descriptionExtractor` that surfaces a definition's
|
||||
* leading doc comment as its `description` (issue #2270). For labels in
|
||||
* {@link DOC_BEARING_LABELS} (which is bounded to embeddable labels) the text
|
||||
* then reaches the embedding metadata header and becomes semantically searchable.
|
||||
*
|
||||
* Language-neutral factory (names no language): guards on
|
||||
* {@link DOC_BEARING_LABELS}; callers pass per-language doc-comment behavior via
|
||||
* {@link LeadingDocCommentOptions} (line prefixes, export-style wrappers, …)
|
||||
* which is threaded straight through to {@link extractLeadingDocComment}.
|
||||
*/
|
||||
export const createLeadingDocDescriptionExtractor = (
|
||||
opts: LeadingDocCommentOptions = {},
|
||||
): ((
|
||||
nodeLabel: NodeLabel,
|
||||
nodeName: string,
|
||||
captureMap: Record<string, SyntaxNode | undefined>,
|
||||
) => string | undefined) => {
|
||||
return (nodeLabel, _nodeName, captureMap) => {
|
||||
if (!DOC_BEARING_LABELS.has(nodeLabel)) return undefined;
|
||||
const definitionNode = getDefinitionNodeFromCaptures(captureMap);
|
||||
return definitionNode ? extractLeadingDocComment(definitionNode, opts) : undefined;
|
||||
};
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Capture + range helpers (formerly python/ast-utils.ts — language-agnostic)
|
||||
// ============================================================================
|
||||
|
|
|
|||
|
|
@ -2151,7 +2151,20 @@ const processFileGroup = (
|
|||
`${file.path}:${qualifiedName}${classTemplateTag}${arityTag}${parameterShapeTag}${constraintsTag}`,
|
||||
);
|
||||
|
||||
const description = provider.descriptionExtractor?.(nodeLabel, nodeName, captureMap);
|
||||
let description: string | undefined;
|
||||
try {
|
||||
description = provider.descriptionExtractor?.(nodeLabel, nodeName, captureMap);
|
||||
} catch (err) {
|
||||
// A throw here (an unexpected tree-sitter node shape, a provider bug) must
|
||||
// NOT propagate — it would escape processFileGroup to the language-group
|
||||
// catch, which treats any throw as "parser unavailable" and silently drops
|
||||
// every remaining file in the group. Mirrors the extractTemplateConstraints
|
||||
// guard above (#2286 review).
|
||||
reportWarning(
|
||||
`Description extraction failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
description = undefined;
|
||||
}
|
||||
|
||||
let frameworkHint = definitionNode
|
||||
? detectFrameworkFromAST(language, (definitionNode.text || '').slice(0, 300))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
/**
|
||||
* End-to-end coverage for issue #2270: a leading doc comment must survive the
|
||||
* full parse pipeline into the node's `description` property (the field the
|
||||
* embedding metadata header reads for semantic search). The unit tests stop at
|
||||
* the `descriptionExtractor` hook; this exercises the real worker pipeline and
|
||||
* the highest-value case — an EXPORTED TS function, whose JSDoc precedes the
|
||||
* wrapping `export_statement` (PR #2286 review fix).
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { runPipelineFromRepo } from './resolvers/helpers.js';
|
||||
import type { PipelineResult } from '../../src/types/pipeline.js';
|
||||
|
||||
function createTsRepo(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'doc-desc-e2e-'));
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'package.json'),
|
||||
JSON.stringify({ name: 'doc-desc-e2e', version: '1.0.0' }),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'index.ts'),
|
||||
[
|
||||
'/** Computes the running balance, marker EXPORTEDDOC. */',
|
||||
'export function computeBalance(userId: number): number {',
|
||||
' return userId;',
|
||||
'}',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe('doc-comment description end-to-end (issue #2270)', () => {
|
||||
it('surfaces an exported function JSDoc as its node description through the pipeline', async () => {
|
||||
const result: PipelineResult = await runPipelineFromRepo(createTsRepo(), () => {}, {
|
||||
skipGraphPhases: true,
|
||||
workerThresholdsForTest: { minFiles: 1, minBytes: 1 },
|
||||
workerPoolSize: 2,
|
||||
});
|
||||
|
||||
const descriptions = new Map<string, unknown>();
|
||||
result.graph.forEachNode((node) => {
|
||||
descriptions.set(`${node.label}:${node.properties.name}`, node.properties.description);
|
||||
});
|
||||
|
||||
expect(result.usedWorkerPool).toBe(true);
|
||||
expect(descriptions.get('Function:computeBalance')).toContain('EXPORTEDDOC');
|
||||
});
|
||||
});
|
||||
70
gitnexus/test/unit/java-description-extractor.test.ts
Normal file
70
gitnexus/test/unit/java-description-extractor.test.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Unit tests for `javaProvider.descriptionExtractor` (issue #2270, U2).
|
||||
*
|
||||
* Confirms Java method/type Javadoc is surfaced as the symbol `description`,
|
||||
* which is what reaches the embedding metadata header and makes Javadoc-only
|
||||
* terms semantically searchable. Drives the real provider hook with a captureMap
|
||||
* built from a parsed tree (matching what the parse worker passes in).
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import Java from 'tree-sitter-java';
|
||||
import { javaProvider } from '../../src/core/ingestion/languages/java.js';
|
||||
import type { CaptureMap } from '../../src/core/ingestion/language-provider.js';
|
||||
|
||||
function captureMapFor(src: string, nodeType: string, captureKey: string): CaptureMap {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(Java);
|
||||
const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0];
|
||||
expect(node, `expected a ${nodeType} node`).toBeDefined();
|
||||
return { [captureKey]: node };
|
||||
}
|
||||
|
||||
const PROBE = `package demo;
|
||||
public class Probe {
|
||||
/**
|
||||
* Computes the running balance across all user accounts.
|
||||
* @param userId the unique user identifier
|
||||
* @deprecated since 2.0, use computeBalanceV2
|
||||
*/
|
||||
public java.math.BigDecimal computeBalance(Long userId) { return null; }
|
||||
}`;
|
||||
|
||||
describe('javaProvider.descriptionExtractor', () => {
|
||||
it('is registered on the provider (regression guard for issue #2270)', () => {
|
||||
expect(javaProvider.descriptionExtractor).toBeDefined();
|
||||
});
|
||||
|
||||
it('extracts the method Javadoc, including the @deprecated marker term', () => {
|
||||
const captureMap = captureMapFor(PROBE, 'method_declaration', 'definition.method');
|
||||
const description = javaProvider.descriptionExtractor?.('Method', 'computeBalance', captureMap);
|
||||
expect(description).toContain('Computes the running balance');
|
||||
expect(description).toContain('computeBalanceV2');
|
||||
});
|
||||
|
||||
it('extracts class-level Javadoc', () => {
|
||||
const captureMap = captureMapFor(
|
||||
`/**\n * A probe class.\n */\npublic class Probe {}`,
|
||||
'class_declaration',
|
||||
'definition.class',
|
||||
);
|
||||
expect(javaProvider.descriptionExtractor?.('Class', 'Probe', captureMap)).toBe(
|
||||
'A probe class.',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns undefined for a Java method with no Javadoc', () => {
|
||||
const captureMap = captureMapFor(
|
||||
`class P { void m() {} }`,
|
||||
'method_declaration',
|
||||
'definition.method',
|
||||
);
|
||||
expect(javaProvider.descriptionExtractor?.('Method', 'm', captureMap)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a non-doc-bearing label (e.g. Variable)', () => {
|
||||
// Even with a definition node present, a Variable label is out of scope.
|
||||
const captureMap = captureMapFor(PROBE, 'method_declaration', 'definition.method');
|
||||
expect(javaProvider.descriptionExtractor?.('Variable', 'x', captureMap)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
68
gitnexus/test/unit/kotlin-description-extractor.test.ts
Normal file
68
gitnexus/test/unit/kotlin-description-extractor.test.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Unit tests for `kotlinProvider.descriptionExtractor` (issue #2270, U3).
|
||||
*
|
||||
* Confirms Kotlin function/type KDoc is surfaced as the symbol `description`,
|
||||
* mirroring the Java behavior. Drives the real provider hook with a captureMap
|
||||
* built from a parsed tree.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js';
|
||||
import { kotlinProvider } from '../../src/core/ingestion/languages/kotlin.js';
|
||||
import type { CaptureMap } from '../../src/core/ingestion/language-provider.js';
|
||||
|
||||
// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111).
|
||||
const Kotlin = requireVendoredGrammar('tree-sitter-kotlin');
|
||||
|
||||
function captureMapFor(src: string, nodeType: string, captureKey: string): CaptureMap {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(Kotlin);
|
||||
const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0];
|
||||
expect(node, `expected a ${nodeType} node`).toBeDefined();
|
||||
return { [captureKey]: node };
|
||||
}
|
||||
|
||||
const PROBE = `package demo
|
||||
class Probe {
|
||||
/**
|
||||
* Computes the running balance, use computeBalanceV2
|
||||
*/
|
||||
fun computeBalance(userId: Long): String? { return null }
|
||||
}`;
|
||||
|
||||
describe('kotlinProvider.descriptionExtractor', () => {
|
||||
it('is registered on the provider (regression guard for issue #2270)', () => {
|
||||
expect(kotlinProvider.descriptionExtractor).toBeDefined();
|
||||
});
|
||||
|
||||
it('extracts the function KDoc, including the marker term', () => {
|
||||
const captureMap = captureMapFor(PROBE, 'function_declaration', 'definition.method');
|
||||
const description = kotlinProvider.descriptionExtractor?.(
|
||||
'Method',
|
||||
'computeBalance',
|
||||
captureMap,
|
||||
);
|
||||
expect(description).toContain('Computes the running balance');
|
||||
expect(description).toContain('computeBalanceV2');
|
||||
});
|
||||
|
||||
it('extracts class-level KDoc', () => {
|
||||
const captureMap = captureMapFor(
|
||||
`/**\n * A probe class.\n */\nclass Probe`,
|
||||
'class_declaration',
|
||||
'definition.class',
|
||||
);
|
||||
expect(kotlinProvider.descriptionExtractor?.('Class', 'Probe', captureMap)).toBe(
|
||||
'A probe class.',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns undefined for a Kotlin function with no KDoc', () => {
|
||||
const captureMap = captureMapFor(
|
||||
`fun m(): String? { return null }`,
|
||||
'function_declaration',
|
||||
'definition.method',
|
||||
);
|
||||
expect(kotlinProvider.descriptionExtractor?.('Function', 'm', captureMap)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
167
gitnexus/test/unit/leading-doc-comment.test.ts
Normal file
167
gitnexus/test/unit/leading-doc-comment.test.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
/**
|
||||
* Unit tests for `extractLeadingDocComment` (issue #2270, U1).
|
||||
*
|
||||
* Verifies the shared helper that pulls a `/** ... *\/` leading doc comment
|
||||
* (Javadoc / KDoc) off the definition node's preceding named sibling. The
|
||||
* helper is grammar-agnostic: it matches on the `/**` text prefix, so it works
|
||||
* for both tree-sitter-java (`block_comment`) and tree-sitter-kotlin
|
||||
* (`multiline_comment`).
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import Java from 'tree-sitter-java';
|
||||
import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js';
|
||||
import {
|
||||
extractLeadingDocComment,
|
||||
type SyntaxNode,
|
||||
} from '../../src/core/ingestion/utils/ast-helpers.js';
|
||||
|
||||
// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111).
|
||||
const Kotlin = requireVendoredGrammar('tree-sitter-kotlin');
|
||||
|
||||
function firstNode(language: unknown, src: string, type: string): SyntaxNode {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(language);
|
||||
const node = parser.parse(src).rootNode.descendantsOfType(type)[0];
|
||||
expect(node, `expected a ${type} node in source`).toBeDefined();
|
||||
return node;
|
||||
}
|
||||
|
||||
describe('extractLeadingDocComment', () => {
|
||||
it('extracts a multi-line Javadoc including tag content (issue #2270 repro)', () => {
|
||||
const src = `package demo;
|
||||
public class Probe {
|
||||
/**
|
||||
* Computes the running balance across all user accounts.
|
||||
* @param userId the unique user identifier
|
||||
* @deprecated since 2.0, use computeBalanceV2
|
||||
*/
|
||||
public java.math.BigDecimal computeBalance(Long userId) { return null; }
|
||||
}`;
|
||||
const method = firstNode(Java, src, 'method_declaration');
|
||||
const doc = extractLeadingDocComment(method);
|
||||
expect(doc).toContain('Computes the running balance');
|
||||
expect(doc).toContain('userId');
|
||||
expect(doc).toContain('computeBalanceV2');
|
||||
});
|
||||
|
||||
it('extracts a class-level Javadoc', () => {
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/**\n * A probe class.\n */\npublic class Probe {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(cls)).toBe('A probe class.');
|
||||
});
|
||||
|
||||
it('returns undefined when there is no preceding comment', () => {
|
||||
const method = firstNode(Java, `class P { void m() {} }`, 'method_declaration');
|
||||
expect(extractLeadingDocComment(method)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a non-doc block comment (license header style)', () => {
|
||||
const method = firstNode(
|
||||
Java,
|
||||
`class P {\n/* not a doc comment */\nvoid m() {}\n}`,
|
||||
'method_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(method)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a // line comment', () => {
|
||||
const method = firstNode(
|
||||
Java,
|
||||
`class P {\n// just a line comment\nvoid m() {}\n}`,
|
||||
'method_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(method)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for an empty doc comment', () => {
|
||||
const method = firstNode(Java, `class P {\n/** */\nvoid m() {}\n}`, 'method_declaration');
|
||||
expect(extractLeadingDocComment(method)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for the degenerate empty comment /**/ (no spurious slash)', () => {
|
||||
const method = firstNode(Java, `class P {\n/**/\nvoid m() {}\n}`, 'method_declaration');
|
||||
expect(extractLeadingDocComment(method)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('strips the */ delimiters and per-line * gutter markers', () => {
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/**\n * Line one.\n * Line two.\n */\nclass P {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
const doc = extractLeadingDocComment(cls);
|
||||
expect(doc).toBe('Line one. Line two.');
|
||||
expect(doc).not.toContain('*');
|
||||
expect(doc).not.toContain('/');
|
||||
});
|
||||
|
||||
it('skips a file-top SPDX license header (no package/import shield)', () => {
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/** SPDX-License-Identifier: MIT */\npublic class Foo {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(cls)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips a file-top copyright header block', () => {
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/**\n * Copyright (c) 2026 Acme Corp. All rights reserved.\n * Licensed under the Apache License 2.0.\n */\npublic class Foo {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(cls)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does NOT over-fire: a real doc that merely mentions copyright is preserved', () => {
|
||||
const method = firstNode(
|
||||
Java,
|
||||
`class P {\n/** Returns the copyright owner name, marker KEEPME. */\nString owner() { return null; }\n}`,
|
||||
'method_declaration',
|
||||
);
|
||||
const doc = extractLeadingDocComment(method);
|
||||
expect(doc).toContain('KEEPME');
|
||||
expect(doc).toContain('copyright owner');
|
||||
});
|
||||
|
||||
it('strips bidi-override and zero-width controls from the description', () => {
|
||||
const rlo = String.fromCharCode(0x202e); // right-to-left override
|
||||
const zwsp = String.fromCharCode(0x200b); // zero-width space
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/** Doc ${rlo}with${zwsp} hidden controls, marker BIDIMARK. */\npublic class Foo {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
const doc = extractLeadingDocComment(cls);
|
||||
expect(doc).toContain('BIDIMARK');
|
||||
expect(doc).not.toContain(rlo);
|
||||
expect(doc).not.toContain(zwsp);
|
||||
});
|
||||
|
||||
it('leaves a plain ASCII doc comment unchanged', () => {
|
||||
const cls = firstNode(
|
||||
Java,
|
||||
`/** Plain doc, marker ASCIIMARK. */\nclass Foo {}`,
|
||||
'class_declaration',
|
||||
);
|
||||
expect(extractLeadingDocComment(cls)).toBe('Plain doc, marker ASCIIMARK.');
|
||||
});
|
||||
|
||||
it('extracts a Kotlin KDoc (grammar-agnostic prefix match, multiline_comment)', () => {
|
||||
const src = `package demo
|
||||
class Probe {
|
||||
/**
|
||||
* Computes the running balance, use computeBalanceV2
|
||||
*/
|
||||
fun computeBalance(userId: Long): String? { return null }
|
||||
}`;
|
||||
const fn = firstNode(Kotlin, src, 'function_declaration');
|
||||
const doc = extractLeadingDocComment(fn);
|
||||
expect(doc).toContain('Computes the running balance');
|
||||
expect(doc).toContain('computeBalanceV2');
|
||||
});
|
||||
});
|
||||
391
gitnexus/test/unit/leading-doc-description-all-languages.test.ts
Normal file
391
gitnexus/test/unit/leading-doc-description-all-languages.test.ts
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
/**
|
||||
* Cross-language coverage for the leading-doc `descriptionExtractor`
|
||||
* (issue #2270). Confirms every documentable language registers the hook, and
|
||||
* exercises each doc-comment family through the real provider hooks:
|
||||
* - block doc comments (double-star / bang): TypeScript, C++ (Java/Kotlin elsewhere)
|
||||
* - triple-slash line runs: Rust, C#
|
||||
* - godoc double-slash runs: Go
|
||||
* - hash runs: Ruby
|
||||
* - PHPDoc docblock fallback: PHP
|
||||
*
|
||||
* Dart and Swift share the default block / triple-slash config already exercised
|
||||
* their native grammars can fail to load in some environments, so they are
|
||||
* covered by the registration check (no parse) rather than a behavior parse.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import TypeScript from 'tree-sitter-typescript';
|
||||
import Go from 'tree-sitter-go';
|
||||
import Rust from 'tree-sitter-rust';
|
||||
import CSharp from 'tree-sitter-c-sharp';
|
||||
import Ruby from 'tree-sitter-ruby';
|
||||
import CPP from 'tree-sitter-cpp';
|
||||
import PHP from 'tree-sitter-php';
|
||||
import { SupportedLanguages } from '../../src/config/supported-languages.js';
|
||||
import { getProvider } from '../../src/core/ingestion/languages/index.js';
|
||||
import { DOC_BEARING_LABELS } from '../../src/core/ingestion/utils/ast-helpers.js';
|
||||
import { EMBEDDABLE_LABELS } from '../../src/core/embeddings/types.js';
|
||||
import type { CaptureMap } from '../../src/core/ingestion/language-provider.js';
|
||||
import type { NodeLabel } from 'gitnexus-shared';
|
||||
|
||||
function describeFromProvider(
|
||||
language: SupportedLanguages,
|
||||
grammar: unknown,
|
||||
src: string,
|
||||
nodeType: string,
|
||||
captureKey: string,
|
||||
label: NodeLabel,
|
||||
name: string,
|
||||
): string | undefined {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(grammar);
|
||||
const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0];
|
||||
expect(node, `expected a ${nodeType} node for ${language}`).toBeDefined();
|
||||
const captureMap: CaptureMap = { [captureKey]: node };
|
||||
return getProvider(language).descriptionExtractor?.(label, name, captureMap);
|
||||
}
|
||||
|
||||
// Languages that should surface a description (everything except Vue/Cobol).
|
||||
const DOCUMENTABLE_LANGUAGES: readonly SupportedLanguages[] = [
|
||||
SupportedLanguages.JavaScript,
|
||||
SupportedLanguages.TypeScript,
|
||||
SupportedLanguages.Python,
|
||||
SupportedLanguages.Java,
|
||||
SupportedLanguages.C,
|
||||
SupportedLanguages.CPlusPlus,
|
||||
SupportedLanguages.CSharp,
|
||||
SupportedLanguages.Go,
|
||||
SupportedLanguages.Ruby,
|
||||
SupportedLanguages.Rust,
|
||||
SupportedLanguages.PHP,
|
||||
SupportedLanguages.Kotlin,
|
||||
SupportedLanguages.Swift,
|
||||
SupportedLanguages.Dart,
|
||||
];
|
||||
|
||||
describe('DOC_BEARING_LABELS is bounded to embeddable labels (issue #2270 review fix)', () => {
|
||||
it('every doc-bearing label is in EMBEDDABLE_LABELS so its description is searchable', () => {
|
||||
const embeddable = new Set<string>(EMBEDDABLE_LABELS);
|
||||
const notEmbeddable = [...DOC_BEARING_LABELS].filter((label) => !embeddable.has(label));
|
||||
expect(notEmbeddable).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('leading-doc descriptionExtractor — registration coverage', () => {
|
||||
it.each(DOCUMENTABLE_LANGUAGES)('%s provider registers descriptionExtractor', (language) => {
|
||||
expect(getProvider(language).descriptionExtractor).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('leading-doc descriptionExtractor — behavior per comment family', () => {
|
||||
it('TypeScript JSDoc block', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.TypeScript,
|
||||
TypeScript.typescript,
|
||||
`/** Adds two numbers, marker TSMARK. */\nfunction add(a: number, b: number) { return a + b; }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('TSMARK');
|
||||
});
|
||||
|
||||
it('Go godoc (// run)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\n// Add returns the sum, marker GOMARK.\nfunc Add(a int) int { return a }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Add',
|
||||
);
|
||||
expect(d).toContain('GOMARK');
|
||||
});
|
||||
|
||||
it('Rust /// doc comment', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Rust,
|
||||
Rust,
|
||||
`/// Adds things, marker RSMARK.\nfn add(a: i32) -> i32 { a }`,
|
||||
'function_item',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('RSMARK');
|
||||
});
|
||||
|
||||
it('C# /// XML doc comment', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.CSharp,
|
||||
CSharp,
|
||||
`class C {\n/// <summary>Adds, marker CSMARK</summary>\nvoid M() {}\n}`,
|
||||
'method_declaration',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'M',
|
||||
);
|
||||
expect(d).toContain('CSMARK');
|
||||
});
|
||||
|
||||
it('Ruby # comment run', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Ruby,
|
||||
Ruby,
|
||||
`# Adds things, marker RBMARK.\ndef add(a)\n a\nend`,
|
||||
'method',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('RBMARK');
|
||||
});
|
||||
|
||||
it('C++ Doxygen block', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.CPlusPlus,
|
||||
CPP,
|
||||
`/** Adds, marker CPPMARK. */\nint add(int a) { return a; }`,
|
||||
'function_definition',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('CPPMARK');
|
||||
});
|
||||
|
||||
it('PHP docblock fallback (non-Eloquent prose)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.PHP,
|
||||
PHP.php,
|
||||
`<?php\nclass C {\n/** Adds, marker PHPMARK. */\npublic function add($a) { return $a; }\n}`,
|
||||
'method_declaration',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('PHPMARK');
|
||||
});
|
||||
|
||||
it('returns undefined for a function with no leading doc comment (Go)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\nfunc Bare() {}`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Bare',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
|
||||
it('collects a multi-line /// run (Rust)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Rust,
|
||||
Rust,
|
||||
`/// First line.\n/// Second line, marker MULTILINE.\nfn add(a: i32) -> i32 { a }`,
|
||||
'function_item',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('First line');
|
||||
expect(d).toContain('MULTILINE');
|
||||
});
|
||||
|
||||
it('does NOT attach a Rust //! inner doc to the following item (inner-doc semantics)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Rust,
|
||||
Rust,
|
||||
`//! Inner doc, marker INNERMARK.\nfn add(a: i32) -> i32 { a }`,
|
||||
'function_item',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does NOT attach a Rust /*! inner block doc to the following item', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Rust,
|
||||
Rust,
|
||||
`/*! Inner block doc, marker INNERBLOCK. */\nfn add(a: i32) -> i32 { a }`,
|
||||
'function_item',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
|
||||
it('collects a /*! Doxygen bang block (C++)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.CPlusPlus,
|
||||
CPP,
|
||||
`/*! Bang block, marker BANGMARK. */\nint add(int a) { return a; }`,
|
||||
'function_definition',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('BANGMARK');
|
||||
});
|
||||
});
|
||||
|
||||
describe('leading-doc descriptionExtractor — exported TS/JS decls (issue #2270 review fix)', () => {
|
||||
it('attaches JSDoc to an exported function (JSDoc precedes export_statement)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.TypeScript,
|
||||
TypeScript.typescript,
|
||||
`/** Exported adder, marker EXPFN. */\nexport function add(a: number) { return a; }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('EXPFN');
|
||||
});
|
||||
|
||||
it('attaches JSDoc to an exported class', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.TypeScript,
|
||||
TypeScript.typescript,
|
||||
`/** Exported widget, marker EXPCLASS. */\nexport class Widget { run() {} }`,
|
||||
'class_declaration',
|
||||
'definition.class',
|
||||
'Class',
|
||||
'Widget',
|
||||
);
|
||||
expect(d).toContain('EXPCLASS');
|
||||
});
|
||||
|
||||
it('attaches JSDoc to an export default function', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.TypeScript,
|
||||
TypeScript.typescript,
|
||||
`/** Default export, marker EXPDEFAULT. */\nexport default function add(a: number) { return a; }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('EXPDEFAULT');
|
||||
});
|
||||
|
||||
it('still attaches JSDoc to a bare (non-exported) function', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.TypeScript,
|
||||
TypeScript.typescript,
|
||||
`/** Bare adder, marker BAREFN. */\nfunction add(a: number) { return a; }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'add',
|
||||
);
|
||||
expect(d).toContain('BAREFN');
|
||||
});
|
||||
});
|
||||
|
||||
describe('leading-doc descriptionExtractor — line-comment adjacency (issue #2270 review fix)', () => {
|
||||
it('does NOT attach a // comment separated from the function by a blank line (Go)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\n// Detached note, marker DETACHED.\n\nfunc Add(a int) int { return a }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
|
||||
it('collects only the adjacent // block when an earlier block is blank-separated (Go)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\n// Unrelated earlier block, marker EARLIER.\n\n// Adjacent doc, marker ADJACENT.\nfunc Add(a int) int { return a }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Add',
|
||||
);
|
||||
expect(d).toContain('ADJACENT');
|
||||
expect(d).not.toContain('EARLIER');
|
||||
});
|
||||
|
||||
it('does NOT absorb a Ruby magic comment separated from the first method by a blank line', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Ruby,
|
||||
Ruby,
|
||||
`# frozen_string_literal: true\n\ndef add(a)\n a\nend`,
|
||||
'method',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('leading-doc descriptionExtractor — directive & magic comments (issue #2270 review fix)', () => {
|
||||
it('does NOT absorb a Go //go: build directive directly above a function', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\n//go:build linux\nfunc Add(a int) int { return a }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps a real godoc line and skips an interleaved //go:generate directive', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Go,
|
||||
Go,
|
||||
`package p\n// Add returns the sum, marker GODOC.\n//go:generate stringer -type=T\nfunc Add(a int) int { return a }`,
|
||||
'function_declaration',
|
||||
'definition.function',
|
||||
'Function',
|
||||
'Add',
|
||||
);
|
||||
expect(d).toContain('GODOC');
|
||||
expect(d).not.toContain('go:generate');
|
||||
});
|
||||
|
||||
it('does NOT absorb a Ruby magic comment directly above the first method (no blank line)', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.Ruby,
|
||||
Ruby,
|
||||
`# frozen_string_literal: true\ndef add(a)\n a\nend`,
|
||||
'method',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'add',
|
||||
);
|
||||
expect(d).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('phpDescriptionExtractor — Eloquent metadata wins over PHPDoc fallback', () => {
|
||||
it('returns the Eloquent relation, not the docblock prose, when both are present', () => {
|
||||
const d = describeFromProvider(
|
||||
SupportedLanguages.PHP,
|
||||
PHP.php,
|
||||
`<?php\nclass User {\n/** Prose docblock, marker DOCPROSE. */\npublic function orders() { return $this->hasMany(Order::class); }\n}`,
|
||||
'method_declaration',
|
||||
'definition.method',
|
||||
'Method',
|
||||
'orders',
|
||||
);
|
||||
expect(d).toBe('hasMany(Order)');
|
||||
expect(d).not.toContain('DOCPROSE');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue