diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3b7d06cba..ad7fa8b0e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -303,6 +303,7 @@ Each language implements `LanguageProvider` (`language-provider.ts`). Key fields | `exportChecker` | Public/exported symbol detection | | `typeConfig` | Type annotation extraction rules | | `mroStrategy` | `first-wins` / `c3` / `none` | +| `descriptionExtractor` | Optional hook returning a symbol's doc-comment text as its `description`; feeds the embedding metadata header so doc-only terms are semantically searchable (issue #2270). Most languages register `createLeadingDocDescriptionExtractor` (shared, language-neutral; per-language comment/wrapper config passed at the call site) | 16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index f01233348..5378c2260 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -31,6 +31,7 @@ const FUNCTION_DECLARATION_TYPES = new Set([ 'function_item', ]); import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import type { NodeLabel } from 'gitnexus-shared'; import type { LanguageProvider } from '../language-provider.js'; import { createFieldExtractor } from '../field-extractors/generic.js'; @@ -397,6 +398,8 @@ export const cProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(cVariableConfig), classExtractor: cClassExtractor, + // ── Doxygen doc comment → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), labelOverride: cppLabelOverride, builtInNames: C_BUILT_INS, @@ -482,6 +485,8 @@ export const cppProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(cppVariableConfig), classExtractor: cppClassExtractor, + // ── Doxygen doc comment → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), labelOverride: cppLabelOverride, builtInNames: C_BUILT_INS, extractTemplateConstraints: extractCppTemplateConstraintsForProvider, diff --git a/gitnexus/src/core/ingestion/languages/csharp.ts b/gitnexus/src/core/ingestion/languages/csharp.ts index 612e33fdb..99bca7a6c 100644 --- a/gitnexus/src/core/ingestion/languages/csharp.ts +++ b/gitnexus/src/core/ingestion/languages/csharp.ts @@ -15,6 +15,7 @@ import { csharpExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { csharpImportConfig } from '../import-resolvers/configs/csharp.js'; import { CSHARP_QUERIES } from '../tree-sitter-queries.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; import { createCallExtractor } from '../call-extractors/generic.js'; import { csharpCallConfig } from '../call-extractors/configs/csharp.js'; @@ -194,6 +195,8 @@ export const csharpProvider = defineLanguage({ methodExtractor: createMethodExtractor(csharpMethodConfig), variableExtractor: createVariableExtractor(csharpVariableConfig), classExtractor: createClassExtractor(csharpClassConfig), + // ── XML doc comments (`///`) → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── diff --git a/gitnexus/src/core/ingestion/languages/dart.ts b/gitnexus/src/core/ingestion/languages/dart.ts index b2a71ec81..a95571dc2 100644 --- a/gitnexus/src/core/ingestion/languages/dart.ts +++ b/gitnexus/src/core/ingestion/languages/dart.ts @@ -9,9 +9,12 @@ * The hook resolves the enclosing function by inspecting the previous sibling. */ -import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { + createLeadingDocDescriptionExtractor, + FUNCTION_NODE_TYPES, + type SyntaxNode, +} from '../utils/ast-helpers.js'; import type { NodeLabel } from 'gitnexus-shared'; -import { FUNCTION_NODE_TYPES } from '../utils/ast-helpers.js'; import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { dartClassConfig } from '../class-extractors/configs/dart.js'; @@ -124,6 +127,8 @@ export const dartProvider = defineLanguage({ methodExtractor: createMethodExtractor(dartMethodConfig), variableExtractor: createVariableExtractor(dartVariableConfig), classExtractor: createClassExtractor(dartClassConfig), + // ── Dartdoc (`///`) → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), enclosingFunctionFinder: dartEnclosingFunctionFinder, builtInNames: DART_BUILT_INS, diff --git a/gitnexus/src/core/ingestion/languages/go.ts b/gitnexus/src/core/ingestion/languages/go.ts index 039a04693..e7a25e82f 100644 --- a/gitnexus/src/core/ingestion/languages/go.ts +++ b/gitnexus/src/core/ingestion/languages/go.ts @@ -11,6 +11,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { goClassConfig } from '../class-extractors/configs/go.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { createGoCfgVisitor } from '../cfg/visitors/go.js'; import { defineLanguage } from '../language-provider.js'; import { typeConfig as goConfig } from '../type-extractors/go.js'; @@ -138,6 +139,12 @@ export const goProvider = defineLanguage({ methodExtractor: createMethodExtractor(goMethodConfig), variableExtractor: createVariableExtractor(goVariableConfig), classExtractor: createClassExtractor(goClassConfig), + // ── godoc (`//` leading comments) → description (issue #2270). Build/tool + // directives (//go:…, // +build, //nolint, //line) are not documentation. ── + descriptionExtractor: createLeadingDocDescriptionExtractor({ + lineCommentPrefixes: ['//'], + lineDirectivePrefixes: ['//go:', '// +build', '//nolint', '//line'], + }), builtInNames: GO_BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks ────────── diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index 22b9a48a3..79ba8db05 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -12,6 +12,7 @@ import { createClassExtractor } from '../class-extractors/generic.js'; import { javaClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { javaTypeConfig } from '../type-extractors/jvm.js'; import { extractSpringRoutes } from '../route-extractors/spring.js'; import { javaExportChecker } from '../export-detection.js'; @@ -117,6 +118,9 @@ export const javaProvider = defineLanguage({ variableExtractor: createVariableExtractor(javaVariableConfig), classExtractor: createClassExtractor(javaClassConfig), + // ── Javadoc → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), + // ── RFC #909 Ring 3: scope-based resolution hooks ── emitScopeCaptures: emitJavaScopeCaptures, diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index d4999d022..35c392dc6 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -11,6 +11,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { kotlinClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { assertCloneable } from '../workers/clone-safety.js'; import { kotlinTypeConfig } from '../type-extractors/jvm.js'; import { kotlinExportChecker } from '../export-detection.js'; @@ -170,6 +171,10 @@ export const kotlinProvider = defineLanguage({ variableExtractor: createVariableExtractor(kotlinVariableConfig), classExtractor: createClassExtractor(kotlinClassConfig), builtInNames: BUILT_INS, + + // ── KDoc → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), + labelOverride: (functionNode, defaultLabel) => { if (defaultLabel !== 'Function') return defaultLabel; if (isKotlinClassMethod(functionNode)) return 'Method'; diff --git a/gitnexus/src/core/ingestion/languages/php.ts b/gitnexus/src/core/ingestion/languages/php.ts index c58db55ad..a7d7fa4da 100644 --- a/gitnexus/src/core/ingestion/languages/php.ts +++ b/gitnexus/src/core/ingestion/languages/php.ts @@ -21,13 +21,22 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { phpClassConfig } from '../class-extractors/configs/php.js'; import { createPhpCfgVisitor } from '../cfg/visitors/php.js'; -import { defineLanguage, type AstFrameworkPatternConfig } from '../language-provider.js'; +import { + defineLanguage, + type AstFrameworkPatternConfig, + type CaptureMap, +} from '../language-provider.js'; import { typeConfig as phpConfig } from '../type-extractors/php.js'; import { phpExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { phpImportConfig } from '../import-resolvers/configs/php.js'; import { PHP_QUERIES } from '../tree-sitter-queries.js'; -import { findDescendant, extractStringContent, type SyntaxNode } from '../utils/ast-helpers.js'; +import { + findDescendant, + extractStringContent, + createLeadingDocDescriptionExtractor, + type SyntaxNode, +} from '../utils/ast-helpers.js'; import type { NodeLabel } from 'gitnexus-shared'; import { createFieldExtractor } from '../field-extractors/generic.js'; import { phpConfig as phpFieldConfig } from '../field-extractors/configs/php.js'; @@ -221,22 +230,32 @@ function extractEloquentRelationDescription(methodNode: SyntaxNode): string | nu return null; } +/** PHPDoc-docblock fallback, shared with the other leading-comment languages. */ +const phpLeadingDocFallback = createLeadingDocDescriptionExtractor(); + /** * LanguageProvider.descriptionExtractor implementation for PHP. - * Extracts Eloquent model property metadata and relationship descriptions. + * Eloquent model property metadata and relationship descriptions take + * precedence (they are richer than prose); otherwise documentable symbols fall + * back to their leading PHPDoc docblock (issue #2270), mirroring the other + * leading-comment languages. */ function phpDescriptionExtractor( nodeLabel: NodeLabel, nodeName: string, - captureMap: Record, + captureMap: CaptureMap, ): string | undefined { - if (nodeLabel === 'Property' && captureMap['definition.property']) { - return extractPhpPropertyDescription(nodeName, captureMap['definition.property']) ?? undefined; + const propertyNode = captureMap['definition.property']; + if (nodeLabel === 'Property' && propertyNode) { + const eloquentProperty = extractPhpPropertyDescription(nodeName, propertyNode); + if (eloquentProperty) return eloquentProperty; } - if (nodeLabel === 'Method' && captureMap['definition.method']) { - return extractEloquentRelationDescription(captureMap['definition.method']) ?? undefined; + const methodNode = captureMap['definition.method']; + if (nodeLabel === 'Method' && methodNode) { + const eloquentRelation = extractEloquentRelationDescription(methodNode); + if (eloquentRelation) return eloquentRelation; } - return undefined; + return phpLeadingDocFallback(nodeLabel, nodeName, captureMap); } /** Detect Laravel route files by path convention. */ diff --git a/gitnexus/src/core/ingestion/languages/ruby.ts b/gitnexus/src/core/ingestion/languages/ruby.ts index d8f340472..6b2a4b59d 100644 --- a/gitnexus/src/core/ingestion/languages/ruby.ts +++ b/gitnexus/src/core/ingestion/languages/ruby.ts @@ -13,7 +13,7 @@ import { createClassExtractor } from '../class-extractors/generic.js'; import { rubyClassConfig } from '../class-extractors/configs/ruby.js'; import { defineLanguage } from '../language-provider.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; -import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createLeadingDocDescriptionExtractor, type SyntaxNode } from '../utils/ast-helpers.js'; import { typeConfig as rubyConfig } from '../type-extractors/ruby.js'; import { routeRubyCall } from '../call-routing.js'; import { rubyExportChecker } from '../export-detection.js'; @@ -197,6 +197,20 @@ export const rubyProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(rubyVariableConfig), classExtractor: createClassExtractor(rubyClassConfig), + // ── Leading `#` comments (RDoc/YARD) → description (issue #2270). Magic + // comments and the shebang are not documentation. ── + descriptionExtractor: createLeadingDocDescriptionExtractor({ + lineCommentPrefixes: ['#'], + lineDirectivePrefixes: [ + '# frozen_string_literal:', + '# encoding:', + '# coding:', + '# -*-', + '#!', + '# rubocop:', + '# typed:', + ], + }), labelOverride: rubyLabelOverride, // Ruby MRO is kind-aware: prepend providers beat the class's own method, // which in turn beats include providers. The graph-level MRO phase diff --git a/gitnexus/src/core/ingestion/languages/rust.ts b/gitnexus/src/core/ingestion/languages/rust.ts index a0102c334..842da9e07 100644 --- a/gitnexus/src/core/ingestion/languages/rust.ts +++ b/gitnexus/src/core/ingestion/languages/rust.ts @@ -13,7 +13,7 @@ import type { NodeLabel } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { rustClassConfig } from '../class-extractors/configs/rust.js'; import { defineLanguage } from '../language-provider.js'; -import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createLeadingDocDescriptionExtractor, type SyntaxNode } from '../utils/ast-helpers.js'; import { typeConfig as rustConfig } from '../type-extractors/rust.js'; import { rustExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; @@ -176,6 +176,13 @@ export const rustProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(rustVariableConfig), classExtractor: createClassExtractor(rustClassConfig), + // ── Rust outer doc comments (`///`, `/** */`) → description (issue #2270). + // `//!` / `/*!` are INNER docs (document the enclosing item), so they must + // not attach to the following item — opt out of both. ── + descriptionExtractor: createLeadingDocDescriptionExtractor({ + lineCommentPrefixes: ['///'], + blockDocPrefixes: ['/**'], + }), builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks ────────── emitScopeCaptures: emitRustScopeCaptures, diff --git a/gitnexus/src/core/ingestion/languages/swift.ts b/gitnexus/src/core/ingestion/languages/swift.ts index 02a7b89e8..d19a8fb52 100644 --- a/gitnexus/src/core/ingestion/languages/swift.ts +++ b/gitnexus/src/core/ingestion/languages/swift.ts @@ -17,6 +17,7 @@ import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { swiftImportConfig } from '../import-resolvers/configs/swift.js'; import { SWIFT_QUERIES } from '../tree-sitter-queries.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { createFieldExtractor } from '../field-extractors/generic.js'; import { swiftConfig as swiftFieldConfig } from '../field-extractors/configs/swift.js'; import { createMethodExtractor } from '../method-extractors/generic.js'; @@ -243,6 +244,8 @@ export const swiftProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(swiftVariableConfig), classExtractor: createClassExtractor(swiftClassConfig), + // ── Swift doc comments (`///`, `/** */`) → description (issue #2270) ── + descriptionExtractor: createLeadingDocDescriptionExtractor(), orderSameNameTypeCandidates: orderSwiftSameNameTypeCandidates, builtInNames: BUILT_INS, // ── Scope-based resolution hooks (RFC #909 Ring 3, issue #937). See diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 7f41fe731..2f1f0a783 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -16,6 +16,7 @@ import { javascriptClassConfig, } from '../class-extractors/configs/typescript-javascript.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createLeadingDocDescriptionExtractor } from '../utils/ast-helpers.js'; import { createTypeScriptCfgVisitor } from '../cfg/visitors/typescript.js'; import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js'; import { tsExportChecker } from '../export-detection.js'; @@ -344,6 +345,11 @@ export const typescriptProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(typescriptVariableConfig), classExtractor: createClassExtractor(typescriptClassConfig), + // ── JSDoc → description (issue #2270). An exported decl is captured as the + // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── + descriptionExtractor: createLeadingDocDescriptionExtractor({ + wrapperNodeTypes: ['export_statement'], + }), builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── @@ -406,6 +412,11 @@ export const javascriptProvider = defineLanguage({ }), variableExtractor: createVariableExtractor(javascriptVariableConfig), classExtractor: createClassExtractor(javascriptClassConfig), + // ── JSDoc → description (issue #2270). An exported decl is captured as the + // inner declaration; its JSDoc precedes the wrapping `export_statement`. ── + descriptionExtractor: createLeadingDocDescriptionExtractor({ + wrapperNodeTypes: ['export_statement'], + }), builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── diff --git a/gitnexus/src/core/ingestion/utils/ast-helpers.ts b/gitnexus/src/core/ingestion/utils/ast-helpers.ts index c391d7a0c..77260ffd6 100644 --- a/gitnexus/src/core/ingestion/utils/ast-helpers.ts +++ b/gitnexus/src/core/ingestion/utils/ast-helpers.ts @@ -87,7 +87,7 @@ export const DEFINITION_CAPTURE_KEYS = [ /** Extract the definition node from a tree-sitter query capture map. */ export const getDefinitionNodeFromCaptures = ( - captureMap: Record, + captureMap: Record, ): SyntaxNode | null => { for (const key of DEFINITION_CAPTURE_KEYS) { if (captureMap[key]) return captureMap[key]; @@ -310,7 +310,7 @@ export function findAncestorBeforeBoundary( * Returns null if the capture should be skipped (import, call, C/C++ duplicate, missing name). */ export function getLabelFromCaptures( - captureMap: Record, + captureMap: Record, provider: LanguageProvider, ): NodeLabel | null { if (captureMap['import'] || captureMap['call']) return null; @@ -926,6 +926,227 @@ export function findChild(node: SyntaxNode, type: string): SyntaxNode | null { return null; } +/** Remove bidi-override and zero-width control characters. Doc text is + * attacker-influenced (any indexed repo) and is returned verbatim to MCP + * clients, so strip Trojan-Source-style hidden controls from the description + * before it leaves the extractor (#2286 review). Scoped to the doc-comment path + * only — global `sanitizeUTF8` is intentionally untouched. */ +const stripBidiAndZeroWidth = (text: string): string => + Array.from(text) + .filter((ch) => { + const c = ch.codePointAt(0) ?? 0; + // Bidi overrides/isolates (U+202A–202E, U+2066–2069), zero-width + // space/joiners (U+200B–200D), and BOM/zero-width-no-break (U+FEFF). + return !( + (c >= 0x202a && c <= 0x202e) || + (c >= 0x2066 && c <= 0x2069) || + (c >= 0x200b && c <= 0x200d) || + c === 0xfeff + ); + }) + .join(''); + +/** Normalize a block doc comment body: strip the opening (double-star or + * bang) delimiter, the closing delimiter, and per-line gutter stars, then + * collapse whitespace so tag content stays as searchable words. */ +const normalizeBlockDocComment = (text: string): string | undefined => { + const inner = stripBidiAndZeroWidth( + text + .replace(/^\/\*[*!]/, '') + // Close delimiter: tolerate the degenerate empty comment `/**/`, where the + // opening strip already consumed the shared `*`, leaving a lone `/`. + .replace(/\*?\/\s*$/, '') + .replace(/^[ \t]*\*[ \t]?/gm, ' ') + .replace(/\s+/g, ' ') + .trim(), + ); + return inner.length > 0 ? inner : undefined; +}; + +/** Default line-comment prefixes treated as documentation: the universal + * triple-slash / bang-slash doc markers (Rust, C#, Dart, Swift, Doxygen). + * Go (`//`) and Ruby (`#`) opt into their conventional markers explicitly. */ +const DEFAULT_LINE_DOC_PREFIXES: readonly string[] = ['///', '//!']; + +/** Default block-comment doc openers: Javadoc/JSDoc-style `/**` and Doxygen + * `/*!`. Rust opts out of `/*!` (and `//!`) because those are *inner* docs that + * document the enclosing item, not the following one. */ +const DEFAULT_BLOCK_DOC_PREFIXES: readonly string[] = ['/**', '/*!']; + +/** A file-top `/** … *\/` license/copyright/file-overview block has no + * package/import sibling to shield it, so it would otherwise be absorbed as the + * first declaration's description (PR #2286 review). These markers identify such + * headers; they are specific enough not to fire on an ordinary symbol doc that + * merely mentions the word "copyright". `@file`/`@fileoverview` are explicitly + * file-level JSDoc tags, so a block carrying them is not a symbol doc. */ +const FILE_HEADER_MARKER = + /SPDX-License-Identifier|@licen[sc]e\b|@fileoverview\b|@file\b|Licen[sc]ed under|copyright\s*(\(c\)|©|\d{4})/i; + +/** + * Extract the normalized text of a leading doc comment immediately preceding a + * definition node — covering both block doc comments (Javadoc / KDoc / JSDoc / + * PHPDoc / Doxygen, opened by `/**` or `/*!`) and runs of line doc comments + * (`///`, `//!`, or the caller-supplied prefixes such as Go's `//` or Ruby's + * `#`). Returns `undefined` when there is no preceding doc comment or it is + * empty. + * + * Grammar-agnostic by design: matches on the comment text prefix rather than a + * grammar node type, because the comment node is named differently across + * grammars (`block_comment`, `multiline_comment`, `comment`, `line_comment`). + * Annotations and modifiers live inside the definition node, so the doc comment + * remains the definition's `previousNamedSibling` even on annotated/decorated + * declarations. + * + * Block comments are taken as the immediately-preceding sibling (intervening + * package/import/code siblings already shield a file-level license block from + * the first declaration). Line doc comments enforce row-adjacency: the first + * comment must sit on the line directly above the definition, and each comment + * walked further up must sit directly above the previous one — so a run stops + * at a blank line. This matches godoc/RDoc/rustdoc convention and prevents an + * unrelated comment block (a license header, a Ruby shebang + magic comment) + * separated by a blank line from being absorbed. Adjacency is checked on + * `startPosition.row` (reliable) rather than `endPosition.row`, since some + * grammars fold the trailing newline into the comment node. + * + * Normalization mirrors Python docstring handling: strip the comment delimiters + * / per-line markers, then collapse whitespace to single spaces so tag content + * (`@param`, `@deprecated since 2.0, use computeBalanceV2`) survives. + * + * When the captured definition is an inner node and its own preceding sibling + * carries no doc, the search retries from a wrapping node whose type is listed in + * `opts.wrapperNodeTypes` (e.g. an `export_statement` wrapping an exported + * function/class — the JSDoc precedes the wrapper, not the inner declaration). + */ +export interface LeadingDocCommentOptions { + /** Line-comment doc prefixes (defaults to {@link DEFAULT_LINE_DOC_PREFIXES}; + * Go passes `['//']`, Ruby passes `['#']`). */ + lineCommentPrefixes?: readonly string[]; + /** Grammar node types that wrap a definition such that the doc comment is the + * wrapper's preceding sibling rather than the definition's. TS/JS pass + * `['export_statement']`. Empty by default → no wrapper retry. */ + wrapperNodeTypes?: readonly string[]; + /** Line-comment prefixes that are tool/build directives or magic comments + * rather than documentation (Go passes `['//go:', '// +build', …]`, Ruby + * passes `['# frozen_string_literal:', '#!', …]`). A matching line is skipped + * in the doc run rather than absorbed. Empty by default. */ + lineDirectivePrefixes?: readonly string[]; + /** Block-comment doc openers (defaults to `['/**', '/*!']`). Rust passes + * `['/**']` so its inner-doc `/*!` does not attach to the following item. */ + blockDocPrefixes?: readonly string[]; +} + +export function extractLeadingDocComment( + node: SyntaxNode, + opts: LeadingDocCommentOptions = {}, +): string | undefined { + const lineCommentPrefixes = opts.lineCommentPrefixes ?? DEFAULT_LINE_DOC_PREFIXES; + const wrapperNodeTypes = opts.wrapperNodeTypes ?? []; + const lineDirectivePrefixes = opts.lineDirectivePrefixes ?? []; + const blockDocPrefixes = opts.blockDocPrefixes ?? DEFAULT_BLOCK_DOC_PREFIXES; + + const fromNode = (anchor: SyntaxNode): string | undefined => { + const prev = anchor.previousNamedSibling; + if (!prev) return undefined; + + // Block doc comment: /** ... */ or /*! ... */ + if (blockDocPrefixes.some((p) => prev.text.startsWith(p))) { + // Skip a file-top license/copyright/overview header (no package/import + // sibling shields it from the first declaration). A strict row-adjacency + // check is unreliable here — some grammars fold the trailing newline into + // the comment node — so match header markers instead. + if (FILE_HEADER_MARKER.test(prev.text)) return undefined; + return normalizeBlockDocComment(prev.text); + } + + // Run of row-adjacent preceding line doc comments (e.g. `///` or `//`). + const matchedPrefix = (text: string): string | undefined => + lineCommentPrefixes.find((prefix) => text.trimStart().startsWith(prefix)); + const isDirective = (text: string): boolean => + lineDirectivePrefixes.some((prefix) => text.trimStart().startsWith(prefix)); + + const lines: string[] = []; + let current: SyntaxNode | null = prev; + let expectedRow = anchor.startPosition.row - 1; + while (current) { + const text = current.text; + const prefix = matchedPrefix(text); + if (prefix === undefined || current.startPosition.row !== expectedRow) break; + // A build/tool directive or magic comment (e.g. `//go:build`, + // `# frozen_string_literal:`) is not documentation: skip it but keep + // walking the adjacent run, so a real doc above it is still collected. + if (!isDirective(text)) lines.unshift(text.trimStart().slice(prefix.length)); + expectedRow = current.startPosition.row - 1; + current = current.previousNamedSibling; + } + + const joined = stripBidiAndZeroWidth(lines.join(' ').replace(/\s+/g, ' ').trim()); + return joined.length > 0 ? joined : undefined; + }; + + const direct = fromNode(node); + if (direct !== undefined) return direct; + + const parent = node.parent; + if (parent && wrapperNodeTypes.includes(parent.type)) { + return fromNode(parent); + } + return undefined; +} + +/** Node labels that can carry a leading doc comment — callables and type-like + * declarations. Field/property/variable/const doc is intentionally excluded + * (issue #2270 scopes this to method/type documentation). Language-neutral: + * a label a given grammar never emits simply never matches. + * + * Bounded to labels that are also in `embeddings/types.ts` `EMBEDDABLE_LABELS`: + * the description is only useful once it reaches the embedding metadata header, + * and the embedding pipeline only queries embeddable labels. Extracting docs + * for a non-embeddable label is a wasted write that never becomes searchable. + * A subset invariant in the unit tests guards against drift. Making currently- + * non-embeddable doc-bearing labels (Module, Delegate, Annotation, and C++ + * `Template`) searchable is tracked as a follow-up — it needs an embedding- + * pipeline/schema change beyond this fix. */ +export const DOC_BEARING_LABELS: ReadonlySet = new Set([ + 'Function', + 'Method', + 'Constructor', + 'Class', + 'Interface', + 'Enum', + 'Struct', + 'Trait', + 'Record', + 'Union', + 'Namespace', + 'TypeAlias', + 'Macro', +]); + +/** + * Build a `LanguageProvider.descriptionExtractor` that surfaces a definition's + * leading doc comment as its `description` (issue #2270). For labels in + * {@link DOC_BEARING_LABELS} (which is bounded to embeddable labels) the text + * then reaches the embedding metadata header and becomes semantically searchable. + * + * Language-neutral factory (names no language): guards on + * {@link DOC_BEARING_LABELS}; callers pass per-language doc-comment behavior via + * {@link LeadingDocCommentOptions} (line prefixes, export-style wrappers, …) + * which is threaded straight through to {@link extractLeadingDocComment}. + */ +export const createLeadingDocDescriptionExtractor = ( + opts: LeadingDocCommentOptions = {}, +): (( + nodeLabel: NodeLabel, + nodeName: string, + captureMap: Record, +) => string | undefined) => { + return (nodeLabel, _nodeName, captureMap) => { + if (!DOC_BEARING_LABELS.has(nodeLabel)) return undefined; + const definitionNode = getDefinitionNodeFromCaptures(captureMap); + return definitionNode ? extractLeadingDocComment(definitionNode, opts) : undefined; + }; +}; + // ============================================================================ // Capture + range helpers (formerly python/ast-utils.ts — language-agnostic) // ============================================================================ diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index ee5c59a9f..a67e37c14 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -2151,7 +2151,20 @@ const processFileGroup = ( `${file.path}:${qualifiedName}${classTemplateTag}${arityTag}${parameterShapeTag}${constraintsTag}`, ); - const description = provider.descriptionExtractor?.(nodeLabel, nodeName, captureMap); + let description: string | undefined; + try { + description = provider.descriptionExtractor?.(nodeLabel, nodeName, captureMap); + } catch (err) { + // A throw here (an unexpected tree-sitter node shape, a provider bug) must + // NOT propagate — it would escape processFileGroup to the language-group + // catch, which treats any throw as "parser unavailable" and silently drops + // every remaining file in the group. Mirrors the extractTemplateConstraints + // guard above (#2286 review). + reportWarning( + `Description extraction failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`, + ); + description = undefined; + } let frameworkHint = definitionNode ? detectFrameworkFromAST(language, (definitionNode.text || '').slice(0, 300)) diff --git a/gitnexus/test/integration/doc-comment-description-e2e.test.ts b/gitnexus/test/integration/doc-comment-description-e2e.test.ts new file mode 100644 index 000000000..41911be67 --- /dev/null +++ b/gitnexus/test/integration/doc-comment-description-e2e.test.ts @@ -0,0 +1,51 @@ +/** + * End-to-end coverage for issue #2270: a leading doc comment must survive the + * full parse pipeline into the node's `description` property (the field the + * embedding metadata header reads for semantic search). The unit tests stop at + * the `descriptionExtractor` hook; this exercises the real worker pipeline and + * the highest-value case — an EXPORTED TS function, whose JSDoc precedes the + * wrapping `export_statement` (PR #2286 review fix). + */ +import { describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { runPipelineFromRepo } from './resolvers/helpers.js'; +import type { PipelineResult } from '../../src/types/pipeline.js'; + +function createTsRepo(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'doc-desc-e2e-')); + fs.writeFileSync( + path.join(dir, 'package.json'), + JSON.stringify({ name: 'doc-desc-e2e', version: '1.0.0' }), + ); + fs.writeFileSync( + path.join(dir, 'index.ts'), + [ + '/** Computes the running balance, marker EXPORTEDDOC. */', + 'export function computeBalance(userId: number): number {', + ' return userId;', + '}', + '', + ].join('\n'), + ); + return dir; +} + +describe('doc-comment description end-to-end (issue #2270)', () => { + it('surfaces an exported function JSDoc as its node description through the pipeline', async () => { + const result: PipelineResult = await runPipelineFromRepo(createTsRepo(), () => {}, { + skipGraphPhases: true, + workerThresholdsForTest: { minFiles: 1, minBytes: 1 }, + workerPoolSize: 2, + }); + + const descriptions = new Map(); + result.graph.forEachNode((node) => { + descriptions.set(`${node.label}:${node.properties.name}`, node.properties.description); + }); + + expect(result.usedWorkerPool).toBe(true); + expect(descriptions.get('Function:computeBalance')).toContain('EXPORTEDDOC'); + }); +}); diff --git a/gitnexus/test/unit/java-description-extractor.test.ts b/gitnexus/test/unit/java-description-extractor.test.ts new file mode 100644 index 000000000..dcd64a920 --- /dev/null +++ b/gitnexus/test/unit/java-description-extractor.test.ts @@ -0,0 +1,70 @@ +/** + * Unit tests for `javaProvider.descriptionExtractor` (issue #2270, U2). + * + * Confirms Java method/type Javadoc is surfaced as the symbol `description`, + * which is what reaches the embedding metadata header and makes Javadoc-only + * terms semantically searchable. Drives the real provider hook with a captureMap + * built from a parsed tree (matching what the parse worker passes in). + */ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { javaProvider } from '../../src/core/ingestion/languages/java.js'; +import type { CaptureMap } from '../../src/core/ingestion/language-provider.js'; + +function captureMapFor(src: string, nodeType: string, captureKey: string): CaptureMap { + const parser = new Parser(); + parser.setLanguage(Java); + const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0]; + expect(node, `expected a ${nodeType} node`).toBeDefined(); + return { [captureKey]: node }; +} + +const PROBE = `package demo; +public class Probe { + /** + * Computes the running balance across all user accounts. + * @param userId the unique user identifier + * @deprecated since 2.0, use computeBalanceV2 + */ + public java.math.BigDecimal computeBalance(Long userId) { return null; } +}`; + +describe('javaProvider.descriptionExtractor', () => { + it('is registered on the provider (regression guard for issue #2270)', () => { + expect(javaProvider.descriptionExtractor).toBeDefined(); + }); + + it('extracts the method Javadoc, including the @deprecated marker term', () => { + const captureMap = captureMapFor(PROBE, 'method_declaration', 'definition.method'); + const description = javaProvider.descriptionExtractor?.('Method', 'computeBalance', captureMap); + expect(description).toContain('Computes the running balance'); + expect(description).toContain('computeBalanceV2'); + }); + + it('extracts class-level Javadoc', () => { + const captureMap = captureMapFor( + `/**\n * A probe class.\n */\npublic class Probe {}`, + 'class_declaration', + 'definition.class', + ); + expect(javaProvider.descriptionExtractor?.('Class', 'Probe', captureMap)).toBe( + 'A probe class.', + ); + }); + + it('returns undefined for a Java method with no Javadoc', () => { + const captureMap = captureMapFor( + `class P { void m() {} }`, + 'method_declaration', + 'definition.method', + ); + expect(javaProvider.descriptionExtractor?.('Method', 'm', captureMap)).toBeUndefined(); + }); + + it('returns undefined for a non-doc-bearing label (e.g. Variable)', () => { + // Even with a definition node present, a Variable label is out of scope. + const captureMap = captureMapFor(PROBE, 'method_declaration', 'definition.method'); + expect(javaProvider.descriptionExtractor?.('Variable', 'x', captureMap)).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/kotlin-description-extractor.test.ts b/gitnexus/test/unit/kotlin-description-extractor.test.ts new file mode 100644 index 000000000..b2d96d6ae --- /dev/null +++ b/gitnexus/test/unit/kotlin-description-extractor.test.ts @@ -0,0 +1,68 @@ +/** + * Unit tests for `kotlinProvider.descriptionExtractor` (issue #2270, U3). + * + * Confirms Kotlin function/type KDoc is surfaced as the symbol `description`, + * mirroring the Java behavior. Drives the real provider hook with a captureMap + * built from a parsed tree. + */ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js'; +import { kotlinProvider } from '../../src/core/ingestion/languages/kotlin.js'; +import type { CaptureMap } from '../../src/core/ingestion/language-provider.js'; + +// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111). +const Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); + +function captureMapFor(src: string, nodeType: string, captureKey: string): CaptureMap { + const parser = new Parser(); + parser.setLanguage(Kotlin); + const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0]; + expect(node, `expected a ${nodeType} node`).toBeDefined(); + return { [captureKey]: node }; +} + +const PROBE = `package demo +class Probe { + /** + * Computes the running balance, use computeBalanceV2 + */ + fun computeBalance(userId: Long): String? { return null } +}`; + +describe('kotlinProvider.descriptionExtractor', () => { + it('is registered on the provider (regression guard for issue #2270)', () => { + expect(kotlinProvider.descriptionExtractor).toBeDefined(); + }); + + it('extracts the function KDoc, including the marker term', () => { + const captureMap = captureMapFor(PROBE, 'function_declaration', 'definition.method'); + const description = kotlinProvider.descriptionExtractor?.( + 'Method', + 'computeBalance', + captureMap, + ); + expect(description).toContain('Computes the running balance'); + expect(description).toContain('computeBalanceV2'); + }); + + it('extracts class-level KDoc', () => { + const captureMap = captureMapFor( + `/**\n * A probe class.\n */\nclass Probe`, + 'class_declaration', + 'definition.class', + ); + expect(kotlinProvider.descriptionExtractor?.('Class', 'Probe', captureMap)).toBe( + 'A probe class.', + ); + }); + + it('returns undefined for a Kotlin function with no KDoc', () => { + const captureMap = captureMapFor( + `fun m(): String? { return null }`, + 'function_declaration', + 'definition.method', + ); + expect(kotlinProvider.descriptionExtractor?.('Function', 'm', captureMap)).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/leading-doc-comment.test.ts b/gitnexus/test/unit/leading-doc-comment.test.ts new file mode 100644 index 000000000..40fc44933 --- /dev/null +++ b/gitnexus/test/unit/leading-doc-comment.test.ts @@ -0,0 +1,167 @@ +/** + * Unit tests for `extractLeadingDocComment` (issue #2270, U1). + * + * Verifies the shared helper that pulls a `/** ... *\/` leading doc comment + * (Javadoc / KDoc) off the definition node's preceding named sibling. The + * helper is grammar-agnostic: it matches on the `/**` text prefix, so it works + * for both tree-sitter-java (`block_comment`) and tree-sitter-kotlin + * (`multiline_comment`). + */ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js'; +import { + extractLeadingDocComment, + type SyntaxNode, +} from '../../src/core/ingestion/utils/ast-helpers.js'; + +// Vendored grammar — loaded from vendor/ by absolute path, never node_modules (#2111). +const Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); + +function firstNode(language: unknown, src: string, type: string): SyntaxNode { + const parser = new Parser(); + parser.setLanguage(language); + const node = parser.parse(src).rootNode.descendantsOfType(type)[0]; + expect(node, `expected a ${type} node in source`).toBeDefined(); + return node; +} + +describe('extractLeadingDocComment', () => { + it('extracts a multi-line Javadoc including tag content (issue #2270 repro)', () => { + const src = `package demo; +public class Probe { + /** + * Computes the running balance across all user accounts. + * @param userId the unique user identifier + * @deprecated since 2.0, use computeBalanceV2 + */ + public java.math.BigDecimal computeBalance(Long userId) { return null; } +}`; + const method = firstNode(Java, src, 'method_declaration'); + const doc = extractLeadingDocComment(method); + expect(doc).toContain('Computes the running balance'); + expect(doc).toContain('userId'); + expect(doc).toContain('computeBalanceV2'); + }); + + it('extracts a class-level Javadoc', () => { + const cls = firstNode( + Java, + `/**\n * A probe class.\n */\npublic class Probe {}`, + 'class_declaration', + ); + expect(extractLeadingDocComment(cls)).toBe('A probe class.'); + }); + + it('returns undefined when there is no preceding comment', () => { + const method = firstNode(Java, `class P { void m() {} }`, 'method_declaration'); + expect(extractLeadingDocComment(method)).toBeUndefined(); + }); + + it('returns undefined for a non-doc block comment (license header style)', () => { + const method = firstNode( + Java, + `class P {\n/* not a doc comment */\nvoid m() {}\n}`, + 'method_declaration', + ); + expect(extractLeadingDocComment(method)).toBeUndefined(); + }); + + it('returns undefined for a // line comment', () => { + const method = firstNode( + Java, + `class P {\n// just a line comment\nvoid m() {}\n}`, + 'method_declaration', + ); + expect(extractLeadingDocComment(method)).toBeUndefined(); + }); + + it('returns undefined for an empty doc comment', () => { + const method = firstNode(Java, `class P {\n/** */\nvoid m() {}\n}`, 'method_declaration'); + expect(extractLeadingDocComment(method)).toBeUndefined(); + }); + + it('returns undefined for the degenerate empty comment /**/ (no spurious slash)', () => { + const method = firstNode(Java, `class P {\n/**/\nvoid m() {}\n}`, 'method_declaration'); + expect(extractLeadingDocComment(method)).toBeUndefined(); + }); + + it('strips the */ delimiters and per-line * gutter markers', () => { + const cls = firstNode( + Java, + `/**\n * Line one.\n * Line two.\n */\nclass P {}`, + 'class_declaration', + ); + const doc = extractLeadingDocComment(cls); + expect(doc).toBe('Line one. Line two.'); + expect(doc).not.toContain('*'); + expect(doc).not.toContain('/'); + }); + + it('skips a file-top SPDX license header (no package/import shield)', () => { + const cls = firstNode( + Java, + `/** SPDX-License-Identifier: MIT */\npublic class Foo {}`, + 'class_declaration', + ); + expect(extractLeadingDocComment(cls)).toBeUndefined(); + }); + + it('skips a file-top copyright header block', () => { + const cls = firstNode( + Java, + `/**\n * Copyright (c) 2026 Acme Corp. All rights reserved.\n * Licensed under the Apache License 2.0.\n */\npublic class Foo {}`, + 'class_declaration', + ); + expect(extractLeadingDocComment(cls)).toBeUndefined(); + }); + + it('does NOT over-fire: a real doc that merely mentions copyright is preserved', () => { + const method = firstNode( + Java, + `class P {\n/** Returns the copyright owner name, marker KEEPME. */\nString owner() { return null; }\n}`, + 'method_declaration', + ); + const doc = extractLeadingDocComment(method); + expect(doc).toContain('KEEPME'); + expect(doc).toContain('copyright owner'); + }); + + it('strips bidi-override and zero-width controls from the description', () => { + const rlo = String.fromCharCode(0x202e); // right-to-left override + const zwsp = String.fromCharCode(0x200b); // zero-width space + const cls = firstNode( + Java, + `/** Doc ${rlo}with${zwsp} hidden controls, marker BIDIMARK. */\npublic class Foo {}`, + 'class_declaration', + ); + const doc = extractLeadingDocComment(cls); + expect(doc).toContain('BIDIMARK'); + expect(doc).not.toContain(rlo); + expect(doc).not.toContain(zwsp); + }); + + it('leaves a plain ASCII doc comment unchanged', () => { + const cls = firstNode( + Java, + `/** Plain doc, marker ASCIIMARK. */\nclass Foo {}`, + 'class_declaration', + ); + expect(extractLeadingDocComment(cls)).toBe('Plain doc, marker ASCIIMARK.'); + }); + + it('extracts a Kotlin KDoc (grammar-agnostic prefix match, multiline_comment)', () => { + const src = `package demo +class Probe { + /** + * Computes the running balance, use computeBalanceV2 + */ + fun computeBalance(userId: Long): String? { return null } +}`; + const fn = firstNode(Kotlin, src, 'function_declaration'); + const doc = extractLeadingDocComment(fn); + expect(doc).toContain('Computes the running balance'); + expect(doc).toContain('computeBalanceV2'); + }); +}); diff --git a/gitnexus/test/unit/leading-doc-description-all-languages.test.ts b/gitnexus/test/unit/leading-doc-description-all-languages.test.ts new file mode 100644 index 000000000..8a2165350 --- /dev/null +++ b/gitnexus/test/unit/leading-doc-description-all-languages.test.ts @@ -0,0 +1,391 @@ +/** + * Cross-language coverage for the leading-doc `descriptionExtractor` + * (issue #2270). Confirms every documentable language registers the hook, and + * exercises each doc-comment family through the real provider hooks: + * - block doc comments (double-star / bang): TypeScript, C++ (Java/Kotlin elsewhere) + * - triple-slash line runs: Rust, C# + * - godoc double-slash runs: Go + * - hash runs: Ruby + * - PHPDoc docblock fallback: PHP + * + * Dart and Swift share the default block / triple-slash config already exercised + * their native grammars can fail to load in some environments, so they are + * covered by the registration check (no parse) rather than a behavior parse. + */ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import Go from 'tree-sitter-go'; +import Rust from 'tree-sitter-rust'; +import CSharp from 'tree-sitter-c-sharp'; +import Ruby from 'tree-sitter-ruby'; +import CPP from 'tree-sitter-cpp'; +import PHP from 'tree-sitter-php'; +import { SupportedLanguages } from '../../src/config/supported-languages.js'; +import { getProvider } from '../../src/core/ingestion/languages/index.js'; +import { DOC_BEARING_LABELS } from '../../src/core/ingestion/utils/ast-helpers.js'; +import { EMBEDDABLE_LABELS } from '../../src/core/embeddings/types.js'; +import type { CaptureMap } from '../../src/core/ingestion/language-provider.js'; +import type { NodeLabel } from 'gitnexus-shared'; + +function describeFromProvider( + language: SupportedLanguages, + grammar: unknown, + src: string, + nodeType: string, + captureKey: string, + label: NodeLabel, + name: string, +): string | undefined { + const parser = new Parser(); + parser.setLanguage(grammar); + const node = parser.parse(src).rootNode.descendantsOfType(nodeType)[0]; + expect(node, `expected a ${nodeType} node for ${language}`).toBeDefined(); + const captureMap: CaptureMap = { [captureKey]: node }; + return getProvider(language).descriptionExtractor?.(label, name, captureMap); +} + +// Languages that should surface a description (everything except Vue/Cobol). +const DOCUMENTABLE_LANGUAGES: readonly SupportedLanguages[] = [ + SupportedLanguages.JavaScript, + SupportedLanguages.TypeScript, + SupportedLanguages.Python, + SupportedLanguages.Java, + SupportedLanguages.C, + SupportedLanguages.CPlusPlus, + SupportedLanguages.CSharp, + SupportedLanguages.Go, + SupportedLanguages.Ruby, + SupportedLanguages.Rust, + SupportedLanguages.PHP, + SupportedLanguages.Kotlin, + SupportedLanguages.Swift, + SupportedLanguages.Dart, +]; + +describe('DOC_BEARING_LABELS is bounded to embeddable labels (issue #2270 review fix)', () => { + it('every doc-bearing label is in EMBEDDABLE_LABELS so its description is searchable', () => { + const embeddable = new Set(EMBEDDABLE_LABELS); + const notEmbeddable = [...DOC_BEARING_LABELS].filter((label) => !embeddable.has(label)); + expect(notEmbeddable).toEqual([]); + }); +}); + +describe('leading-doc descriptionExtractor — registration coverage', () => { + it.each(DOCUMENTABLE_LANGUAGES)('%s provider registers descriptionExtractor', (language) => { + expect(getProvider(language).descriptionExtractor).toBeDefined(); + }); +}); + +describe('leading-doc descriptionExtractor — behavior per comment family', () => { + it('TypeScript JSDoc block', () => { + const d = describeFromProvider( + SupportedLanguages.TypeScript, + TypeScript.typescript, + `/** Adds two numbers, marker TSMARK. */\nfunction add(a: number, b: number) { return a + b; }`, + 'function_declaration', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('TSMARK'); + }); + + it('Go godoc (// run)', () => { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\n// Add returns the sum, marker GOMARK.\nfunc Add(a int) int { return a }`, + 'function_declaration', + 'definition.function', + 'Function', + 'Add', + ); + expect(d).toContain('GOMARK'); + }); + + it('Rust /// doc comment', () => { + const d = describeFromProvider( + SupportedLanguages.Rust, + Rust, + `/// Adds things, marker RSMARK.\nfn add(a: i32) -> i32 { a }`, + 'function_item', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('RSMARK'); + }); + + it('C# /// XML doc comment', () => { + const d = describeFromProvider( + SupportedLanguages.CSharp, + CSharp, + `class C {\n/// Adds, marker CSMARK\nvoid M() {}\n}`, + 'method_declaration', + 'definition.method', + 'Method', + 'M', + ); + expect(d).toContain('CSMARK'); + }); + + it('Ruby # comment run', () => { + const d = describeFromProvider( + SupportedLanguages.Ruby, + Ruby, + `# Adds things, marker RBMARK.\ndef add(a)\n a\nend`, + 'method', + 'definition.method', + 'Method', + 'add', + ); + expect(d).toContain('RBMARK'); + }); + + it('C++ Doxygen block', () => { + const d = describeFromProvider( + SupportedLanguages.CPlusPlus, + CPP, + `/** Adds, marker CPPMARK. */\nint add(int a) { return a; }`, + 'function_definition', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('CPPMARK'); + }); + + it('PHP docblock fallback (non-Eloquent prose)', () => { + const d = describeFromProvider( + SupportedLanguages.PHP, + PHP.php, + ` { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\nfunc Bare() {}`, + 'function_declaration', + 'definition.function', + 'Function', + 'Bare', + ); + expect(d).toBeUndefined(); + }); + + it('collects a multi-line /// run (Rust)', () => { + const d = describeFromProvider( + SupportedLanguages.Rust, + Rust, + `/// First line.\n/// Second line, marker MULTILINE.\nfn add(a: i32) -> i32 { a }`, + 'function_item', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('First line'); + expect(d).toContain('MULTILINE'); + }); + + it('does NOT attach a Rust //! inner doc to the following item (inner-doc semantics)', () => { + const d = describeFromProvider( + SupportedLanguages.Rust, + Rust, + `//! Inner doc, marker INNERMARK.\nfn add(a: i32) -> i32 { a }`, + 'function_item', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toBeUndefined(); + }); + + it('does NOT attach a Rust /*! inner block doc to the following item', () => { + const d = describeFromProvider( + SupportedLanguages.Rust, + Rust, + `/*! Inner block doc, marker INNERBLOCK. */\nfn add(a: i32) -> i32 { a }`, + 'function_item', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toBeUndefined(); + }); + + it('collects a /*! Doxygen bang block (C++)', () => { + const d = describeFromProvider( + SupportedLanguages.CPlusPlus, + CPP, + `/*! Bang block, marker BANGMARK. */\nint add(int a) { return a; }`, + 'function_definition', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('BANGMARK'); + }); +}); + +describe('leading-doc descriptionExtractor — exported TS/JS decls (issue #2270 review fix)', () => { + it('attaches JSDoc to an exported function (JSDoc precedes export_statement)', () => { + const d = describeFromProvider( + SupportedLanguages.TypeScript, + TypeScript.typescript, + `/** Exported adder, marker EXPFN. */\nexport function add(a: number) { return a; }`, + 'function_declaration', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('EXPFN'); + }); + + it('attaches JSDoc to an exported class', () => { + const d = describeFromProvider( + SupportedLanguages.TypeScript, + TypeScript.typescript, + `/** Exported widget, marker EXPCLASS. */\nexport class Widget { run() {} }`, + 'class_declaration', + 'definition.class', + 'Class', + 'Widget', + ); + expect(d).toContain('EXPCLASS'); + }); + + it('attaches JSDoc to an export default function', () => { + const d = describeFromProvider( + SupportedLanguages.TypeScript, + TypeScript.typescript, + `/** Default export, marker EXPDEFAULT. */\nexport default function add(a: number) { return a; }`, + 'function_declaration', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('EXPDEFAULT'); + }); + + it('still attaches JSDoc to a bare (non-exported) function', () => { + const d = describeFromProvider( + SupportedLanguages.TypeScript, + TypeScript.typescript, + `/** Bare adder, marker BAREFN. */\nfunction add(a: number) { return a; }`, + 'function_declaration', + 'definition.function', + 'Function', + 'add', + ); + expect(d).toContain('BAREFN'); + }); +}); + +describe('leading-doc descriptionExtractor — line-comment adjacency (issue #2270 review fix)', () => { + it('does NOT attach a // comment separated from the function by a blank line (Go)', () => { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\n// Detached note, marker DETACHED.\n\nfunc Add(a int) int { return a }`, + 'function_declaration', + 'definition.function', + 'Function', + 'Add', + ); + expect(d).toBeUndefined(); + }); + + it('collects only the adjacent // block when an earlier block is blank-separated (Go)', () => { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\n// Unrelated earlier block, marker EARLIER.\n\n// Adjacent doc, marker ADJACENT.\nfunc Add(a int) int { return a }`, + 'function_declaration', + 'definition.function', + 'Function', + 'Add', + ); + expect(d).toContain('ADJACENT'); + expect(d).not.toContain('EARLIER'); + }); + + it('does NOT absorb a Ruby magic comment separated from the first method by a blank line', () => { + const d = describeFromProvider( + SupportedLanguages.Ruby, + Ruby, + `# frozen_string_literal: true\n\ndef add(a)\n a\nend`, + 'method', + 'definition.method', + 'Method', + 'add', + ); + expect(d).toBeUndefined(); + }); +}); + +describe('leading-doc descriptionExtractor — directive & magic comments (issue #2270 review fix)', () => { + it('does NOT absorb a Go //go: build directive directly above a function', () => { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\n//go:build linux\nfunc Add(a int) int { return a }`, + 'function_declaration', + 'definition.function', + 'Function', + 'Add', + ); + expect(d).toBeUndefined(); + }); + + it('keeps a real godoc line and skips an interleaved //go:generate directive', () => { + const d = describeFromProvider( + SupportedLanguages.Go, + Go, + `package p\n// Add returns the sum, marker GODOC.\n//go:generate stringer -type=T\nfunc Add(a int) int { return a }`, + 'function_declaration', + 'definition.function', + 'Function', + 'Add', + ); + expect(d).toContain('GODOC'); + expect(d).not.toContain('go:generate'); + }); + + it('does NOT absorb a Ruby magic comment directly above the first method (no blank line)', () => { + const d = describeFromProvider( + SupportedLanguages.Ruby, + Ruby, + `# frozen_string_literal: true\ndef add(a)\n a\nend`, + 'method', + 'definition.method', + 'Method', + 'add', + ); + expect(d).toBeUndefined(); + }); +}); + +describe('phpDescriptionExtractor — Eloquent metadata wins over PHPDoc fallback', () => { + it('returns the Eloquent relation, not the docblock prose, when both are present', () => { + const d = describeFromProvider( + SupportedLanguages.PHP, + PHP.php, + `hasMany(Order::class); }\n}`, + 'method_declaration', + 'definition.method', + 'Method', + 'orders', + ); + expect(d).toBe('hasMany(Order)'); + expect(d).not.toContain('DOCPROSE'); + }); +});