From 640d326a0158255809a72c324ed4e6ea95d17d7a Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sun, 14 Jun 2026 12:31:59 +0000 Subject: [PATCH] feat(cfg): Python CFG visitor + def/use harvest (#2195 U8) Add createPythonCfgVisitor + python-harvest -- the most structurally divergent target (indentation blocks, elif, for/while-else, with, try/ except/except-group/else/finally, match/case, comprehensions, walrus), confirming the shared CfgBuilder/ControlFlowContext core carries no brace-family assumptions. for/while else-clause sits on the normal- completion edge (not break); with modeled as try/finally dispose; match has no fallthrough. Wire into pythonProvider. Every literal validated against tree-sitter-python via the probe. while True: keeps EXIT reverse-reachable (production CDG probe: 3 edges; fixture: 42 CDG edges). 37 real-parser tests; gate green; no regression (cfg unit 391, tsc clean). Gaps: async/generator suspension, comprehension scope over-approximation. No sites[] (taint substrate, separate). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ingestion/cfg/visitors/python-harvest.ts | 713 +++++++++++++++++ .../src/core/ingestion/cfg/visitors/python.ts | 755 ++++++++++++++++++ .../src/core/ingestion/languages/python.ts | 2 + .../cfg/fixtures/python-hazards.py | 125 +++ gitnexus/test/unit/cfg/python-visitor.test.ts | 410 ++++++++++ 5 files changed, 2005 insertions(+) create mode 100644 gitnexus/src/core/ingestion/cfg/visitors/python-harvest.ts create mode 100644 gitnexus/src/core/ingestion/cfg/visitors/python.ts create mode 100644 gitnexus/test/integration/cfg/fixtures/python-hazards.py create mode 100644 gitnexus/test/unit/cfg/python-visitor.test.ts diff --git a/gitnexus/src/core/ingestion/cfg/visitors/python-harvest.ts b/gitnexus/src/core/ingestion/cfg/visitors/python-harvest.ts new file mode 100644 index 000000000..c5dd39e1f --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/python-harvest.ts @@ -0,0 +1,713 @@ +/** + * Python def/use harvester — the Python analogue of + * {@link import('./typescript-harvest.js').TsHarvester} and the C-family + * harvesters ({@link import('./go-harvest.js').GoHarvester} et al.). Python is + * the most structurally divergent CFG target (indentation blocks, no braces, + * comprehensions, `with`, `try/except/else/finally`, `match/case`), so this + * harvester exercises the shared reaching-defs / CDG substrate against a grammar + * with none of the brace-family assumptions. + * + * Runs in the parse worker next to the Python CFG visitor, extracting + * per-statement variable definition/use facts that ride the side channel for the + * reaching-defs / CDG solvers. Output is the per-function binding table + * ({@link BindingEntry}[]) plus {@link StatementFacts} the visitor attaches to + * blocks as it walks. NO `sites[]` are harvested here — the call-site taint + * substrate is a later step (this unit emits only bindings + defs/uses + mayDefs + * via the local {@link FactAccumulator}, which has no site machinery at all). + * + * Every node type and field literal below was grammar-validated against + * tree-sitter-python (0.23.x) via the introspection probe before use (mandatory + * pre-step). Python shapes pre-empted (verified by a real parse): + * - functions: `function_definition` (fields `name`/`parameters`/`body`; async + * is the SAME node with an `async` token child) and `lambda` (fields + * `parameters`=`lambda_parameters`, `body`). + * - parameters: bare `identifier`, `default_parameter` (fields `name`/`value`), + * `typed_parameter` (no `name` field — the binder is a named child: + * `identifier` / `list_splat_pattern` / `dictionary_splat_pattern`), + * `typed_default_parameter` (fields `name`/`type`/`value`), + * `list_splat_pattern` (`*args`), `dictionary_splat_pattern` (`**kwargs`). + * - assignment targets: `assignment` (fields `left`/`right`/optional `type`; + * LHS may be `identifier`, `pattern_list`, `tuple_pattern`, `list_pattern`, + * `attribute`, or `subscript`), `augmented_assignment` (fields + * `left`/`operator`/`right` — read+write), `named_expression` walrus (fields + * `name`/`value`). + * - unpacking patterns: `pattern_list` / `tuple_pattern` / `list_pattern` + * nest `identifier` and `list_splat_pattern` (`*rest`) targets. + * - binders: `for_statement` (fields `left`/`right`), `for_in_clause` + * (comprehension binder, fields `left`/`right`), `with_item` (field `value` + * = `as_pattern` whose `alias`=`as_pattern_target`, or a bare expression), + * `except_clause` / `except_group_clause` (`as_pattern` → `as_pattern_target`), + * `global_statement` / `nonlocal_statement` (identifier children). + * - reads: `attribute` (fields `object`/`attribute`), `subscript` (fields + * `value`/`subscript`), `call` (fields `function`/`arguments`), + * `boolean_operator` (fields `left`/`operator`/`right`), + * `conditional_expression` (ternary: consequent / condition / alternative in + * source order), `parenthesized_expression`. + * + * TWO-PHASE, ORDER-INDEPENDENT (load-bearing — mirrors the TS / Go harvesters): + * the CFG walk is NOT source-order, so resolving names against a scope stack + * populated *during* the walk would mis-resolve. Phase 1 pre-scans the whole + * function subtree once, declaring every in-function name (Python's def-on-first- + * assignment scoping has a SINGLE function scope — there is no block scope, so + * the whole function body shares one table). Phase 2 resolves defs/uses against + * that finished table from any walk order. + * + * Python scope model (deliberately simplified, documented): Python binds names + * at FUNCTION scope on first assignment anywhere in the body (no block scope). + * We therefore declare all assignment / for / with / except / walrus / + * comprehension targets and parameters into the single function table. + * `global x` / `nonlocal x` names are recorded as SYNTHETIC module-level + * bindings (`name@module`) so their writes/reads share one binding with the + * outer scope rather than minting a confusing function-local. Comprehension + * targets technically have their OWN scope in Py3 (a leaked `i` after `[i for + * i in xs]` does NOT exist), but we declare them in the function table anyway — + * a documented over-approximation that keeps the comprehension target a real + * def (the plan's explicit ask) without modeling nested comprehension scopes. + * + * v1 def-semantics scope: + * - `assignment` plain `=` — each identifier target in the (possibly nested) + * LHS pattern is a def; a `*rest` splat target is a def; an `attribute` / + * `subscript` target is NOT a scalar def (its root identifier is a use). + * - `augmented_assignment` (`x += 1`) — def AND use the lvalue. + * - `named_expression` walrus (`(n := f())`) — `n` is a def. + * - `for_statement` / `for_in_clause` `left` — loop/comprehension targets are + * defs; `right` is a use. + * - `with_item` `as` alias (`with cm as fh`) — `fh` is a def. + * - `except ... as e` — `e` is a `catch`-kind def (matters to the taint pass). + * - parameters (incl. defaults, `*args`, `**kwargs`, typed) — `param`-kind defs. + * EXCLUDED, deliberately (TypeScript-CFA precedent): attribute / subscript + * writes (`obj.f = …`, `arr[i] = …`) are NOT scalar defs — their root + * identifiers are uses only. Nested function (`function_definition` / `lambda`) + * bodies are opaque in BOTH directions (reads of and writes to captured outer + * variables are invisible — callback flows are later-pass territory). + * + * MAY-DEFS: a def inside a conditionally-evaluated subexpression is a may-def + * (gen WITHOUT kill), so the not-taken path's prior def is not falsely killed. + * Python's conditional-def shapes: a walrus in the right operand of `or` / `and` + * short-circuit (`a or (x := b)`), a walrus in either non-test arm of a ternary + * (`(a := p) if c else (b := q)`), and a `case`-clause guard / pattern test. + * + * NOTE: nothing serialized here may carry a field named `nodeId` — the durable + * parsedfile-store reviver dedups objects keyed on that field name. + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import type { BindingEntry, StatementFacts } from '../types.js'; + +/** Node types that own a nested CFG — their subtrees are opaque to harvesting. */ +const NESTED_FUNCTION_TYPES = new Set(['function_definition', 'lambda']); + +/** LHS pattern containers whose identifier/splat leaves are assignment targets. */ +const PATTERN_LIST_TYPES = new Set(['pattern_list', 'tuple_pattern', 'list_pattern']); + +/** + * Minimal ordered, deduplicating def/use collector for one statement record. + * Deliberately NOT the shared {@link import('./call-site-harvest.js') + * CallSiteFactAccumulator} — this unit harvests NO call sites (taint substrate + * is a later step), so a local accumulator with only the def/use/may-def + * machinery keeps `python-harvest.ts` free of site logic and guarantees the + * emitted facts carry no `sites` key. + */ +class FactAccumulator { + private readonly defs: number[] = []; + private readonly uses: number[] = []; + private readonly mayDefs: number[] = []; + private readonly defSeen = new Set(); + private readonly useSeen = new Set(); + private readonly mayDefSeen = new Set(); + + constructor(private readonly line: number) {} + + addDef(idx: number): void { + if (this.defSeen.has(idx)) return; + this.defSeen.add(idx); + this.defs.push(idx); + } + + /** A def that may not execute (conditional context) — gen without kill. */ + addMayDef(idx: number): void { + if (this.mayDefSeen.has(idx)) return; + this.mayDefSeen.add(idx); + this.mayDefs.push(idx); + } + + addUse(idx: number): void { + if (this.useSeen.has(idx)) return; + this.useSeen.add(idx); + this.uses.push(idx); + } + + defCount(): number { + return this.defs.length + this.mayDefs.length; + } + + finish(): StatementFacts { + return { + line: this.line, + defs: this.defs, + uses: this.uses, + // Stay absent when empty — keeps the serialized side-channel payload lean. + ...(this.mayDefs.length > 0 ? { mayDefs: this.mayDefs } : {}), + }; + } +} + +export class PythonHarvester { + private readonly bindings: BindingEntry[] = []; + /** Single function-scope name → binding index (Python has no block scope). */ + private readonly table = new Map(); + private readonly synthetic = new Map(); + /** Names declared `global`/`nonlocal` — resolve to the synthetic module binding. */ + private readonly globalNames = new Set(); + private readonly fnId: number; + /** >0 while walking a conditionally-evaluated subexpression — defs become may-defs. */ + private conditionalDepth = 0; + + constructor(private readonly fnNode: SyntaxNode) { + this.fnId = fnNode.id; + this.declareParams(fnNode); + const body = this.bodyOf(fnNode); + if (body) this.prescan(body); + } + + /** The completed binding table — pass to `CfgBuilder.finish`. */ + bindingTable(): readonly BindingEntry[] { + return this.bindings; + } + + /** The function/lambda body node (a `block` for `def`, an expression for `lambda`). */ + private bodyOf(fnNode: SyntaxNode): SyntaxNode | undefined { + return fnNode.childForFieldName('body') ?? undefined; + } + + // ── phase 1: declaration pre-scan ──────────────────────────────────────── + + private declare(nameNode: SyntaxNode, kind: BindingEntry['kind']): void { + const name = nameNode.text; + if (!name || name === '_' || this.table.has(name) || this.globalNames.has(name)) return; + this.table.set(name, this.bindings.length); + this.bindings.push({ + name, + declLine: nameNode.startPosition.row + 1, + declColumn: nameNode.startPosition.column, + kind, + }); + } + + /** Declare every parameter binder (incl. defaults, typed, `*args`, `**kwargs`). */ + private declareParams(fnNode: SyntaxNode): void { + const params = + fnNode.childForFieldName('parameters') ?? + fnNode.namedChildren.find( + (c) => c.type === 'parameters' || c.type === 'lambda_parameters', + ); + if (!params) return; + for (let i = 0; i < params.namedChildCount; i++) { + const p = params.namedChild(i); + if (p) this.declareParam(p); + } + } + + /** Declare the binder identifier(s) of one parameter node. */ + private declareParam(p: SyntaxNode): void { + switch (p.type) { + case 'identifier': + this.declare(p, 'param'); + return; + case 'default_parameter': + case 'typed_default_parameter': { + const name = p.childForFieldName('name'); + if (name) this.declareParamBinder(name); + return; + } + case 'typed_parameter': { + // No `name` field — the binder is the first non-`type` named child + // (an identifier or a splat pattern). + const typeNode = p.childForFieldName('type'); + for (let i = 0; i < p.namedChildCount; i++) { + const c = p.namedChild(i); + if (c && c.id !== typeNode?.id) { + this.declareParamBinder(c); + break; + } + } + return; + } + case 'list_splat_pattern': + case 'dictionary_splat_pattern': + this.declareParamBinder(p); + return; + default: + // tuple-grouped params and the like — declare any identifier leaves. + this.declareParamBinder(p); + } + } + + /** Declare a binder that may be an identifier or a `*`/`**` splat pattern. */ + private declareParamBinder(node: SyntaxNode): void { + if (node.type === 'identifier') { + this.declare(node, 'param'); + return; + } + if (node.type === 'list_splat_pattern' || node.type === 'dictionary_splat_pattern') { + const id = node.namedChild(0); + if (id?.type === 'identifier') this.declare(id, 'param'); + return; + } + // Nested grouping — declare identifier descendants up to the next binder. + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c?.type === 'identifier') this.declare(c, 'param'); + } + } + + /** + * Pre-scan the function body once, declaring every in-function name. Recurses + * into compound statements but NOT into nested `function_definition` / `lambda` + * bodies (opaque). `global`/`nonlocal` are processed FIRST in a sibling sweep + * so a later assignment to a global name does not mint a function-local. + */ + private prescan(node: SyntaxNode): void { + const t = node.type; + if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId) return; + + switch (t) { + case 'global_statement': + case 'nonlocal_statement': { + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c?.type === 'identifier') this.globalNames.add(c.text); + } + return; + } + case 'assignment': { + const left = node.childForFieldName('left'); + if (left) this.declareTargets(left); + break; + } + case 'augmented_assignment': { + const left = node.childForFieldName('left'); + if (left) this.declareTargets(left); + break; + } + case 'named_expression': { + const name = node.childForFieldName('name'); + if (name?.type === 'identifier') this.declare(name, 'let'); + break; + } + case 'for_statement': + case 'for_in_clause': { + const left = node.childForFieldName('left'); + if (left) this.declareTargets(left); + break; + } + case 'with_item': { + this.declareWithItem(node); + break; + } + case 'except_clause': + case 'except_group_clause': { + this.declareExceptAlias(node); + break; + } + default: + break; + } + + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.prescan(c); + } + } + + /** Declare identifier/splat leaves of an assignment / loop target pattern. */ + private declareTargets(target: SyntaxNode): void { + const t = target.type; + if (t === 'identifier') { + this.declare(target, 'let'); + return; + } + if (PATTERN_LIST_TYPES.has(t)) { + for (let i = 0; i < target.namedChildCount; i++) { + const c = target.namedChild(i); + if (c) this.declareTargets(c); + } + return; + } + if (t === 'list_splat_pattern') { + const id = target.namedChild(0); + if (id?.type === 'identifier') this.declare(id, 'let'); + return; + } + // attribute / subscript target — not a scalar def (root is a use only). + } + + /** `with EXPR as TARGET` — declare the alias target(s). */ + private declareWithItem(item: SyntaxNode): void { + const value = item.childForFieldName('value') ?? item.namedChild(0); + if (value?.type !== 'as_pattern') return; + const alias = value.childForFieldName('alias') ?? this.asPatternTarget(value); + if (alias) this.declareAsTarget(alias); + } + + /** `except E as e` — declare `e`. */ + private declareExceptAlias(clause: SyntaxNode): void { + for (let i = 0; i < clause.namedChildCount; i++) { + const c = clause.namedChild(i); + if (c?.type === 'as_pattern') { + const alias = c.childForFieldName('alias') ?? this.asPatternTarget(c); + if (alias) this.declareAsTarget(alias, 'catch'); + } + } + } + + /** The `as_pattern_target` child of an `as_pattern` (when no `alias` field). */ + private asPatternTarget(asPattern: SyntaxNode): SyntaxNode | undefined { + return asPattern.namedChildren.find((c) => c.type === 'as_pattern_target'); + } + + /** Declare an `as_pattern_target` (or its identifier child) as a binding. */ + private declareAsTarget(target: SyntaxNode, kind: BindingEntry['kind'] = 'let'): void { + if (target.type === 'identifier') { + this.declare(target, kind); + return; + } + // `as_pattern_target` wraps an identifier (or a tuple pattern). + const inner = target.namedChild(0); + if (inner?.type === 'identifier') this.declare(inner, kind); + else if (inner) this.declareTargets(inner); + else this.declare(target, kind); // a bare `as_pattern_target` text IS the name + } + + // ── phase 2: per-statement fact extraction ─────────────────────────────── + + /** Def/use facts for one statement (or construct-header expression) node. */ + facts(node: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(node.startPosition.row + 1); + this.walkValue(node, acc); + return acc.finish(); + } + + /** Facts for an expression whose WHOLE evaluation is conditional (case tests). */ + factsConditional(node: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(node.startPosition.row + 1); + this.conditional(() => this.walkValue(node, acc)); + return acc.finish(); + } + + /** + * Facts for a `for TARGET in ITER` / `for_in_clause` head: the loop target(s) + * are defs, the iterated expression is a use. + */ + loopHeadFacts(headNode: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(headNode.startPosition.row + 1); + const left = headNode.childForFieldName('left'); + const right = headNode.childForFieldName('right'); + if (right) this.walkValue(right, acc); + if (left) this.defTargets(left, acc); + return acc.finish(); + } + + /** Facts for a `with_item`: the `as` alias is a def, the value an use. */ + withItemFacts(item: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(item.startPosition.row + 1); + const value = item.childForFieldName('value') ?? item.namedChild(0); + if (!value) return acc.finish(); + if (value.type === 'as_pattern') { + const inner = value.namedChild(0); + if (inner) this.walkValue(inner, acc); + const alias = value.childForFieldName('alias') ?? this.asPatternTarget(value); + if (alias) this.defAsTarget(alias, acc); + } else { + this.walkValue(value, acc); + } + return acc.finish(); + } + + /** Facts for an `except E as e:` header: `e` is a def, `E` a use. */ + exceptHeadFacts(clause: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(clause.startPosition.row + 1); + const block = clause.namedChildren.find((c) => c.type === 'block'); + for (let i = 0; i < clause.namedChildCount; i++) { + const c = clause.namedChild(i); + if (!c || c.id === block?.id) continue; + if (c.type === 'as_pattern') { + const exc = c.namedChild(0); + if (exc) this.walkValue(exc, acc); + const alias = c.childForFieldName('alias') ?? this.asPatternTarget(c); + if (alias) this.defAsTarget(alias, acc); + } else { + this.walkValue(c, acc); + } + } + return acc.finish(); + } + + /** ENTRY-block facts for the parameters (defs only — incl. default-value uses). */ + paramFacts(): StatementFacts | undefined { + const params = + this.fnNode.childForFieldName('parameters') ?? + this.fnNode.namedChildren.find( + (c) => c.type === 'parameters' || c.type === 'lambda_parameters', + ); + if (!params) return undefined; + const acc = new FactAccumulator(this.fnNode.startPosition.row + 1); + for (let i = 0; i < params.namedChildCount; i++) { + const p = params.namedChild(i); + if (p) this.defParam(p, acc); + } + return acc.defCount() || acc.finish().uses.length ? acc.finish() : undefined; + } + + /** Def the binder(s) of one parameter node and use any default-value expr. */ + private defParam(p: SyntaxNode, acc: FactAccumulator): void { + switch (p.type) { + case 'identifier': + this.def(p, acc); + return; + case 'default_parameter': + case 'typed_default_parameter': { + const value = p.childForFieldName('value'); + if (value) this.walkValue(value, acc); + const name = p.childForFieldName('name'); + if (name) this.defParamBinder(name, acc); + return; + } + case 'typed_parameter': { + const typeNode = p.childForFieldName('type'); + for (let i = 0; i < p.namedChildCount; i++) { + const c = p.namedChild(i); + if (c && c.id !== typeNode?.id) { + this.defParamBinder(c, acc); + break; + } + } + return; + } + case 'list_splat_pattern': + case 'dictionary_splat_pattern': + this.defParamBinder(p, acc); + return; + default: + this.defParamBinder(p, acc); + } + } + + private defParamBinder(node: SyntaxNode, acc: FactAccumulator): void { + if (node.type === 'identifier') { + this.def(node, acc); + return; + } + if (node.type === 'list_splat_pattern' || node.type === 'dictionary_splat_pattern') { + const id = node.namedChild(0); + if (id?.type === 'identifier') this.def(id, acc); + return; + } + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c?.type === 'identifier') this.def(c, acc); + } + } + + private resolve(nameNode: SyntaxNode): number { + const name = nameNode.text; + if (!this.globalNames.has(name)) { + const idx = this.table.get(name); + if (idx !== undefined) return idx; + } + let idx = this.synthetic.get(name); + if (idx === undefined) { + idx = this.bindings.length; + this.synthetic.set(name, idx); + this.bindings.push({ name, declLine: 0, declColumn: 0, kind: 'module', synthetic: true }); + } + return idx; + } + + private def(nameNode: SyntaxNode, acc: FactAccumulator): void { + if (nameNode.text === '_') return; // blank target defines nothing of interest + if (this.conditionalDepth > 0) acc.addMayDef(this.resolve(nameNode)); + else acc.addDef(this.resolve(nameNode)); + } + + private use(nameNode: SyntaxNode, acc: FactAccumulator): void { + if (nameNode.text === '_') return; + acc.addUse(this.resolve(nameNode)); + } + + /** Run `fn` with defs demoted to may-defs (conditionally-evaluated context). */ + private conditional(fn: () => void): void { + this.conditionalDepth++; + try { + fn(); + } finally { + this.conditionalDepth--; + } + } + + /** + * Def each identifier/splat leaf of an assignment / loop target pattern; route + * attribute / subscript targets to the value walk (root identifier is a use). + */ + private defTargets(target: SyntaxNode, acc: FactAccumulator): void { + const t = target.type; + if (t === 'identifier') { + this.def(target, acc); + return; + } + if (PATTERN_LIST_TYPES.has(t)) { + for (let i = 0; i < target.namedChildCount; i++) { + const c = target.namedChild(i); + if (c) this.defTargets(c, acc); + } + return; + } + if (t === 'list_splat_pattern') { + const id = target.namedChild(0); + if (id?.type === 'identifier') this.def(id, acc); + else if (id) this.defTargets(id, acc); + return; + } + // attribute / subscript / call target — uses only (the root identifier). + this.walkValue(target, acc); + } + + /** Def an `as_pattern_target` (or its identifier child). */ + private defAsTarget(target: SyntaxNode, acc: FactAccumulator): void { + if (target.type === 'identifier') { + this.def(target, acc); + return; + } + const inner = target.namedChild(0); + if (inner?.type === 'identifier') this.def(inner, acc); + else if (inner) this.defTargets(inner, acc); + else this.def(target, acc); + } + + /** Value-position walk: collect uses; route def positions to the target handler. */ + private walkValue(node: SyntaxNode, acc: FactAccumulator): void { + const t = node.type; + if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId) return; // opaque + + switch (t) { + case 'identifier': + this.use(node, acc); + return; + case 'assignment': { + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (right) this.walkValue(right, acc); + if (left) this.defTargets(left, acc); + return; + } + case 'augmented_assignment': { + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (right) this.walkValue(right, acc); + if (left) { + // `x += v` reads AND writes a plain-identifier lvalue. + if (left.type === 'identifier') { + this.use(left, acc); + this.def(left, acc); + } else { + this.walkValue(left, acc); // attribute/subscript lvalue — use only + } + } + return; + } + case 'named_expression': { + // walrus `(n := v)` — `n` is a def, `v` a use. + const name = node.childForFieldName('name'); + const value = node.childForFieldName('value'); + if (value) this.walkValue(value, acc); + if (name?.type === 'identifier') this.def(name, acc); + return; + } + case 'boolean_operator': { + // `a or b` / `a and b` — the right operand is conditionally evaluated, so + // any def inside it (a walrus) is a may-def; uses are still recorded. + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (left) this.walkValue(left, acc); + if (right) this.conditional(() => this.walkValue(right, acc)); + return; + } + case 'conditional_expression': { + // `consequent if condition else alternative`. The condition always + // evaluates; each arm is conditional (its walrus defs are may-defs). + const children = node.namedChildren; + const [consequent, condition, alternative] = children; + if (condition) this.walkValue(condition, acc); + if (consequent) this.conditional(() => this.walkValue(consequent, acc)); + if (alternative) this.conditional(() => this.walkValue(alternative, acc)); + return; + } + case 'attribute': { + // `a.b` — value read of the operand root only; the attribute name is not + // a scalar binding. + const obj = node.childForFieldName('object'); + if (obj) this.walkValue(obj, acc); + return; + } + case 'subscript': { + // `a[i]` — both the container root and the index are uses. + const value = node.childForFieldName('value'); + const sub = node.childForFieldName('subscript'); + if (value) this.walkValue(value, acc); + if (sub) this.walkValue(sub, acc); + return; + } + case 'call': { + // Reproduce default-descent uses: callee chain + arguments. No site + // record (taint substrate is a later step). + const fn = node.childForFieldName('function'); + const args = node.childForFieldName('arguments'); + if (fn) this.walkValue(fn, acc); + if (args) this.walkValue(args, acc); + return; + } + case 'list_comprehension': + case 'set_comprehension': + case 'dictionary_comprehension': + case 'generator_expression': { + this.walkComprehension(node, acc); + return; + } + case 'keyword_argument': { + // `f(k=v)` — `k` is a parameter name, not a use; only `v` is a use. + const value = node.childForFieldName('value'); + if (value) this.walkValue(value, acc); + return; + } + default: + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.walkValue(c, acc); + } + } + } + + /** + * A comprehension (`[body for t in src if g]`): each `for_in_clause` target is + * a def, its source a use; the `body` and any `if_clause` are uses. Kept INLINE + * (no separate CFG blocks) per the plan — the target binding is harvested so it + * is a real def. The `for_in_clause` source is walked in the ENCLOSING context + * (it reads outer names), the body/filter after the targets are bound. + */ + private walkComprehension(node: SyntaxNode, acc: FactAccumulator): void { + const body = node.childForFieldName('body'); + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (!c) continue; + if (c.type === 'for_in_clause') { + const left = c.childForFieldName('left'); + const right = c.childForFieldName('right'); + if (right) this.walkValue(right, acc); + if (left) this.defTargets(left, acc); + } else if (c.id !== body?.id) { + // `if_clause` filter (and any other auxiliary clause) — uses only. + this.walkValue(c, acc); + } + } + if (body) this.walkValue(body, acc); + } +} diff --git a/gitnexus/src/core/ingestion/cfg/visitors/python.ts b/gitnexus/src/core/ingestion/cfg/visitors/python.ts new file mode 100644 index 000000000..24bb5cd9a --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/python.ts @@ -0,0 +1,755 @@ +/** + * Python CfgVisitor — the most STRUCTURALLY DIVERGENT CFG target (PDG layer + * beyond the C-family). Python has no braces (suites are indented `block` nodes), + * `elif` clauses, `for`/`while ... else` (the else runs on NORMAL completion, not + * on `break`), `with` (deterministic `__exit__` on both normal AND exception + * exit — a try/finally analogue), `try` / `except` / `except-group` (except-star) + * / `else` / `finally`, and `match`/`case` (no fallthrough). A good stress test + * that the shared CFG core + * carries no hidden brace-family assumptions. + * + * Walks a Python `function_definition` / `lambda`'s tree-sitter AST and drives + * the language-agnostic {@link CfgBuilder} to produce a serializable + * {@link FunctionCfg}, plus a def/use harvest ({@link PythonHarvester}) for the + * reaching-defs / CDG solvers. Structured like the C-family visitors — a + * `visit_` dispatch over the statement taxonomy, driving a + * per-function {@link ControlFlowContext} for break/continue and the `with` / + * `finally` completion chain (finalizer route-through). NO call-site `sites[]` + * are harvested (taint substrate is a later step — see python-harvest.ts). + * + * Every node type and field literal below was grammar-validated against + * tree-sitter-python (0.23.x) via the introspection probe before use (mandatory + * pre-step). Python shapes pre-empted (verified by a real parse): + * - functions: `function_definition` (fields `name`/`parameters`/`body`; async + * is the SAME node with an `async` token child), `lambda` (fields + * `parameters`/`body`; the body is an EXPRESSION, not a `block`). + * - `if_statement` fields `condition`/`consequence` plus ZERO-OR-MORE + * `alternative` fields, each an `elif_clause` (fields `condition`/`consequence`) + * or an `else_clause` (field `body`) — Python has no nested-`if` else chain. + * - `for_statement` fields `left`/`right`/`body` + optional `alternative` + * (`else_clause`); `while_statement` fields `condition`/`body` + optional + * `alternative` (`else_clause`). The loop `else` runs on the cond-false / + * normal-completion path, NOT on `break`. + * - `with_statement` field `body`; a `with_clause` of `with_item`s (field + * `value` = `as_pattern` or a bare expression). `__exit__` runs on normal AND + * exception exit — modeled as a finalizer (try/finally analogue). + * - `try_statement` field `body`; children `except_clause` / + * `except_group_clause` (each holds the exception expr/`as_pattern` + a + * `block`), `else_clause` (field `body`, runs if NO exception), and + * `finally_clause` (holds a `block`). + * - `match_statement` fields `subject`/`body`; the `body` `block` holds + * `case_clause`s (field `alternative`), each with `case_pattern` child(ren), + * an optional `guard` (`if_clause`), and a `consequence` `block`. No + * fallthrough between cases. + * - `return_statement` / `raise_statement` / `break_statement` / + * `continue_statement` / `pass_statement`. + * + * Edge-kind contract (matches the existing visitors — RD/CDG consume these): + * - if/elif/else → `cond-true` / `cond-false` + * - for/while → `cond-true` / `loop-back` / `cond-false`; the loop `else` runs + * on the `cond-false` / normal-completion path (NOT on `break`) + * - match dispatch → `switch-case` (no fallthrough — like Go's switch) + * - try/except → `throw` (every protected block → each except handler) + * - a `break`/`continue`/`return` crossing a `with` `__exit__` or a `finally` + * threads through as `break`/`continue`/`return` (first leg) + + * `finally-break`/`finally-continue`/`finally-return` (each completion leg) + * - return/raise/break/continue → the matching terminator kind + * - straight-line → `seq` + * + * Python-specific modeling decisions (documented approximations): + * - `with EXPR as t:` runs the body then `__exit__` deterministically on BOTH + * the normal exit and an exception (it can suppress the exception, but the + * common case re-raises). Modeled exactly like `try/finally`: a finalizer + * frame for the body's exit-dispose, with the protected body edging the + * dispose block on `throw`. APPROXIMATION: exception SUPPRESSION by a context + * manager is not modeled (the dispose re-propagates), the sound direction. + * - the loop `else` clause runs once on normal completion (the loop ran to + * exhaustion without `break`). It sits on the `cond-false` edge BEFORE the + * join; a `break` targets the loop exit AFTER the else, so `break` skips it. + * - `match`/`case` cases do NOT fall through (like Go). The dispatch fans a + * `switch-case` edge to each case body; a guarded / non-wildcard tail with no + * `case _` also reaches the join directly (no-match path), keeping EXIT + * reverse-reachable. + * - `while True:` (and any loop with no statically-false exit) STILL emits the + * structural `header → loopExit` `cond-false` edge — exactly like the + * C-family visitors — so EXIT stays reverse-reachable and the post-dominator / + * CDG pass is not silently skipped for the function. Highest-risk property. + * - `lambda` has an EXPRESSION body (no `block`): one block whose value is the + * returned expression. + * - comprehensions are harvested for their target bindings but kept INLINE (no + * separate CFG blocks) — the plan's explicit choice. + * + * Known limitations: + * - async (`async def`, `await`, `async for`, `async with`) is the same node + * shape as the sync form plus an `async` token; the suspension points are + * modeled as normal straight-line control flow (no scheduler edges). + * - generators (`yield` / `yield from`): a `yield` is a normal expression here + * (no suspend/resume edge); the generator's resumption flow is not modeled. + * - comprehension target scoping: comprehension targets are declared in the + * single function table (Python 3 gives them their own scope; the leaked-name + * distinction is not modeled) — see python-harvest.ts. + * - context-manager exception SUPPRESSION and `recover`-style flow are not + * modeled (the `with` dispose always re-propagates). + * + * Returns `undefined` (never throws) for an AST shape it cannot model, so a + * malformed function never drops the whole file's CFG group (R4). + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { CfgBuilder } from '../cfg-builder.js'; +import { + ControlFlowContext, + drainFinalizerPending, + wireJumpThroughFinalizers, +} from '../control-flow-context.js'; +import type { FinalizerFrame } from '../control-flow-context.js'; +import type { TraversalResult } from '../traversal-result.js'; +import type { CfgVisitor, FunctionCfg } from '../types.js'; +import { PythonHarvester } from './python-harvest.js'; + +/** Python node types that own a CFG-bearing function body. */ +const PY_FUNCTION_TYPES = new Set(['function_definition', 'lambda']); + +/** Statement node types that break a basic block (everything else coalesces). */ +const CONTROL_FLOW_TYPES = new Set([ + 'if_statement', + 'for_statement', + 'while_statement', + 'with_statement', + 'try_statement', + 'match_statement', + 'return_statement', + 'raise_statement', + 'break_statement', + 'continue_statement', + 'block', +]); + +const startLineOf = (n: SyntaxNode): number => n.startPosition.row + 1; +const endLineOf = (n: SyntaxNode): number => n.endPosition.row + 1; + +/** A statement sequence that produced no blocks (empty body) is "transparent". */ +type SeqResult = TraversalResult | null; + +/** + * Per-function Python walk state. One instance per function so the + * {@link ControlFlowContext}, the exception-handler stack, and the `with` / + * `finally` finalizer chain are scoped to that function and never leak across + * functions. + */ +class PythonCfgWalk { + private readonly cfc = new ControlFlowContext(); + /** Stack of exception-handler entry blocks (except/finally/with-dispose) a `raise` jumps to. */ + private readonly handlers: number[] = []; + + constructor( + private readonly builder: CfgBuilder, + private readonly harvest: PythonHarvester, + ) {} + + /** Statements of a block node, ignoring comments. */ + private statementsOf(block: SyntaxNode): SyntaxNode[] { + return block.namedChildren.filter((c) => c.type !== 'comment'); + } + + /** The `body` block of a node (field, or the first `block` child). */ + private bodyBlockOf(node: SyntaxNode): SyntaxNode | undefined { + return node.childForFieldName('body') ?? node.namedChildren.find((c) => c.type === 'block'); + } + + /** Visit a body that may be a `block` or a single statement. */ + private visitBody(node: SyntaxNode | undefined | null): SeqResult { + if (!node) return null; + if (node.type === 'block') return this.visitSeq(this.statementsOf(node)); + return this.visitStmt(node); + } + + /** Wire a sequence of statements, coalescing straight-line runs into blocks. */ + visitSeq(stmts: SyntaxNode[]): SeqResult { + let entry: number | undefined; + let dangling: number[] = []; + let openSimple: number | undefined; + + for (const stmt of stmts) { + if (CONTROL_FLOW_TYPES.has(stmt.type)) { + openSimple = undefined; // close any open straight-line block + const res = this.visitStmt(stmt); + if (res === null) continue; // transparent (empty nested block) + if (entry === undefined) entry = res.entry; + else this.builder.connect(dangling, res.entry, 'seq'); + dangling = [...res.exits]; + } else { + if (openSimple === undefined) { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + if (entry === undefined) entry = idx; + else this.builder.connect(dangling, idx, 'seq'); + openSimple = idx; + dangling = [idx]; + } else { + this.builder.extendBlock(openSimple, endLineOf(stmt), stmt.text, this.harvest.facts(stmt)); + } + } + } + + if (entry === undefined) return null; + return { entry, exits: dangling }; + } + + /** Dispatch one statement to its handler. Non-null except for empty blocks. */ + visitStmt(stmt: SyntaxNode): SeqResult { + switch (stmt.type) { + case 'if_statement': + return this.visitIf(stmt); + case 'for_statement': + return this.visitFor(stmt); + case 'while_statement': + return this.visitWhile(stmt); + case 'with_statement': + return this.visitWith(stmt); + case 'try_statement': + return this.visitTry(stmt); + case 'match_statement': + return this.visitMatch(stmt); + case 'return_statement': + return this.visitReturn(stmt); + case 'raise_statement': + return this.visitRaise(stmt); + case 'break_statement': + return this.visitBreak(stmt); + case 'continue_statement': + return this.visitContinue(stmt); + case 'block': + return this.visitSeq(this.statementsOf(stmt)); + default: + return this.visitSimple(stmt); + } + } + + private visitSimple(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + return { entry: idx, exits: [idx] }; + } + + /** `return [expr]` — threads through EVERY active `with`/`finally` before EXIT. */ + private visitReturn(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + wireJumpThroughFinalizers( + this.builder, + idx, + this.cfc.finalizersForReturn(), + this.builder.exitIndex, + 'return', + ); + return { entry: idx, exits: [] }; + } + + /** `raise [expr]` — jumps to the nearest handler (except / with-dispose / EXIT). */ + private visitRaise(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + this.builder.edge(idx, this.currentHandler(), 'throw'); + return { entry: idx, exits: [] }; + } + + private visitBreak(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const res = this.cfc.resolveBreak(); + const { target, finalizers } = res ?? { + target: this.builder.exitIndex, + finalizers: this.cfc.finalizersForReturn(), + }; + wireJumpThroughFinalizers(this.builder, idx, finalizers, target, 'break'); + return { entry: idx, exits: [] }; + } + + private visitContinue(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const res = this.cfc.resolveContinue(); + const { target, finalizers } = res ?? { + target: this.builder.exitIndex, + finalizers: this.cfc.finalizersForReturn(), + }; + wireJumpThroughFinalizers(this.builder, idx, finalizers, target, 'continue'); + return { entry: idx, exits: [] }; + } + + /** + * `if cond: … elif cond: … else: …`. Python has NO nested-if else chain: an + * `if_statement` carries the condition + consequence plus zero-or-more + * `alternative` fields, each an `elif_clause` (its own condition + consequence) + * or a single trailing `else_clause`. The elif chain is threaded on the + * `cond-false` edge. + */ + private visitIf(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + const header = this.builder.newBlock( + startLineOf(stmt), + endLineOf(cond), + cond.text, + 'normal', + this.harvest.facts(cond), + ); + + const exits: number[] = []; + const thenRes = this.visitBody(stmt.childForFieldName('consequence')); + if (thenRes) { + this.builder.edge(header, thenRes.entry, 'cond-true'); + exits.push(...thenRes.exits); + } else { + exits.push(header); // empty then — true path falls through + } + + // The alternatives, in source order: elif_clause* then optional else_clause. + const alternatives = this.alternativesOf(stmt); + let falseFrom = header; // block whose cond-false edge feeds the next alternative + for (const alt of alternatives) { + if (alt.type === 'elif_clause') { + const elifCond = alt.childForFieldName('condition') ?? alt; + const elifHeader = this.builder.newBlock( + startLineOf(alt), + endLineOf(elifCond), + elifCond.text, + 'normal', + this.harvest.facts(elifCond), + ); + this.builder.edge(falseFrom, elifHeader, 'cond-false'); + const elifRes = this.visitBody(alt.childForFieldName('consequence')); + if (elifRes) { + this.builder.edge(elifHeader, elifRes.entry, 'cond-true'); + exits.push(...elifRes.exits); + } else { + exits.push(elifHeader); + } + falseFrom = elifHeader; + } else if (alt.type === 'else_clause') { + const elseRes = this.visitBody(alt.childForFieldName('body')); + if (elseRes) { + this.builder.edge(falseFrom, elseRes.entry, 'cond-false'); + exits.push(...elseRes.exits); + } else { + exits.push(falseFrom); + } + falseFrom = -1; // an else consumes the false path entirely + } + } + // No trailing else: the last header's cond-false falls through to the join. + if (falseFrom >= 0) exits.push(falseFrom); + + return { entry: header, exits: [...new Set(exits)] }; + } + + /** The `alternative`-field children of an `if_statement`, in source order. */ + private alternativesOf(stmt: SyntaxNode): SyntaxNode[] { + const out: SyntaxNode[] = []; + for (let i = 0; i < stmt.childCount; i++) { + if (stmt.fieldNameForChild(i) === 'alternative') { + const c = stmt.child(i); + if (c) out.push(c); + } + } + return out; + } + + /** + * `for TARGET in ITER: … [else: …]`. Header = the iteration test (a use of the + * iterable + a def of the target). The loop `else` runs on NORMAL completion + * (the cond-false path) — a `break` targets the loop exit AFTER the else, so it + * skips the else. + */ + private visitFor(stmt: SyntaxNode): TraversalResult { + const left = stmt.childForFieldName('left'); + const right = stmt.childForFieldName('right'); + const headEnd = right ? endLineOf(right) : startLineOf(stmt); + const header = this.builder.newBlock( + startLineOf(stmt), + headEnd, + this.loopHeaderText(stmt, left, right), + 'normal', + this.harvest.loopHeadFacts(stmt), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, []); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); // empty body re-tests + } + + this.wireLoopElse(stmt, header, loopExit); + return { entry: header, exits: [loopExit] }; + } + + /** `while cond: … [else: …]`. Same `else`-on-normal-completion semantics as `for`. */ + private visitWhile(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + const header = this.builder.newBlock( + startLineOf(stmt), + endLineOf(cond), + cond.text, + 'normal', + this.harvest.facts(cond), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, []); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); // empty `while c: pass` re-tests + } + + this.wireLoopElse(stmt, header, loopExit); + return { entry: header, exits: [loopExit] }; + } + + /** + * Wire the optional loop `else` clause. The else runs once on normal completion + * (the header's `cond-false` edge). With an else, the cond-false edge goes + * `header → elseEntry` and the else's exits reach `loopExit`; without one, the + * structural `header → loopExit` `cond-false` edge keeps EXIT reverse-reachable + * (critical for `while True:` — and matches the C-family visitors). A `break` + * always targets `loopExit` directly, so it never runs the else. + */ + private wireLoopElse(stmt: SyntaxNode, header: number, loopExit: number): void { + const elseClause = this.loopElseOf(stmt); + if (elseClause) { + const elseRes = this.visitBody(elseClause.childForFieldName('body')); + if (elseRes) { + this.builder.edge(header, elseRes.entry, 'cond-false'); + this.builder.connect(elseRes.exits, loopExit, 'seq'); + return; + } + } + // No (or empty) else — normal completion falls straight to the loop exit. + this.builder.edge(header, loopExit, 'cond-false'); + } + + /** The `else_clause` of a `for`/`while` (its `alternative` field), if any. */ + private loopElseOf(stmt: SyntaxNode): SyntaxNode | undefined { + const alt = stmt.childForFieldName('alternative'); + return alt?.type === 'else_clause' ? alt : undefined; + } + + private loopHeaderText(stmt: SyntaxNode, left: SyntaxNode | null, right: SyntaxNode | null): string { + const l = left?.text ?? ''; + const r = right?.text ?? ''; + return l || r ? `for ${l} in ${r}` : stmt.text.split('\n')[0]; + } + + /** + * `with EXPR [as t], …: BODY`. The context managers' `__exit__` runs + * deterministically on BOTH the normal exit and an exception — modeled exactly + * like `try/finally`: a finalizer frame holding the dispose block, plus a + * `throw` edge from every protected-body block to the dispose. A + * `return`/`break`/`continue` inside the body threads through the dispose. The + * dispose re-propagates on the exception path (suppression is not modeled). + */ + private visitWith(stmt: SyntaxNode): SeqResult { + // The dispose block carries the `with`-header facts (the `as` aliases are + // defs, the manager expressions uses) — it runs on every exit, so attaching + // the binding facts here is the single execution point of the bindings. + const items = this.withItems(stmt); + const dispose = this.builder.newBlock( + startLineOf(stmt), + startLineOf(stmt), + this.withHeaderText(stmt), + ); + for (const item of items) this.builder.attachFacts(dispose, this.harvest.withItemFacts(item)); + + const finFrame = this.cfc.pushFinalizer(dispose); + // The body raises into the dispose (which re-propagates to the outer handler). + this.handlers.push(dispose); + const protectedStart = this.builder.blockCount; + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.handlers.pop(); + + // Conservative exceptional edges: ANY block in the with-body may raise to the + // dispose (an exception fires mid-block) — sound over-approximation. + for (let b = protectedStart; b < this.builder.blockCount; b++) { + this.builder.edge(b, dispose, 'throw'); + } + + this.cfc.pop(); + drainFinalizerPending(this.builder, finFrame, [dispose]); + + // Normal completion of the body flows into the dispose; the dispose's normal + // exit is the with-statement's exit. The dispose re-propagates the exception + // path to the OUTER handler (a CM normally re-raises). + if (body) this.builder.connect(body.exits, dispose, 'seq'); + this.builder.edge(dispose, this.currentHandler(), 'throw'); + + const entry = body?.entry ?? dispose; + return { entry, exits: [dispose] }; + } + + /** The `with_item`s of a `with_statement` (under its `with_clause`). */ + private withItems(stmt: SyntaxNode): SyntaxNode[] { + const clause = stmt.namedChildren.find((c) => c.type === 'with_clause'); + if (!clause) return []; + return clause.namedChildren.filter((c) => c.type === 'with_item'); + } + + private withHeaderText(stmt: SyntaxNode): string { + const clause = stmt.namedChildren.find((c) => c.type === 'with_clause'); + return clause ? `with ${clause.text}` : 'with'; + } + + /** + * `try: BODY [except …: H]* [else: E] [finally: F]`. Mirrors the TS visitor's + * try-route-through: + * - `finally` runs on every exit (normal, exception, and early jumps) — a + * finalizer frame for early-exit threading + a normal/exceptional join. + * - each `except` / except-group handler catches from the protected body. + * - `else` runs only if the body completed with no exception. + */ + private visitTry(stmt: SyntaxNode): SeqResult { + const bodyNode = stmt.childForFieldName('body'); + const exceptClauses: SyntaxNode[] = []; + let elseClause: SyntaxNode | undefined; + let finallyClause: SyntaxNode | undefined; + for (let i = 0; i < stmt.namedChildCount; i++) { + const c = stmt.namedChild(i); + if (!c) continue; + if (c.type === 'except_clause' || c.type === 'except_group_clause') exceptClauses.push(c); + else if (c.type === 'else_clause') elseClause = c; + else if (c.type === 'finally_clause') finallyClause = c; + } + + // Build finally first — known as both a normal join and a handler target. It + // runs OUTSIDE this try's finalizer frame (a return inside finally threads + // only OUTER finallys). + const finallyBlock = finallyClause + ? this.bodyBlockOf(finallyClause) ?? finallyClause.namedChildren.find((c) => c.type === 'block') + : undefined; + const finallyRes = finallyBlock ? this.visitSeq(this.statementsOf(finallyBlock)) : null; + const finFrame = finallyRes ? this.cfc.pushFinalizer(finallyRes.entry) : null; + + // Each except handler. A `raise` inside a handler propagates to finally (if + // any), else the outer handler. + const handlerEntries: number[] = []; + const handlerExits: number[] = []; + for (const clause of exceptClauses) { + if (finallyRes) this.handlers.push(finallyRes.entry); + const handlerBlock = clause.namedChildren.find((c) => c.type === 'block'); + // The `except E as e:` header binds `e` — its own facts-only block in front + // of the handler body (the binding happens once, on handler entry). + const headFacts = this.harvest.exceptHeadFacts(clause); + const headBlock = this.builder.newBlock(startLineOf(clause), startLineOf(clause), '', 'normal', headFacts); + const bodyRes = handlerBlock ? this.visitSeq(this.statementsOf(handlerBlock)) : null; + if (bodyRes) { + this.builder.edge(headBlock, bodyRes.entry, 'seq'); + handlerExits.push(...bodyRes.exits); + } else { + handlerExits.push(headBlock); // empty handler body — header is the exit + } + handlerEntries.push(headBlock); + if (finallyRes) this.handlers.pop(); + } + + // Handler for the try body: the first except if present, else finally, else + // the outer handler. + const tryHandler = handlerEntries[0] ?? finallyRes?.entry ?? this.currentHandler(); + const protectedStart = this.builder.blockCount; + this.handlers.push(tryHandler); + const bodyRes = bodyNode ? this.visitSeq(this.statementsOf(bodyNode)) : null; + this.handlers.pop(); + + // Conservative exceptional edges: ANY protected-region block may raise to + // EACH handler (an unmatched exception type tries the next handler). + if (exceptClauses.length > 0 || finallyClause) { + const targets = + handlerEntries.length > 0 ? handlerEntries : finallyRes ? [finallyRes.entry] : []; + for (let b = protectedStart; b < this.builder.blockCount; b++) { + for (const h of targets) this.builder.edge(b, h, 'throw'); + } + } + + // The `else` runs only on no-exception normal completion of the body. + let normalAfterBody: number[] = bodyRes ? [...bodyRes.exits] : []; + if (elseClause) { + const elseRes = this.visitBody(elseClause.childForFieldName('body')); + if (elseRes && bodyRes) { + this.builder.connect(bodyRes.exits, elseRes.entry, 'seq'); + normalAfterBody = [...elseRes.exits]; + } else if (elseRes) { + normalAfterBody = [...elseRes.exits]; + } + } + + // Close the finalizer frame; wire crossing-jump completion legs. + if (finFrame && finallyRes) { + this.cfc.pop(); + drainFinalizerPending(this.builder, finFrame, finallyRes.exits); + } + + const exits: number[] = []; + if (finallyRes) { + // Normal completion of (body→else) AND each handler flows through finally. + this.builder.connect(normalAfterBody, finallyRes.entry, 'seq'); + this.builder.connect(handlerExits, finallyRes.entry, 'seq'); + exits.push(...finallyRes.exits); + // A try with no except → an uncaught exception re-propagates after finally. + if (handlerEntries.length === 0) { + this.builder.connect(finallyRes.exits, this.currentHandler(), 'throw'); + } + } else { + exits.push(...normalAfterBody); + exits.push(...handlerExits); + } + + const entry = bodyRes?.entry ?? finallyRes?.entry ?? handlerEntries[0]; + if (entry === undefined) return null; + return { entry, exits: [...new Set(exits)] }; + } + + /** + * `match SUBJECT: case P [if guard]: BODY …`. Cases do NOT fall through (like + * Go's switch). Each case body is dispatched from the subject block with a + * `switch-case` edge; a `match` with no `case _` wildcard also reaches the join + * directly (no-match path), keeping EXIT reverse-reachable. + */ + private visitMatch(stmt: SyntaxNode): TraversalResult { + const subject = stmt.childForFieldName('subject'); + const dispatch = this.builder.newBlock( + startLineOf(stmt), + subject ? endLineOf(subject) : startLineOf(stmt), + subject ? `match ${subject.text}` : 'match', + 'normal', + subject ? this.harvest.facts(subject) : undefined, + ); + const matchExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + const body = stmt.childForFieldName('body') ?? stmt.namedChildren.find((c) => c.type === 'block'); + const cases = body ? body.namedChildren.filter((c) => c.type === 'case_clause') : []; + + // A case guard (`case P if g:`) evaluates conditionally — harvest its uses + // onto the dispatch block (a later case only tests when earlier patterns + // didn't match; defs there are may-defs). + for (const c of cases) { + const guard = c.childForFieldName('guard'); + if (guard) this.builder.attachFacts(dispatch, this.harvest.factsConditional(guard)); + } + + this.cfc.pushSwitch(matchExit, []); + let hasWildcard = false; + for (const c of cases) { + const caseBody = this.visitBody(c.childForFieldName('consequence')); + const entry = caseBody?.entry ?? matchExit; + this.builder.edge(dispatch, entry, 'switch-case'); + if (caseBody) this.builder.connect(caseBody.exits, matchExit, 'seq'); + if (this.isWildcardCase(c)) hasWildcard = true; + } + this.cfc.pop(); + + // No catch-all `case _` (or no cases) → a no-match path reaches the exit + // directly. Keeps EXIT reverse-reachable even when every case body jumps. + if (!hasWildcard) this.builder.edge(dispatch, matchExit, 'switch-case'); + + return { entry: dispatch, exits: [matchExit] }; + } + + /** A `case _:` (bare wildcard with no guard) is the unconditional catch-all. */ + private isWildcardCase(caseClause: SyntaxNode): boolean { + if (caseClause.childForFieldName('guard')) return false; + const pattern = caseClause.namedChildren.find((c) => c.type === 'case_pattern'); + return pattern?.text.trim() === '_'; + } + + /** Nearest enclosing exception handler, or the function EXIT. */ + private currentHandler(): number { + return this.handlers.length ? this.handlers[this.handlers.length - 1] : this.builder.exitIndex; + } +} + +/** Build the CFG for one Python function/lambda node, or `undefined` if not modelable. */ +function buildFunctionCfg(fnNode: SyntaxNode, filePath: string): FunctionCfg | undefined { + try { + if (!PY_FUNCTION_TYPES.has(fnNode.type)) return undefined; + const startLine = startLineOf(fnNode); + const endLine = endLineOf(fnNode); + const startColumn = fnNode.startPosition.column; + + const body = fnNode.childForFieldName('body'); + if (!body) return undefined; // no body — nothing to model + + const builder = new CfgBuilder(filePath, startLine, endLine, startColumn); + const harvest = new PythonHarvester(fnNode); + + const paramFacts = harvest.paramFacts(); + if (paramFacts) builder.attachFacts(builder.entryIndex, paramFacts); + + if (fnNode.type === 'lambda' || body.type !== 'block') { + // `lambda x: expr` — the body is an EXPRESSION (no `block`): one block whose + // value is returned. Threads through no finally (a lambda has none). + const blk = builder.newBlock( + startLineOf(body), + endLineOf(body), + body.text, + 'normal', + harvest.facts(body), + ); + builder.edge(builder.entryIndex, blk, 'seq'); + builder.edge(blk, builder.exitIndex, 'return'); + return builder.finish(harvest.bindingTable()); + } + + const walk = new PythonCfgWalk(builder, harvest); + const res = walk.visitSeq(body.namedChildren.filter((c) => c.type !== 'comment')); + if (!res) { + builder.edge(builder.entryIndex, builder.exitIndex, 'seq'); // empty body + return builder.finish(harvest.bindingTable()); + } + builder.edge(builder.entryIndex, res.entry, 'seq'); + builder.connect(res.exits, builder.exitIndex, 'seq'); // normal fall-off → EXIT + return builder.finish(harvest.bindingTable()); + } catch (err) { + // Never throw out of buildFunctionCfg — a malformed AST shape must skip only + // this one function's CFG, never drop the whole file's language group (R4). + // eslint-disable-next-line no-console + console.warn(`[cfg] Python buildFunctionCfg skipped a function in ${filePath}: ${String(err)}`); + return undefined; + } +} + +/** Whether a node is a Python function this visitor builds a CFG for. */ +function isFunction(node: SyntaxNode): boolean { + return PY_FUNCTION_TYPES.has(node.type); +} + +/** The Python CFG visitor. */ +export function createPythonCfgVisitor(): CfgVisitor { + return { buildFunctionCfg, isFunction }; +} + +export { PY_FUNCTION_TYPES }; diff --git a/gitnexus/src/core/ingestion/languages/python.ts b/gitnexus/src/core/ingestion/languages/python.ts index 67eeca7e0..8ce49accd 100644 --- a/gitnexus/src/core/ingestion/languages/python.ts +++ b/gitnexus/src/core/ingestion/languages/python.ts @@ -27,6 +27,7 @@ import { createVariableExtractor } from '../variable-extractors/generic.js'; import { pythonVariableConfig } from '../variable-extractors/configs/python.js'; import { createCallExtractor } from '../call-extractors/generic.js'; import { pythonCallConfig } from '../call-extractors/configs/python.js'; +import { createPythonCfgVisitor } from '../cfg/visitors/python.js'; import type { CaptureMap } from '../language-provider.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; import { @@ -137,6 +138,7 @@ export const pythonProvider = defineLanguage({ // full per-hook rationale and the canonical capture vocabulary in // ./python/query.ts (PYTHON_SCOPE_QUERY constant). emitScopeCaptures: emitPythonScopeCaptures, + cfgVisitor: createPythonCfgVisitor(), interpretImport: interpretPythonImport, interpretTypeBinding: interpretPythonTypeBinding, bindingScopeFor: pythonBindingScopeFor, diff --git a/gitnexus/test/integration/cfg/fixtures/python-hazards.py b/gitnexus/test/integration/cfg/fixtures/python-hazards.py new file mode 100644 index 000000000..ee01a83de --- /dev/null +++ b/gitnexus/test/integration/cfg/fixtures/python-hazards.py @@ -0,0 +1,125 @@ +# Python CFG hazard fixture. Exercises every control-flow construct the Python +# visitor models, including the EXIT-reachability hazard (`while True:` with no +# static exit) the CDG soundness gate depends on, plus the structural divergences +# from the brace family: indentation blocks, elif, for/while-else, with, +# try/except/except*/else/finally, match/case, comprehensions, and the def shapes +# (tuple/list unpack, walrus, augmented assign, global/nonlocal, *args/**kwargs). + + +def if_elif_else(x): + """if / elif / else branch senses.""" + if x > 0: + r = positive() + elif x < 0: + r = negative() + else: + r = zero() + return r + + +def for_else(xs): + """for-else: the else runs on NORMAL completion, not on break.""" + found = None + for i in xs: + if matches(i): + found = i + break + else: + found = default() + return found + + +def while_else(x): + """while-else: same else-on-normal-completion / not-on-break semantics.""" + while x > 0: + if done(x): + break + x -= 1 + else: + finished() + return x + + +def while_true_loop(x): + """while True: keeps EXIT reverse-reachable (CDG soundness hazard).""" + while True: + if should_stop(x): + return collect() + x = step(x) + + +def with_dispose(path): + """with runs __exit__ on BOTH the normal and the exception exit.""" + with open(path) as fh, lock() as l: + data = fh.read() + use(l) + return process(data) + + +def with_early_return(path): + """a return inside a `with` threads through the dispose.""" + with open(path) as fh: + return fh.read() + + +def try_except_else_finally(payload): + """try / except / except* / else / finally completion edges.""" + try: + result = parse(payload) + except ValueError as e: + result = recover(e) + except (TypeError, KeyError): + result = fallback() + else: + validate(result) + finally: + cleanup() + return result + + +def try_except_group(payload): + """except* group handler.""" + try: + body(payload) + except* ValueError as eg: + handle_group(eg) + + +def match_dispatch(command): + """match / case: no fallthrough; the no-wildcard path keeps EXIT reachable.""" + match command: + case "start": + on_start() + case ["move", x, y]: + on_move(x, y) + case {"kind": k} if k > 0: + on_kind(k) + case _: + on_default() + return ack() + + +def defs_and_uses(a, b=1, *args, **kwargs): + """tuple/list unpack, walrus, augmented assign, comprehension, global.""" + global COUNTER + x, y = compute(a, b) + [p, q] = pair() + first, *rest = sequence(args) + if (n := length(kwargs)) > 0: + total = n + x + y + p + q + squares = [i * i for i in rest if i > 0] + COUNTER += 1 + return total if a else squares + + +def nested_loops_labels(matrix): + """nested loops with continue/break and a comprehension target.""" + out = [] + for row in matrix: + for cell in row: + if cell is None: + continue + if cell < 0: + break + out.append(cell) + return [v for v in out] diff --git a/gitnexus/test/unit/cfg/python-visitor.test.ts b/gitnexus/test/unit/cfg/python-visitor.test.ts new file mode 100644 index 000000000..0da2a79fe --- /dev/null +++ b/gitnexus/test/unit/cfg/python-visitor.test.ts @@ -0,0 +1,410 @@ +import { describe, it, expect } from 'vitest'; +import { createRequire } from 'node:module'; +import { createPythonCfgVisitor } from '../../../src/core/ingestion/cfg/visitors/python.js'; +import type { FunctionCfg } from '../../../src/core/ingestion/cfg/types.js'; +import { makeCfgHarness, type CfgHarness } from '../../helpers/cfg-harness.js'; +import { isExitReachableFromAllBlocks } from '../../../src/core/ingestion/cfg/post-dominators.js'; +import { computeControlDependence } from '../../../src/core/ingestion/cfg/control-dependence.js'; + +// The Python CfgVisitor — one hazard per test (real-parser regression, NOT +// snapshot-pinning). Each fixture's distinctive statement text (a(), one(), +// done(), …) lets us locate the block for a region by text and assert the +// control-flow topology around it. Python is the most structurally divergent +// target: indentation blocks, elif, for/while-else, with, try/except/else/finally, +// match/case. The `while True:` EXIT-reachability + CDG regressions are +// load-bearing. + +const pyGrammar = createRequire(import.meta.url)('tree-sitter-python') as Parameters< + typeof makeCfgHarness +>[0]; + +const py: CfgHarness = makeCfgHarness(pyGrammar, createPythonCfgVisitor(), 'fixture.py'); + +const block = (cfg: FunctionCfg, substr: string): number => { + const b = cfg.blocks.find((bl) => bl.text.includes(substr)); + if (!b) throw new Error(`no block containing ${JSON.stringify(substr)}`); + return b.index; +}; + +const edgeKinds = (cfg: FunctionCfg): Set => new Set(cfg.edges.map((e) => e.kind)); + +function reaches(cfg: FunctionCfg, from: number, to: number): boolean { + const adj = new Map(); + for (const e of cfg.edges) (adj.get(e.from) ?? adj.set(e.from, []).get(e.from)!).push(e.to); + const seen = new Set([from]); + const stack = [from]; + while (stack.length) { + const n = stack.pop() as number; + if (n === to) return true; + for (const nx of adj.get(n) ?? []) if (!seen.has(nx)) (seen.add(nx), stack.push(nx)); + } + return seen.has(to); +} +const reachable = (cfg: FunctionCfg, idx: number): boolean => reaches(cfg, cfg.entryIndex, idx); + +/** Resolve a binding by name → its index in the function's binding table. */ +function bindingIdx(cfg: FunctionCfg, name: string): number { + const i = (cfg.bindings ?? []).findIndex((b) => b.name === name); + if (i < 0) throw new Error(`no binding ${name}`); + return i; +} + +const hasDef = (cfg: FunctionCfg, idx: number): boolean => + cfg.blocks.some((bl) => bl.statements?.some((s) => s.defs.includes(idx))); +const hasUse = (cfg: FunctionCfg, idx: number): boolean => + cfg.blocks.some((bl) => bl.statements?.some((s) => s.uses.includes(idx))); +const hasMayDef = (cfg: FunctionCfg, idx: number): boolean => + cfg.blocks.some((bl) => bl.statements?.some((s) => (s.mayDefs ?? []).includes(idx))); + +/** An edge {from, to, kind} exists. */ +const hasEdge = (cfg: FunctionCfg, from: number, to: number, kind: string): boolean => + cfg.edges.some((e) => e.from === from && e.to === to && e.kind === kind); + +describe('Python CfgVisitor — structure', () => { + it('straight-line body: ENTRY → block → EXIT (seq)', () => { + const cfg = py.cfgOf(`def f():\n a()\n b()\n c()\n`); + expect(cfg.blocks.filter((b) => b.kind === 'normal')).toHaveLength(1); + const body = block(cfg, 'a()'); + expect(cfg.edges).toContainEqual({ from: cfg.entryIndex, to: body, kind: 'seq' }); + expect(reaches(cfg, body, cfg.exitIndex)).toBe(true); + }); + + it('empty body (pass only): ENTRY → block → EXIT', () => { + const cfg = py.cfgOf(`def f():\n pass\n`); + expect(reaches(cfg, cfg.entryIndex, cfg.exitIndex)).toBe(true); + }); + + it('function and lambda are both CFG-bearing', () => { + const cfgs = py.cfgsOf(`def f():\n x()\n\ng = lambda y: y + 1\n`); + expect(cfgs.length).toBeGreaterThanOrEqual(2); + for (const cfg of cfgs) expect(reaches(cfg, cfg.entryIndex, cfg.exitIndex)).toBe(true); + }); + + it('async def is the same node; its body builds a CFG', () => { + const cfg = py.cfgOf(`async def f(c):\n await c\n work()\n`); + expect(reaches(cfg, cfg.entryIndex, cfg.exitIndex)).toBe(true); + expect(reachable(cfg, block(cfg, 'work()'))).toBe(true); + }); + + it('unmodeled / no-body shape → undefined, never throws', () => { + const root = py.parse(`class C:\n x = 1\n`); + const fns = py.collectFunctions(root); + for (const fn of fns) { + expect(() => createPythonCfgVisitor().buildFunctionCfg(fn, 'f.py')).not.toThrow(); + } + }); +}); + +describe('Python CfgVisitor — if / elif / else', () => { + it('if/elif/else: branch senses; every arm reaches the join', () => { + const cfg = py.cfgOf( + `def f(x):\n if x > 0:\n a()\n elif x < 0:\n b()\n else:\n c()\n after()\n`, + ); + const kinds = edgeKinds(cfg); + expect(kinds.has('cond-true')).toBe(true); + expect(kinds.has('cond-false')).toBe(true); + const join = block(cfg, 'after()'); + expect(reaches(cfg, block(cfg, 'a()'), join)).toBe(true); + expect(reaches(cfg, block(cfg, 'b()'), join)).toBe(true); + expect(reaches(cfg, block(cfg, 'c()'), join)).toBe(true); + // the if-true arm is reached by cond-true; the elif chains on cond-false. + const ifHeader = block(cfg, 'x > 0'); + const elifHeader = block(cfg, 'x < 0'); + expect(hasEdge(cfg, ifHeader, block(cfg, 'a()'), 'cond-true')).toBe(true); + expect(hasEdge(cfg, ifHeader, elifHeader, 'cond-false')).toBe(true); + expect(hasEdge(cfg, elifHeader, block(cfg, 'b()'), 'cond-true')).toBe(true); + expect(hasEdge(cfg, elifHeader, block(cfg, 'c()'), 'cond-false')).toBe(true); + }); + + it('if with no else: cond-true to the body, seq fall-through to the join', () => { + // Mirrors the shared TS/Go visitor contract: a no-else `if` emits only the + // taken `cond-true` arm; the not-taken path falls through to the join as + // `seq`, which the CDG pass treats as the complement (false) arm. + const cfg = py.cfgOf(`def f(x):\n if x:\n a()\n after()\n`); + const header = cfg.blocks.find((b) => b.text === 'x')!.index; + const after = block(cfg, 'after()'); + expect(hasEdge(cfg, header, block(cfg, 'a()'), 'cond-true')).toBe(true); + // both the taken arm and the fall-through reach the join. + expect(reaches(cfg, header, after)).toBe(true); + expect(reaches(cfg, block(cfg, 'a()'), after)).toBe(true); + // the `if` is a real control point — CDG is non-empty. + expect(computeControlDependence(cfg).edges.length).toBeGreaterThan(0); + }); +}); + +describe('Python CfgVisitor — for / while with the loop else', () => { + it('for: header + body + loop-back; loop var is a def, iterable a use', () => { + const cfg = py.cfgOf(`def f(xs):\n for i in xs:\n step(i)\n done()\n`); + const body = block(cfg, 'step(i)'); + expect(edgeKinds(cfg).has('cond-true')).toBe(true); + expect(edgeKinds(cfg).has('loop-back')).toBe(true); + const header = cfg.edges.find((e) => e.kind === 'loop-back' && e.from === body)?.to; + expect(header).toBeDefined(); + expect(reaches(cfg, header!, block(cfg, 'done()'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'i'))).toBe(true); + expect(hasUse(cfg, bindingIdx(cfg, 'xs'))).toBe(true); + }); + + it('for-else: the else runs on NORMAL completion (cond-false), NOT on break', () => { + const cfg = py.cfgOf( + `def f(xs):\n for i in xs:\n if i:\n break\n else:\n noBreak()\n after()\n`, + ); + const header = cfg.blocks.find((b) => b.text.startsWith('for '))!.index; + const elseB = block(cfg, 'noBreak()'); + const breakB = block(cfg, 'break'); + // the else is on the cond-false (normal completion) edge from the header... + expect(hasEdge(cfg, header, elseB, 'cond-false')).toBe(true); + // ...and the break does NOT route through the else. + expect(reaches(cfg, breakB, elseB)).toBe(false); + // both the else and the break reach the post-loop join. + expect(reaches(cfg, elseB, block(cfg, 'after()'))).toBe(true); + expect(reaches(cfg, breakB, block(cfg, 'after()'))).toBe(true); + }); + + it('while-else: same else-on-normal-completion / not-on-break semantics', () => { + const cfg = py.cfgOf( + `def f(x):\n while x > 0:\n if cond():\n break\n x -= 1\n else:\n clean()\n after()\n`, + ); + const header = block(cfg, 'x > 0'); + const elseB = block(cfg, 'clean()'); + const breakB = block(cfg, 'break'); + expect(hasEdge(cfg, header, elseB, 'cond-false')).toBe(true); + expect(reaches(cfg, breakB, elseB)).toBe(false); + expect(reaches(cfg, breakB, block(cfg, 'after()'))).toBe(true); + }); + + it('while True: keeps EXIT reverse-reachable AND emits CDG > 0', () => { + const cfg = py.cfgOf(`def f(x):\n while True:\n if x:\n g()\n`); + expect(edgeKinds(cfg).has('cond-false')).toBe(true); + expect(isExitReachableFromAllBlocks(cfg)).toBe(true); + expect(computeControlDependence(cfg).edges.length).toBeGreaterThan(0); + }); + + it('continue re-tests the loop header', () => { + const cfg = py.cfgOf(`def f(xs):\n for i in xs:\n if i:\n continue\n use(i)\n`); + const header = cfg.blocks.find((b) => b.text.startsWith('for '))!.index; + expect(edgeKinds(cfg).has('continue')).toBe(true); + expect(reaches(cfg, block(cfg, 'continue'), header)).toBe(true); + }); +}); + +describe('Python CfgVisitor — with (deterministic dispose on normal AND exception exit)', () => { + it('with runs the body then __exit__ (dispose) on the normal path', () => { + const cfg = py.cfgOf(`def f():\n with open('x') as fh:\n use(fh)\n after()\n`); + const dispose = cfg.blocks.find((b) => b.text.startsWith('with '))!.index; + const body = block(cfg, 'use(fh)'); + // body → dispose → after on the normal path. + expect(reaches(cfg, body, dispose)).toBe(true); + expect(reaches(cfg, dispose, block(cfg, 'after()'))).toBe(true); + // the `as fh` alias is a def. + expect(hasDef(cfg, bindingIdx(cfg, 'fh'))).toBe(true); + }); + + it('with runs __exit__ on the EXCEPTION path too (body raises → dispose via throw)', () => { + const cfg = py.cfgOf(`def f():\n with lock() as l:\n boom()\n after()\n`); + const dispose = cfg.blocks.find((b) => b.text.startsWith('with '))!.index; + const body = block(cfg, 'boom()'); + // an exception inside the body routes to the dispose via a throw edge. + expect(hasEdge(cfg, body, dispose, 'throw')).toBe(true); + expect(edgeKinds(cfg).has('throw')).toBe(true); + }); + + it('return inside a with threads through the dispose', () => { + const cfg = py.cfgOf(`def f():\n with open('x') as fh:\n return read(fh)\n`); + const dispose = cfg.blocks.find((b) => b.text.startsWith('with '))!.index; + const ret = block(cfg, 'return read(fh)'); + expect(reaches(cfg, ret, dispose)).toBe(true); + expect(reaches(cfg, dispose, cfg.exitIndex)).toBe(true); + }); +}); + +describe('Python CfgVisitor — try / except / else / finally', () => { + it('try/except/else/finally: completion edges; finally runs on all paths', () => { + const cfg = py.cfgOf( + `def f():\n try:\n body()\n except ValueError as e:\n handle(e)\n else:\n noerr()\n finally:\n cleanup()\n after()\n`, + ); + const finallyB = block(cfg, 'cleanup()'); + expect(reaches(cfg, block(cfg, 'body()'), finallyB)).toBe(true); + expect(reaches(cfg, block(cfg, 'handle(e)'), finallyB)).toBe(true); + expect(reaches(cfg, block(cfg, 'noerr()'), finallyB)).toBe(true); + expect(reaches(cfg, finallyB, block(cfg, 'after()'))).toBe(true); + // the protected body can throw to the handler. + expect(edgeKinds(cfg).has('throw')).toBe(true); + // the else runs only after the body (no exception). + expect(reaches(cfg, block(cfg, 'body()'), block(cfg, 'noerr()'))).toBe(true); + // `except E as e` binds e (catch-kind). + expect(hasDef(cfg, bindingIdx(cfg, 'e'))).toBe(true); + }); + + it('except group (except*) is a handler from the body', () => { + const cfg = py.cfgOf( + `def f():\n try:\n body()\n except* ValueError as e:\n handle(e)\n after()\n`, + ); + expect(reaches(cfg, block(cfg, 'body()'), block(cfg, 'handle(e)'))).toBe(true); + expect(reaches(cfg, block(cfg, 'handle(e)'), block(cfg, 'after()'))).toBe(true); + expect(edgeKinds(cfg).has('throw')).toBe(true); + }); + + it('multiple except clauses are both reachable from the protected body', () => { + const cfg = py.cfgOf( + `def f():\n try:\n body()\n except ValueError:\n handleV()\n except (TypeError, KeyError):\n handleT()\n after()\n`, + ); + expect(reaches(cfg, block(cfg, 'body()'), block(cfg, 'handleV()'))).toBe(true); + expect(reaches(cfg, block(cfg, 'body()'), block(cfg, 'handleT()'))).toBe(true); + }); + + it('return inside try threads through finally', () => { + const cfg = py.cfgOf( + `def f(x):\n try:\n if x:\n return 1\n body()\n finally:\n cleanup()\n`, + ); + const finallyB = block(cfg, 'cleanup()'); + expect(reaches(cfg, block(cfg, 'return 1'), finallyB)).toBe(true); + expect(edgeKinds(cfg).has('finally-return')).toBe(true); + expect(reaches(cfg, finallyB, cfg.exitIndex)).toBe(true); + }); +}); + +describe('Python CfgVisitor — match / case (no fallthrough)', () => { + it('match dispatches across cases; no fallthrough between case bodies', () => { + const cfg = py.cfgOf( + `def f(x):\n match x:\n case 1:\n one()\n case 2:\n two()\n case _:\n other()\n after()\n`, + ); + expect(edgeKinds(cfg).has('switch-case')).toBe(true); + // each case body rejoins after the match... + expect(reaches(cfg, block(cfg, 'one()'), block(cfg, 'after()'))).toBe(true); + expect(reaches(cfg, block(cfg, 'two()'), block(cfg, 'after()'))).toBe(true); + expect(reaches(cfg, block(cfg, 'other()'), block(cfg, 'after()'))).toBe(true); + // ...but case 1 does NOT fall into case 2 (no implicit fallthrough). + expect(reaches(cfg, block(cfg, 'one()'), block(cfg, 'two()'))).toBe(false); + }); + + it('match with no wildcard keeps EXIT reverse-reachable (no-match path)', () => { + const cfg = py.cfgOf( + `def f(x):\n match x:\n case 1:\n one()\n case 2:\n two()\n after()\n`, + ); + // the dispatch reaches the exit directly (no `case _`) so after() is reachable. + const dispatch = cfg.blocks.find((b) => b.text.startsWith('match '))!.index; + expect(reaches(cfg, dispatch, block(cfg, 'after()'))).toBe(true); + expect(isExitReachableFromAllBlocks(cfg)).toBe(true); + }); + + it('a case guard is harvested as a conditional use on the dispatch', () => { + const cfg = py.cfgOf( + `def f(x, k):\n match x:\n case y if y > k:\n use(y)\n case _:\n other()\n`, + ); + expect(hasUse(cfg, bindingIdx(cfg, 'k'))).toBe(true); + }); +}); + +describe('Python CfgVisitor — def/use harvest', () => { + it('x = a then use(x): def then use', () => { + const cfg = py.cfgOf(`def f(a):\n x = a\n use(x)\n`); + const x = bindingIdx(cfg, 'x'); + expect(hasDef(cfg, x)).toBe(true); + expect(hasUse(cfg, x)).toBe(true); + }); + + it('tuple unpack a, b = f() defines BOTH a and b', () => { + const cfg = py.cfgOf(`def f():\n a, b = load()\n use(a, b)\n`); + expect(hasDef(cfg, bindingIdx(cfg, 'a'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'b'))).toBe(true); + expect(hasUse(cfg, bindingIdx(cfg, 'a'))).toBe(true); + expect(hasUse(cfg, bindingIdx(cfg, 'b'))).toBe(true); + }); + + it('list unpack [c, d] = pair() and star target first, *rest = seq() define all', () => { + const cfg = py.cfgOf(`def f():\n [c, d] = pair()\n first, *rest = seq()\n`); + expect(hasDef(cfg, bindingIdx(cfg, 'c'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'd'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'first'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'rest'))).toBe(true); + }); + + it('augmented assignment x += 1 reads AND writes the lvalue', () => { + const cfg = py.cfgOf(`def f():\n x = 1\n x += 3\n`); + const x = bindingIdx(cfg, 'x'); + expect(hasDef(cfg, x)).toBe(true); + expect(hasUse(cfg, x)).toBe(true); + }); + + it('walrus (n := f()) defines n; the condition uses it', () => { + const cfg = py.cfgOf(`def f():\n if (n := compute()) > 0:\n use(n)\n`); + const n = bindingIdx(cfg, 'n'); + expect(hasDef(cfg, n)).toBe(true); + expect(hasUse(cfg, n)).toBe(true); + }); + + it('comprehension target [i for i in xs] defines i; xs is a use', () => { + const cfg = py.cfgOf(`def f(xs):\n a = [i for i in xs if i > 0]\n use(a)\n`); + expect(hasDef(cfg, bindingIdx(cfg, 'i'))).toBe(true); + expect(hasUse(cfg, bindingIdx(cfg, 'xs'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'a'))).toBe(true); + }); + + it('with ... as t defines t; for ... in src uses src', () => { + const cfg = py.cfgOf(`def f(src):\n with src as t:\n use(t)\n`); + expect(hasDef(cfg, bindingIdx(cfg, 't'))).toBe(true); + expect(hasUse(cfg, bindingIdx(cfg, 'src'))).toBe(true); + }); + + it('a walrus in a ternary arm is a may-def (conditional context)', () => { + const cfg = py.cfgOf(`def f(c, p, q):\n x = (a := p) if c else (b := q)\n use(x)\n`); + // both arm walruses are conditionally evaluated → may-defs, not must-defs. + expect(hasMayDef(cfg, bindingIdx(cfg, 'a'))).toBe(true); + expect(hasMayDef(cfg, bindingIdx(cfg, 'b'))).toBe(true); + expect(hasDef(cfg, bindingIdx(cfg, 'a'))).toBe(false); + }); + + it('a walrus in an or/and right operand is a may-def', () => { + const cfg = py.cfgOf(`def f(a, b):\n z = a or (w := b)\n use(z)\n`); + expect(hasMayDef(cfg, bindingIdx(cfg, 'w'))).toBe(true); + }); + + it('attribute write obj.f = 1 is NOT a scalar def; root is a use', () => { + const cfg = py.cfgOf(`def f(obj):\n obj.field = 1\n use(obj)\n`); + expect(hasUse(cfg, bindingIdx(cfg, 'obj'))).toBe(true); + // no `field` binding exists (member writes are not scalar defs). + expect((cfg.bindings ?? []).some((b) => b.name === 'field')).toBe(false); + }); + + it('parameters (incl. *args / **kwargs / defaults) define at ENTRY', () => { + const cfg = py.cfgOf(`def f(a, b=1, *args, **kwargs):\n use(a, b, args, kwargs)\n`); + for (const name of ['a', 'b', 'args', 'kwargs']) { + expect(hasDef(cfg, bindingIdx(cfg, name))).toBe(true); + } + }); + + it('global names resolve to a shared synthetic module binding (no false local)', () => { + const cfg = py.cfgOf(`def f():\n global G\n G = 1\n use(G)\n`); + const g = bindingIdx(cfg, 'G'); + expect((cfg.bindings ?? [])[g].synthetic).toBe(true); + expect(hasUse(cfg, g)).toBe(true); + }); +}); + +describe('Python CfgVisitor — functionStartColumn', () => { + it('two same-line lambdas get distinct functionStartColumn', () => { + const cfgs = py.cfgsOf(`d = {"a": lambda: x(), "b": lambda: y()}\n`); + expect(cfgs.length).toBeGreaterThanOrEqual(2); + expect(cfgs[0].functionStartLine).toBe(cfgs[1].functionStartLine); + expect(cfgs[0].functionStartColumn).not.toBe(cfgs[1].functionStartColumn); + }); +}); + +describe('Python CfgVisitor — production CDG probe (plan-required)', () => { + it('while True / nested if gives exitReachable=true and CDG edges > 0', () => { + const cfg = py.cfgOf(`def f(x):\n while True:\n if x:\n g()\n`); + expect(isExitReachableFromAllBlocks(cfg)).toBe(true); + expect(computeControlDependence(cfg).edges.length).toBeGreaterThan(0); + }); +}); + +describe('Python CfgVisitor — no taint sites harvested (this unit)', () => { + it('statements carry NO sites key (taint substrate is a later step)', () => { + const cfg = py.cfgOf(`def f(cmd):\n exec(cmd)\n x = escape(cmd)\n use(x)\n`); + const anySites = cfg.blocks.some((b) => + (b.statements ?? []).some((s) => (s as { sites?: unknown }).sites !== undefined), + ); + expect(anySites).toBe(false); + }); +});