diff --git a/gitnexus/src/core/group/extractors/python-workspace-extractor.ts b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts new file mode 100644 index 000000000..2707d4c46 --- /dev/null +++ b/gitnexus/src/core/group/extractors/python-workspace-extractor.ts @@ -0,0 +1,260 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { CypherExecutor } from '../contract-extractor.js'; +import type { GroupManifestLink, ContractRole } from '../types.js'; + +interface PythonPackageMeta { + name: string; + importName: string; + groupPath: string; + repoPath: string; + workspaceDeps: string[]; +} + +interface ImportedSymbol { + packageName: string; + symbolName: string; + filePath: string; +} + +async function parsePythonManifest( + repoPath: string, +): Promise<{ name: string; importName: string; deps: string[] } | null> { + const pyprojectPath = path.join(repoPath, 'pyproject.toml'); + let content: string | null = null; + try { + content = await fs.readFile(pyprojectPath, 'utf-8'); + } catch { + // fall through to setup.py + } + + if (content) return parsePyproject(content); + + const setupPyPath = path.join(repoPath, 'setup.py'); + try { + content = await fs.readFile(setupPyPath, 'utf-8'); + } catch { + return null; + } + return parseSetupPy(content); +} + +function parsePyproject(content: string): { name: string; importName: string; deps: string[] } | null { + const nameMatch = content.match( + /^\[project\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m, + ); + if (!nameMatch) return null; + const name = nameMatch[1]; + const importName = name.replace(/-/g, '_'); + + const deps: string[] = []; + const depsMatch = content.match( + /^\[project\]\s*\n[\s\S]*?dependencies\s*=\s*\[([\s\S]*?)\]/m, + ); + if (depsMatch) { + const depLines = depsMatch[1].matchAll(/"([^"]+)"/g); + for (const m of depLines) { + deps.push(extractPepName(m[1])); + } + } + + const optMatch = content.match( + /\[project\.optional-dependencies\]\s*\n([\s\S]*?)(?=\n\[|$)/, + ); + if (optMatch) { + const optDeps = optMatch[1].matchAll(/"([^"]+)"/g); + for (const m of optDeps) { + deps.push(extractPepName(m[1])); + } + } + + return { name, importName, deps: [...new Set(deps)] }; +} + +function parseSetupPy(content: string): { name: string; importName: string; deps: string[] } | null { + const nameMatch = content.match(/name\s*=\s*['"]([^'"]+)['"]/); + if (!nameMatch) return null; + const name = nameMatch[1]; + const importName = name.replace(/-/g, '_'); + + const deps: string[] = []; + const installMatch = content.match( + /install_requires\s*=\s*\[([\s\S]*?)\]/, + ); + if (installMatch) { + const depLines = installMatch[1].matchAll(/['"]([^'"]+)['"]/g); + for (const m of depLines) { + deps.push(extractPepName(m[1])); + } + } + + return { name, importName, deps: [...new Set(deps)] }; +} + +function extractPepName(spec: string): string { + return spec.split(/[><=!~;\[]/)[0].trim(); +} + +async function scanPythonImports( + repoPath: string, + knownPackages: Map, +): Promise { + const results: ImportedSymbol[] = []; + const sourceFiles = await findPythonFiles(repoPath); + + for (const relFile of sourceFiles) { + const absPath = path.join(repoPath, relFile); + let content: string; + try { + content = await fs.readFile(absPath, 'utf-8'); + } catch { + continue; + } + + // from import Foo, Bar + // from .module import Foo + const fromImportRegex = + /^from\s+(\w[\w.]*)\s+import\s+(.+)/gm; + let match; + while ((match = fromImportRegex.exec(content)) !== null) { + const modulePath = match[1]; + const importClause = match[2]; + const rootModule = modulePath.split('.')[0]; + const originalName = knownPackages.get(rootModule); + if (!originalName) continue; + + if (importClause.trim() === '(') continue; + + const symbols = importClause + .replace(/\(|\)/g, '') + .split(',') + .map((s) => { + const trimmed = s.trim(); + const asMatch = trimmed.match(/^(\S+)\s+as\s+/); + return asMatch ? asMatch[1] : trimmed; + }) + .filter(Boolean); + + for (const sym of symbols) { + if (isPascalCase(sym)) { + results.push({ packageName: originalName, symbolName: sym, filePath: relFile }); + } + } + } + } + + return results; +} + +function isPascalCase(name: string): boolean { + return /^[A-Z][A-Za-z0-9]*$/.test(name); +} + +async function findPythonFiles(repoPath: string): Promise { + const results: string[] = []; + const IGNORE = new Set([ + '__pycache__', + '.git', + '.gitnexus', + 'node_modules', + '.venv', + 'venv', + '.tox', + '.eggs', + 'dist', + 'build', + '.mypy_cache', + '.pytest_cache', + ]); + + async function walk(dir: string, rel: string): Promise { + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (IGNORE.has(entry.name)) continue; + const childRel = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + await walk(path.join(dir, entry.name), childRel); + } else if (entry.name.endsWith('.py')) { + results.push(childRel); + } + } + } + + await walk(repoPath, ''); + return results; +} + +export interface PythonWorkspaceResult { + links: GroupManifestLink[]; + discoveredPackages: Map; +} + +export async function extractPythonWorkspaceLinks( + repos: Record, + repoPaths: Map, + _dbExecutors?: Map, +): Promise { + const packagesByImportName = new Map(); + const packagesByGroupPath = new Map(); + + for (const [groupPath] of Object.entries(repos)) { + const repoPath = repoPaths.get(groupPath); + if (!repoPath) continue; + + const manifest = await parsePythonManifest(repoPath); + if (!manifest) continue; + + const meta: PythonPackageMeta = { + name: manifest.name, + importName: manifest.importName, + groupPath, + repoPath, + workspaceDeps: manifest.deps, + }; + packagesByImportName.set(manifest.importName, meta); + packagesByGroupPath.set(groupPath, meta); + } + + const links: GroupManifestLink[] = []; + const seen = new Set(); + + for (const [, pkg] of packagesByGroupPath) { + const normalizedDeps = pkg.workspaceDeps.map((d) => d.replace(/-/g, '_')); + const groupPkgDeps = normalizedDeps.filter((d) => packagesByImportName.has(d)); + if (groupPkgDeps.length === 0) continue; + + const knownPackages = new Map(); + for (const dep of groupPkgDeps) { + const meta = packagesByImportName.get(dep); + if (meta) knownPackages.set(dep, meta.name); + } + + const imports = await scanPythonImports(pkg.repoPath, knownPackages); + + for (const imp of imports) { + const providerImportName = imp.packageName.replace(/-/g, '_'); + const providerPkg = packagesByImportName.get(providerImportName); + if (!providerPkg) continue; + + const key = `${pkg.groupPath}→${providerPkg.groupPath}::${imp.symbolName}`; + if (seen.has(key)) continue; + seen.add(key); + + const link: GroupManifestLink = { + from: providerPkg.groupPath, + to: pkg.groupPath, + type: 'custom', + contract: imp.symbolName, + role: 'provider' as ContractRole, + }; + links.push(link); + } + } + + return { links, discoveredPackages: packagesByGroupPath }; +} diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 918811128..1068c03b0 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -10,6 +10,7 @@ import { TopicExtractor } from './extractors/topic-extractor.js'; import { ManifestExtractor } from './extractors/manifest-extractor.js'; import { extractRustWorkspaceLinks } from './extractors/rust-workspace-extractor.js'; import { extractNodeWorkspaceLinks } from './extractors/node-workspace-extractor.js'; +import { extractPythonWorkspaceLinks } from './extractors/python-workspace-extractor.js'; import { runExactMatch } from './matching.js'; import { detectServiceBoundaries, assignService } from './service-boundary-detector.js'; import type { CypherExecutor } from './contract-extractor.js'; @@ -213,6 +214,16 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis ); } } + + const pyResult = await extractPythonWorkspaceLinks(config.repos, repoPaths, dbExecutors); + if (pyResult.links.length > 0) { + allLinks = [...allLinks, ...pyResult.links]; + if (opts?.verbose) { + console.log( + ` workspace-deps: discovered ${pyResult.links.length} cross-package links from ${pyResult.discoveredPackages.size} Python packages`, + ); + } + } } // Process manifest links declared in group.yaml (plus any auto-discovered). diff --git a/gitnexus/test/unit/group/python-workspace-extractor.test.ts b/gitnexus/test/unit/group/python-workspace-extractor.test.ts new file mode 100644 index 000000000..05394a490 --- /dev/null +++ b/gitnexus/test/unit/group/python-workspace-extractor.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; +import { extractPythonWorkspaceLinks } from '../../../src/core/group/extractors/python-workspace-extractor.js'; + +describe('PythonWorkspaceExtractor', () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-py-ws-')); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + async function writeFile(relPath: string, content: string) { + const absPath = path.join(tmpDir, relPath); + await fs.mkdir(path.dirname(absPath), { recursive: true }); + await fs.writeFile(absPath, content, 'utf-8'); + } + + it('discovers cross-package imports via pyproject.toml', async () => { + await writeFile( + 'models/pyproject.toml', + '[project]\nname = "shared-models"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('models/shared_models/__init__.py', 'class Schema: pass\n'); + + await writeFile( + 'api/pyproject.toml', + '[project]\nname = "api-server"\nversion = "0.1.0"\ndependencies = [\n "shared-models>=0.1.0",\n]\n', + ); + await writeFile( + 'api/api_server/main.py', + 'from shared_models import Schema\n', + ); + + const repos = { models: 'shared-models', api: 'api-server' }; + const repoPaths = new Map([ + ['models', path.join(tmpDir, 'models')], + ['api', path.join(tmpDir, 'api')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0]).toEqual({ + from: 'models', + to: 'api', + type: 'custom', + contract: 'Schema', + role: 'provider', + }); + }); + + it('discovers imports via setup.py', async () => { + await writeFile( + 'core/setup.py', + "from setuptools import setup\nsetup(name='mycore', version='1.0', install_requires=[])\n", + ); + await writeFile('core/mycore/__init__.py', 'class Engine: pass\n'); + + await writeFile( + 'app/setup.py', + "from setuptools import setup\nsetup(name='myapp', version='1.0', install_requires=['mycore>=1.0'])\n", + ); + await writeFile('app/myapp/run.py', 'from mycore import Engine\n'); + + const repos = { core: 'mycore', app: 'myapp' }; + const repoPaths = new Map([ + ['core', path.join(tmpDir, 'core')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Engine'); + }); + + it('handles hyphenated package names (normalized to underscore in imports)', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "my-utils"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/my_utils/__init__.py', 'class Helper: pass\n'); + + await writeFile( + 'svc/pyproject.toml', + '[project]\nname = "my-service"\nversion = "0.1.0"\ndependencies = [\n "my-utils",\n]\n', + ); + await writeFile('svc/my_service/main.py', 'from my_utils import Helper\n'); + + const repos = { lib: 'my-utils', svc: 'my-service' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['svc', path.join(tmpDir, 'svc')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Helper'); + }); + + it('handles submodule imports (from pkg.sub import Class)', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "datalib"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/datalib/models.py', 'class Record: pass\n'); + + await writeFile( + 'app/pyproject.toml', + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "datalib",\n]\n', + ); + await writeFile('app/myapp/main.py', 'from datalib.models import Record\n'); + + const repos = { lib: 'datalib', app: 'myapp' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Record'); + }); + + it('ignores snake_case imports (functions, not types)', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "utils"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/utils/__init__.py', 'def helper(): pass\nclass Config: pass\n'); + + await writeFile( + 'app/pyproject.toml', + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "utils",\n]\n', + ); + await writeFile('app/myapp/main.py', 'from utils import helper, Config\n'); + + const repos = { lib: 'utils', app: 'myapp' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Config'); + }); + + it('skips repos without Python manifest', async () => { + await writeFile('js-app/package.json', '{"name": "js-app"}'); + + const repos = { app: 'js-app' }; + const repoPaths = new Map([['app', path.join(tmpDir, 'js-app')]]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(0); + expect(result.discoveredPackages.size).toBe(0); + }); + + it('deduplicates identical imports from multiple files', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "shared"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/shared/__init__.py', 'class Config: pass\n'); + + await writeFile( + 'app/pyproject.toml', + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "shared",\n]\n', + ); + await writeFile('app/myapp/a.py', 'from shared import Config\n'); + await writeFile('app/myapp/b.py', 'from shared import Config\n'); + + const repos = { lib: 'shared', app: 'myapp' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + }); + + it('handles aliased imports (from pkg import Foo as Bar)', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "models"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/models/__init__.py', 'class Entity: pass\n'); + + await writeFile( + 'app/pyproject.toml', + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "models",\n]\n', + ); + await writeFile( + 'app/myapp/main.py', + 'from models import Entity as BaseEntity\n', + ); + + const repos = { lib: 'models', app: 'myapp' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Entity'); + }); + + it('reads optional-dependencies from pyproject.toml', async () => { + await writeFile( + 'lib/pyproject.toml', + '[project]\nname = "extras"\nversion = "0.1.0"\ndependencies = []\n', + ); + await writeFile('lib/extras/__init__.py', 'class Plugin: pass\n'); + + await writeFile( + 'app/pyproject.toml', + '[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = []\n\n[project.optional-dependencies]\ndev = [\n "extras>=0.1",\n]\n', + ); + await writeFile('app/myapp/main.py', 'from extras import Plugin\n'); + + const repos = { lib: 'extras', app: 'myapp' }; + const repoPaths = new Map([ + ['lib', path.join(tmpDir, 'lib')], + ['app', path.join(tmpDir, 'app')], + ]); + + const result = await extractPythonWorkspaceLinks(repos, repoPaths); + + expect(result.links).toHaveLength(1); + expect(result.links[0].contract).toBe('Plugin'); + }); +});