mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-29 01:41:42 +00:00
feat(group): auto-discover Python workspace cross-package contracts
Scan pyproject.toml/setup.py dependencies and `from <pkg> import` statements to find PascalCase type exports crossing workspace package boundaries. Handles hyphenated names (PEP 503 normalization), submodule imports, aliased imports, and optional-dependencies.
This commit is contained in:
parent
a537f1f153
commit
825ece09f0
3 changed files with 518 additions and 0 deletions
260
gitnexus/src/core/group/extractors/python-workspace-extractor.ts
Normal file
260
gitnexus/src/core/group/extractors/python-workspace-extractor.ts
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { CypherExecutor } from '../contract-extractor.js';
|
||||
import type { GroupManifestLink, ContractRole } from '../types.js';
|
||||
|
||||
interface PythonPackageMeta {
|
||||
name: string;
|
||||
importName: string;
|
||||
groupPath: string;
|
||||
repoPath: string;
|
||||
workspaceDeps: string[];
|
||||
}
|
||||
|
||||
interface ImportedSymbol {
|
||||
packageName: string;
|
||||
symbolName: string;
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
async function parsePythonManifest(
|
||||
repoPath: string,
|
||||
): Promise<{ name: string; importName: string; deps: string[] } | null> {
|
||||
const pyprojectPath = path.join(repoPath, 'pyproject.toml');
|
||||
let content: string | null = null;
|
||||
try {
|
||||
content = await fs.readFile(pyprojectPath, 'utf-8');
|
||||
} catch {
|
||||
// fall through to setup.py
|
||||
}
|
||||
|
||||
if (content) return parsePyproject(content);
|
||||
|
||||
const setupPyPath = path.join(repoPath, 'setup.py');
|
||||
try {
|
||||
content = await fs.readFile(setupPyPath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return parseSetupPy(content);
|
||||
}
|
||||
|
||||
function parsePyproject(content: string): { name: string; importName: string; deps: string[] } | null {
|
||||
const nameMatch = content.match(
|
||||
/^\[project\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m,
|
||||
);
|
||||
if (!nameMatch) return null;
|
||||
const name = nameMatch[1];
|
||||
const importName = name.replace(/-/g, '_');
|
||||
|
||||
const deps: string[] = [];
|
||||
const depsMatch = content.match(
|
||||
/^\[project\]\s*\n[\s\S]*?dependencies\s*=\s*\[([\s\S]*?)\]/m,
|
||||
);
|
||||
if (depsMatch) {
|
||||
const depLines = depsMatch[1].matchAll(/"([^"]+)"/g);
|
||||
for (const m of depLines) {
|
||||
deps.push(extractPepName(m[1]));
|
||||
}
|
||||
}
|
||||
|
||||
const optMatch = content.match(
|
||||
/\[project\.optional-dependencies\]\s*\n([\s\S]*?)(?=\n\[|$)/,
|
||||
);
|
||||
if (optMatch) {
|
||||
const optDeps = optMatch[1].matchAll(/"([^"]+)"/g);
|
||||
for (const m of optDeps) {
|
||||
deps.push(extractPepName(m[1]));
|
||||
}
|
||||
}
|
||||
|
||||
return { name, importName, deps: [...new Set(deps)] };
|
||||
}
|
||||
|
||||
function parseSetupPy(content: string): { name: string; importName: string; deps: string[] } | null {
|
||||
const nameMatch = content.match(/name\s*=\s*['"]([^'"]+)['"]/);
|
||||
if (!nameMatch) return null;
|
||||
const name = nameMatch[1];
|
||||
const importName = name.replace(/-/g, '_');
|
||||
|
||||
const deps: string[] = [];
|
||||
const installMatch = content.match(
|
||||
/install_requires\s*=\s*\[([\s\S]*?)\]/,
|
||||
);
|
||||
if (installMatch) {
|
||||
const depLines = installMatch[1].matchAll(/['"]([^'"]+)['"]/g);
|
||||
for (const m of depLines) {
|
||||
deps.push(extractPepName(m[1]));
|
||||
}
|
||||
}
|
||||
|
||||
return { name, importName, deps: [...new Set(deps)] };
|
||||
}
|
||||
|
||||
function extractPepName(spec: string): string {
|
||||
return spec.split(/[><=!~;\[]/)[0].trim();
|
||||
}
|
||||
|
||||
async function scanPythonImports(
|
||||
repoPath: string,
|
||||
knownPackages: Map<string, string>,
|
||||
): Promise<ImportedSymbol[]> {
|
||||
const results: ImportedSymbol[] = [];
|
||||
const sourceFiles = await findPythonFiles(repoPath);
|
||||
|
||||
for (const relFile of sourceFiles) {
|
||||
const absPath = path.join(repoPath, relFile);
|
||||
let content: string;
|
||||
try {
|
||||
content = await fs.readFile(absPath, 'utf-8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
// from <pkg> import Foo, Bar
|
||||
// from <pkg>.module import Foo
|
||||
const fromImportRegex =
|
||||
/^from\s+(\w[\w.]*)\s+import\s+(.+)/gm;
|
||||
let match;
|
||||
while ((match = fromImportRegex.exec(content)) !== null) {
|
||||
const modulePath = match[1];
|
||||
const importClause = match[2];
|
||||
const rootModule = modulePath.split('.')[0];
|
||||
const originalName = knownPackages.get(rootModule);
|
||||
if (!originalName) continue;
|
||||
|
||||
if (importClause.trim() === '(') continue;
|
||||
|
||||
const symbols = importClause
|
||||
.replace(/\(|\)/g, '')
|
||||
.split(',')
|
||||
.map((s) => {
|
||||
const trimmed = s.trim();
|
||||
const asMatch = trimmed.match(/^(\S+)\s+as\s+/);
|
||||
return asMatch ? asMatch[1] : trimmed;
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
for (const sym of symbols) {
|
||||
if (isPascalCase(sym)) {
|
||||
results.push({ packageName: originalName, symbolName: sym, filePath: relFile });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function isPascalCase(name: string): boolean {
|
||||
return /^[A-Z][A-Za-z0-9]*$/.test(name);
|
||||
}
|
||||
|
||||
async function findPythonFiles(repoPath: string): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
const IGNORE = new Set([
|
||||
'__pycache__',
|
||||
'.git',
|
||||
'.gitnexus',
|
||||
'node_modules',
|
||||
'.venv',
|
||||
'venv',
|
||||
'.tox',
|
||||
'.eggs',
|
||||
'dist',
|
||||
'build',
|
||||
'.mypy_cache',
|
||||
'.pytest_cache',
|
||||
]);
|
||||
|
||||
async function walk(dir: string, rel: string): Promise<void> {
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (IGNORE.has(entry.name)) continue;
|
||||
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
await walk(path.join(dir, entry.name), childRel);
|
||||
} else if (entry.name.endsWith('.py')) {
|
||||
results.push(childRel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await walk(repoPath, '');
|
||||
return results;
|
||||
}
|
||||
|
||||
export interface PythonWorkspaceResult {
|
||||
links: GroupManifestLink[];
|
||||
discoveredPackages: Map<string, PythonPackageMeta>;
|
||||
}
|
||||
|
||||
export async function extractPythonWorkspaceLinks(
|
||||
repos: Record<string, string>,
|
||||
repoPaths: Map<string, string>,
|
||||
_dbExecutors?: Map<string, CypherExecutor>,
|
||||
): Promise<PythonWorkspaceResult> {
|
||||
const packagesByImportName = new Map<string, PythonPackageMeta>();
|
||||
const packagesByGroupPath = new Map<string, PythonPackageMeta>();
|
||||
|
||||
for (const [groupPath] of Object.entries(repos)) {
|
||||
const repoPath = repoPaths.get(groupPath);
|
||||
if (!repoPath) continue;
|
||||
|
||||
const manifest = await parsePythonManifest(repoPath);
|
||||
if (!manifest) continue;
|
||||
|
||||
const meta: PythonPackageMeta = {
|
||||
name: manifest.name,
|
||||
importName: manifest.importName,
|
||||
groupPath,
|
||||
repoPath,
|
||||
workspaceDeps: manifest.deps,
|
||||
};
|
||||
packagesByImportName.set(manifest.importName, meta);
|
||||
packagesByGroupPath.set(groupPath, meta);
|
||||
}
|
||||
|
||||
const links: GroupManifestLink[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const [, pkg] of packagesByGroupPath) {
|
||||
const normalizedDeps = pkg.workspaceDeps.map((d) => d.replace(/-/g, '_'));
|
||||
const groupPkgDeps = normalizedDeps.filter((d) => packagesByImportName.has(d));
|
||||
if (groupPkgDeps.length === 0) continue;
|
||||
|
||||
const knownPackages = new Map<string, string>();
|
||||
for (const dep of groupPkgDeps) {
|
||||
const meta = packagesByImportName.get(dep);
|
||||
if (meta) knownPackages.set(dep, meta.name);
|
||||
}
|
||||
|
||||
const imports = await scanPythonImports(pkg.repoPath, knownPackages);
|
||||
|
||||
for (const imp of imports) {
|
||||
const providerImportName = imp.packageName.replace(/-/g, '_');
|
||||
const providerPkg = packagesByImportName.get(providerImportName);
|
||||
if (!providerPkg) continue;
|
||||
|
||||
const key = `${pkg.groupPath}→${providerPkg.groupPath}::${imp.symbolName}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
const link: GroupManifestLink = {
|
||||
from: providerPkg.groupPath,
|
||||
to: pkg.groupPath,
|
||||
type: 'custom',
|
||||
contract: imp.symbolName,
|
||||
role: 'provider' as ContractRole,
|
||||
};
|
||||
links.push(link);
|
||||
}
|
||||
}
|
||||
|
||||
return { links, discoveredPackages: packagesByGroupPath };
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import { TopicExtractor } from './extractors/topic-extractor.js';
|
|||
import { ManifestExtractor } from './extractors/manifest-extractor.js';
|
||||
import { extractRustWorkspaceLinks } from './extractors/rust-workspace-extractor.js';
|
||||
import { extractNodeWorkspaceLinks } from './extractors/node-workspace-extractor.js';
|
||||
import { extractPythonWorkspaceLinks } from './extractors/python-workspace-extractor.js';
|
||||
import { runExactMatch } from './matching.js';
|
||||
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
|
||||
import type { CypherExecutor } from './contract-extractor.js';
|
||||
|
|
@ -213,6 +214,16 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
const pyResult = await extractPythonWorkspaceLinks(config.repos, repoPaths, dbExecutors);
|
||||
if (pyResult.links.length > 0) {
|
||||
allLinks = [...allLinks, ...pyResult.links];
|
||||
if (opts?.verbose) {
|
||||
console.log(
|
||||
` workspace-deps: discovered ${pyResult.links.length} cross-package links from ${pyResult.discoveredPackages.size} Python packages`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process manifest links declared in group.yaml (plus any auto-discovered).
|
||||
|
|
|
|||
247
gitnexus/test/unit/group/python-workspace-extractor.test.ts
Normal file
247
gitnexus/test/unit/group/python-workspace-extractor.test.ts
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { extractPythonWorkspaceLinks } from '../../../src/core/group/extractors/python-workspace-extractor.js';
|
||||
|
||||
describe('PythonWorkspaceExtractor', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-py-ws-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeFile(relPath: string, content: string) {
|
||||
const absPath = path.join(tmpDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, content, 'utf-8');
|
||||
}
|
||||
|
||||
it('discovers cross-package imports via pyproject.toml', async () => {
|
||||
await writeFile(
|
||||
'models/pyproject.toml',
|
||||
'[project]\nname = "shared-models"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('models/shared_models/__init__.py', 'class Schema: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'api/pyproject.toml',
|
||||
'[project]\nname = "api-server"\nversion = "0.1.0"\ndependencies = [\n "shared-models>=0.1.0",\n]\n',
|
||||
);
|
||||
await writeFile(
|
||||
'api/api_server/main.py',
|
||||
'from shared_models import Schema\n',
|
||||
);
|
||||
|
||||
const repos = { models: 'shared-models', api: 'api-server' };
|
||||
const repoPaths = new Map([
|
||||
['models', path.join(tmpDir, 'models')],
|
||||
['api', path.join(tmpDir, 'api')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0]).toEqual({
|
||||
from: 'models',
|
||||
to: 'api',
|
||||
type: 'custom',
|
||||
contract: 'Schema',
|
||||
role: 'provider',
|
||||
});
|
||||
});
|
||||
|
||||
it('discovers imports via setup.py', async () => {
|
||||
await writeFile(
|
||||
'core/setup.py',
|
||||
"from setuptools import setup\nsetup(name='mycore', version='1.0', install_requires=[])\n",
|
||||
);
|
||||
await writeFile('core/mycore/__init__.py', 'class Engine: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/setup.py',
|
||||
"from setuptools import setup\nsetup(name='myapp', version='1.0', install_requires=['mycore>=1.0'])\n",
|
||||
);
|
||||
await writeFile('app/myapp/run.py', 'from mycore import Engine\n');
|
||||
|
||||
const repos = { core: 'mycore', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['core', path.join(tmpDir, 'core')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Engine');
|
||||
});
|
||||
|
||||
it('handles hyphenated package names (normalized to underscore in imports)', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "my-utils"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/my_utils/__init__.py', 'class Helper: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'svc/pyproject.toml',
|
||||
'[project]\nname = "my-service"\nversion = "0.1.0"\ndependencies = [\n "my-utils",\n]\n',
|
||||
);
|
||||
await writeFile('svc/my_service/main.py', 'from my_utils import Helper\n');
|
||||
|
||||
const repos = { lib: 'my-utils', svc: 'my-service' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['svc', path.join(tmpDir, 'svc')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Helper');
|
||||
});
|
||||
|
||||
it('handles submodule imports (from pkg.sub import Class)', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "datalib"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/datalib/models.py', 'class Record: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/pyproject.toml',
|
||||
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "datalib",\n]\n',
|
||||
);
|
||||
await writeFile('app/myapp/main.py', 'from datalib.models import Record\n');
|
||||
|
||||
const repos = { lib: 'datalib', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Record');
|
||||
});
|
||||
|
||||
it('ignores snake_case imports (functions, not types)', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "utils"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/utils/__init__.py', 'def helper(): pass\nclass Config: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/pyproject.toml',
|
||||
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "utils",\n]\n',
|
||||
);
|
||||
await writeFile('app/myapp/main.py', 'from utils import helper, Config\n');
|
||||
|
||||
const repos = { lib: 'utils', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Config');
|
||||
});
|
||||
|
||||
it('skips repos without Python manifest', async () => {
|
||||
await writeFile('js-app/package.json', '{"name": "js-app"}');
|
||||
|
||||
const repos = { app: 'js-app' };
|
||||
const repoPaths = new Map([['app', path.join(tmpDir, 'js-app')]]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(0);
|
||||
expect(result.discoveredPackages.size).toBe(0);
|
||||
});
|
||||
|
||||
it('deduplicates identical imports from multiple files', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "shared"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/shared/__init__.py', 'class Config: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/pyproject.toml',
|
||||
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "shared",\n]\n',
|
||||
);
|
||||
await writeFile('app/myapp/a.py', 'from shared import Config\n');
|
||||
await writeFile('app/myapp/b.py', 'from shared import Config\n');
|
||||
|
||||
const repos = { lib: 'shared', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('handles aliased imports (from pkg import Foo as Bar)', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "models"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/models/__init__.py', 'class Entity: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/pyproject.toml',
|
||||
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "models",\n]\n',
|
||||
);
|
||||
await writeFile(
|
||||
'app/myapp/main.py',
|
||||
'from models import Entity as BaseEntity\n',
|
||||
);
|
||||
|
||||
const repos = { lib: 'models', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Entity');
|
||||
});
|
||||
|
||||
it('reads optional-dependencies from pyproject.toml', async () => {
|
||||
await writeFile(
|
||||
'lib/pyproject.toml',
|
||||
'[project]\nname = "extras"\nversion = "0.1.0"\ndependencies = []\n',
|
||||
);
|
||||
await writeFile('lib/extras/__init__.py', 'class Plugin: pass\n');
|
||||
|
||||
await writeFile(
|
||||
'app/pyproject.toml',
|
||||
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = []\n\n[project.optional-dependencies]\ndev = [\n "extras>=0.1",\n]\n',
|
||||
);
|
||||
await writeFile('app/myapp/main.py', 'from extras import Plugin\n');
|
||||
|
||||
const repos = { lib: 'extras', app: 'myapp' };
|
||||
const repoPaths = new Map([
|
||||
['lib', path.join(tmpDir, 'lib')],
|
||||
['app', path.join(tmpDir, 'app')],
|
||||
]);
|
||||
|
||||
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
|
||||
|
||||
expect(result.links).toHaveLength(1);
|
||||
expect(result.links[0].contract).toBe('Plugin');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue