feat(group): auto-discover Python workspace cross-package contracts

Scan pyproject.toml/setup.py dependencies and `from <pkg> import`
statements to find PascalCase type exports crossing workspace package
boundaries. Handles hyphenated names (PEP 503 normalization),
submodule imports, aliased imports, and optional-dependencies.
This commit is contained in:
Christian C. Berclaz 2026-05-01 19:44:58 +02:00
parent a537f1f153
commit 825ece09f0
No known key found for this signature in database
3 changed files with 518 additions and 0 deletions

View file

@ -0,0 +1,260 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import type { CypherExecutor } from '../contract-extractor.js';
import type { GroupManifestLink, ContractRole } from '../types.js';
interface PythonPackageMeta {
name: string;
importName: string;
groupPath: string;
repoPath: string;
workspaceDeps: string[];
}
interface ImportedSymbol {
packageName: string;
symbolName: string;
filePath: string;
}
async function parsePythonManifest(
repoPath: string,
): Promise<{ name: string; importName: string; deps: string[] } | null> {
const pyprojectPath = path.join(repoPath, 'pyproject.toml');
let content: string | null = null;
try {
content = await fs.readFile(pyprojectPath, 'utf-8');
} catch {
// fall through to setup.py
}
if (content) return parsePyproject(content);
const setupPyPath = path.join(repoPath, 'setup.py');
try {
content = await fs.readFile(setupPyPath, 'utf-8');
} catch {
return null;
}
return parseSetupPy(content);
}
function parsePyproject(content: string): { name: string; importName: string; deps: string[] } | null {
const nameMatch = content.match(
/^\[project\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m,
);
if (!nameMatch) return null;
const name = nameMatch[1];
const importName = name.replace(/-/g, '_');
const deps: string[] = [];
const depsMatch = content.match(
/^\[project\]\s*\n[\s\S]*?dependencies\s*=\s*\[([\s\S]*?)\]/m,
);
if (depsMatch) {
const depLines = depsMatch[1].matchAll(/"([^"]+)"/g);
for (const m of depLines) {
deps.push(extractPepName(m[1]));
}
}
const optMatch = content.match(
/\[project\.optional-dependencies\]\s*\n([\s\S]*?)(?=\n\[|$)/,
);
if (optMatch) {
const optDeps = optMatch[1].matchAll(/"([^"]+)"/g);
for (const m of optDeps) {
deps.push(extractPepName(m[1]));
}
}
return { name, importName, deps: [...new Set(deps)] };
}
function parseSetupPy(content: string): { name: string; importName: string; deps: string[] } | null {
const nameMatch = content.match(/name\s*=\s*['"]([^'"]+)['"]/);
if (!nameMatch) return null;
const name = nameMatch[1];
const importName = name.replace(/-/g, '_');
const deps: string[] = [];
const installMatch = content.match(
/install_requires\s*=\s*\[([\s\S]*?)\]/,
);
if (installMatch) {
const depLines = installMatch[1].matchAll(/['"]([^'"]+)['"]/g);
for (const m of depLines) {
deps.push(extractPepName(m[1]));
}
}
return { name, importName, deps: [...new Set(deps)] };
}
function extractPepName(spec: string): string {
return spec.split(/[><=!~;\[]/)[0].trim();
}
async function scanPythonImports(
repoPath: string,
knownPackages: Map<string, string>,
): Promise<ImportedSymbol[]> {
const results: ImportedSymbol[] = [];
const sourceFiles = await findPythonFiles(repoPath);
for (const relFile of sourceFiles) {
const absPath = path.join(repoPath, relFile);
let content: string;
try {
content = await fs.readFile(absPath, 'utf-8');
} catch {
continue;
}
// from <pkg> import Foo, Bar
// from <pkg>.module import Foo
const fromImportRegex =
/^from\s+(\w[\w.]*)\s+import\s+(.+)/gm;
let match;
while ((match = fromImportRegex.exec(content)) !== null) {
const modulePath = match[1];
const importClause = match[2];
const rootModule = modulePath.split('.')[0];
const originalName = knownPackages.get(rootModule);
if (!originalName) continue;
if (importClause.trim() === '(') continue;
const symbols = importClause
.replace(/\(|\)/g, '')
.split(',')
.map((s) => {
const trimmed = s.trim();
const asMatch = trimmed.match(/^(\S+)\s+as\s+/);
return asMatch ? asMatch[1] : trimmed;
})
.filter(Boolean);
for (const sym of symbols) {
if (isPascalCase(sym)) {
results.push({ packageName: originalName, symbolName: sym, filePath: relFile });
}
}
}
}
return results;
}
function isPascalCase(name: string): boolean {
return /^[A-Z][A-Za-z0-9]*$/.test(name);
}
async function findPythonFiles(repoPath: string): Promise<string[]> {
const results: string[] = [];
const IGNORE = new Set([
'__pycache__',
'.git',
'.gitnexus',
'node_modules',
'.venv',
'venv',
'.tox',
'.eggs',
'dist',
'build',
'.mypy_cache',
'.pytest_cache',
]);
async function walk(dir: string, rel: string): Promise<void> {
let entries;
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
if (IGNORE.has(entry.name)) continue;
const childRel = rel ? `${rel}/${entry.name}` : entry.name;
if (entry.isDirectory()) {
await walk(path.join(dir, entry.name), childRel);
} else if (entry.name.endsWith('.py')) {
results.push(childRel);
}
}
}
await walk(repoPath, '');
return results;
}
export interface PythonWorkspaceResult {
links: GroupManifestLink[];
discoveredPackages: Map<string, PythonPackageMeta>;
}
export async function extractPythonWorkspaceLinks(
repos: Record<string, string>,
repoPaths: Map<string, string>,
_dbExecutors?: Map<string, CypherExecutor>,
): Promise<PythonWorkspaceResult> {
const packagesByImportName = new Map<string, PythonPackageMeta>();
const packagesByGroupPath = new Map<string, PythonPackageMeta>();
for (const [groupPath] of Object.entries(repos)) {
const repoPath = repoPaths.get(groupPath);
if (!repoPath) continue;
const manifest = await parsePythonManifest(repoPath);
if (!manifest) continue;
const meta: PythonPackageMeta = {
name: manifest.name,
importName: manifest.importName,
groupPath,
repoPath,
workspaceDeps: manifest.deps,
};
packagesByImportName.set(manifest.importName, meta);
packagesByGroupPath.set(groupPath, meta);
}
const links: GroupManifestLink[] = [];
const seen = new Set<string>();
for (const [, pkg] of packagesByGroupPath) {
const normalizedDeps = pkg.workspaceDeps.map((d) => d.replace(/-/g, '_'));
const groupPkgDeps = normalizedDeps.filter((d) => packagesByImportName.has(d));
if (groupPkgDeps.length === 0) continue;
const knownPackages = new Map<string, string>();
for (const dep of groupPkgDeps) {
const meta = packagesByImportName.get(dep);
if (meta) knownPackages.set(dep, meta.name);
}
const imports = await scanPythonImports(pkg.repoPath, knownPackages);
for (const imp of imports) {
const providerImportName = imp.packageName.replace(/-/g, '_');
const providerPkg = packagesByImportName.get(providerImportName);
if (!providerPkg) continue;
const key = `${pkg.groupPath}→${providerPkg.groupPath}::${imp.symbolName}`;
if (seen.has(key)) continue;
seen.add(key);
const link: GroupManifestLink = {
from: providerPkg.groupPath,
to: pkg.groupPath,
type: 'custom',
contract: imp.symbolName,
role: 'provider' as ContractRole,
};
links.push(link);
}
}
return { links, discoveredPackages: packagesByGroupPath };
}

View file

@ -10,6 +10,7 @@ import { TopicExtractor } from './extractors/topic-extractor.js';
import { ManifestExtractor } from './extractors/manifest-extractor.js';
import { extractRustWorkspaceLinks } from './extractors/rust-workspace-extractor.js';
import { extractNodeWorkspaceLinks } from './extractors/node-workspace-extractor.js';
import { extractPythonWorkspaceLinks } from './extractors/python-workspace-extractor.js';
import { runExactMatch } from './matching.js';
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
import type { CypherExecutor } from './contract-extractor.js';
@ -213,6 +214,16 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
);
}
}
const pyResult = await extractPythonWorkspaceLinks(config.repos, repoPaths, dbExecutors);
if (pyResult.links.length > 0) {
allLinks = [...allLinks, ...pyResult.links];
if (opts?.verbose) {
console.log(
` workspace-deps: discovered ${pyResult.links.length} cross-package links from ${pyResult.discoveredPackages.size} Python packages`,
);
}
}
}
// Process manifest links declared in group.yaml (plus any auto-discovered).

View file

@ -0,0 +1,247 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fs from 'node:fs/promises';
import path from 'node:path';
import os from 'node:os';
import { extractPythonWorkspaceLinks } from '../../../src/core/group/extractors/python-workspace-extractor.js';
describe('PythonWorkspaceExtractor', () => {
let tmpDir: string;
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-py-ws-'));
});
afterEach(async () => {
await fs.rm(tmpDir, { recursive: true, force: true });
});
async function writeFile(relPath: string, content: string) {
const absPath = path.join(tmpDir, relPath);
await fs.mkdir(path.dirname(absPath), { recursive: true });
await fs.writeFile(absPath, content, 'utf-8');
}
it('discovers cross-package imports via pyproject.toml', async () => {
await writeFile(
'models/pyproject.toml',
'[project]\nname = "shared-models"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('models/shared_models/__init__.py', 'class Schema: pass\n');
await writeFile(
'api/pyproject.toml',
'[project]\nname = "api-server"\nversion = "0.1.0"\ndependencies = [\n "shared-models>=0.1.0",\n]\n',
);
await writeFile(
'api/api_server/main.py',
'from shared_models import Schema\n',
);
const repos = { models: 'shared-models', api: 'api-server' };
const repoPaths = new Map([
['models', path.join(tmpDir, 'models')],
['api', path.join(tmpDir, 'api')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0]).toEqual({
from: 'models',
to: 'api',
type: 'custom',
contract: 'Schema',
role: 'provider',
});
});
it('discovers imports via setup.py', async () => {
await writeFile(
'core/setup.py',
"from setuptools import setup\nsetup(name='mycore', version='1.0', install_requires=[])\n",
);
await writeFile('core/mycore/__init__.py', 'class Engine: pass\n');
await writeFile(
'app/setup.py',
"from setuptools import setup\nsetup(name='myapp', version='1.0', install_requires=['mycore>=1.0'])\n",
);
await writeFile('app/myapp/run.py', 'from mycore import Engine\n');
const repos = { core: 'mycore', app: 'myapp' };
const repoPaths = new Map([
['core', path.join(tmpDir, 'core')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Engine');
});
it('handles hyphenated package names (normalized to underscore in imports)', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "my-utils"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/my_utils/__init__.py', 'class Helper: pass\n');
await writeFile(
'svc/pyproject.toml',
'[project]\nname = "my-service"\nversion = "0.1.0"\ndependencies = [\n "my-utils",\n]\n',
);
await writeFile('svc/my_service/main.py', 'from my_utils import Helper\n');
const repos = { lib: 'my-utils', svc: 'my-service' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['svc', path.join(tmpDir, 'svc')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Helper');
});
it('handles submodule imports (from pkg.sub import Class)', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "datalib"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/datalib/models.py', 'class Record: pass\n');
await writeFile(
'app/pyproject.toml',
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "datalib",\n]\n',
);
await writeFile('app/myapp/main.py', 'from datalib.models import Record\n');
const repos = { lib: 'datalib', app: 'myapp' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Record');
});
it('ignores snake_case imports (functions, not types)', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "utils"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/utils/__init__.py', 'def helper(): pass\nclass Config: pass\n');
await writeFile(
'app/pyproject.toml',
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "utils",\n]\n',
);
await writeFile('app/myapp/main.py', 'from utils import helper, Config\n');
const repos = { lib: 'utils', app: 'myapp' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Config');
});
it('skips repos without Python manifest', async () => {
await writeFile('js-app/package.json', '{"name": "js-app"}');
const repos = { app: 'js-app' };
const repoPaths = new Map([['app', path.join(tmpDir, 'js-app')]]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(0);
expect(result.discoveredPackages.size).toBe(0);
});
it('deduplicates identical imports from multiple files', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "shared"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/shared/__init__.py', 'class Config: pass\n');
await writeFile(
'app/pyproject.toml',
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "shared",\n]\n',
);
await writeFile('app/myapp/a.py', 'from shared import Config\n');
await writeFile('app/myapp/b.py', 'from shared import Config\n');
const repos = { lib: 'shared', app: 'myapp' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
});
it('handles aliased imports (from pkg import Foo as Bar)', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "models"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/models/__init__.py', 'class Entity: pass\n');
await writeFile(
'app/pyproject.toml',
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = [\n "models",\n]\n',
);
await writeFile(
'app/myapp/main.py',
'from models import Entity as BaseEntity\n',
);
const repos = { lib: 'models', app: 'myapp' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Entity');
});
it('reads optional-dependencies from pyproject.toml', async () => {
await writeFile(
'lib/pyproject.toml',
'[project]\nname = "extras"\nversion = "0.1.0"\ndependencies = []\n',
);
await writeFile('lib/extras/__init__.py', 'class Plugin: pass\n');
await writeFile(
'app/pyproject.toml',
'[project]\nname = "myapp"\nversion = "0.1.0"\ndependencies = []\n\n[project.optional-dependencies]\ndev = [\n "extras>=0.1",\n]\n',
);
await writeFile('app/myapp/main.py', 'from extras import Plugin\n');
const repos = { lib: 'extras', app: 'myapp' };
const repoPaths = new Map([
['lib', path.join(tmpDir, 'lib')],
['app', path.join(tmpDir, 'app')],
]);
const result = await extractPythonWorkspaceLinks(repos, repoPaths);
expect(result.links).toHaveLength(1);
expect(result.links[0].contract).toBe('Plugin');
});
});