diff --git a/src/services/package-manager/GitFetcher.ts b/src/services/package-manager/GitFetcher.ts index 5eba83db3c..c07bd1f52e 100644 --- a/src/services/package-manager/GitFetcher.ts +++ b/src/services/package-manager/GitFetcher.ts @@ -68,12 +68,17 @@ export class GitFetcher { const metadata = await this.parseRepositoryMetadata(repoDir) // Parse package manager items + // Get current branch + const git = simpleGit(repoDir) + const branch = await git.revparse(["--abbrev-ref", "HEAD"]) + const items = await this.parsePackageManagerItems(repoDir, repoUrl, sourceName || metadata.name) return { metadata, - items, + items: items.map((item) => ({ ...item, defaultBranch: branch })), url: repoUrl, + defaultBranch: branch, } } diff --git a/src/services/package-manager/MetadataScanner.ts b/src/services/package-manager/MetadataScanner.ts index 0e7277ec36..1659abaaca 100644 --- a/src/services/package-manager/MetadataScanner.ts +++ b/src/services/package-manager/MetadataScanner.ts @@ -20,6 +20,7 @@ import { getUserLocale } from "./utils" export class MetadataScanner { private readonly git?: SimpleGit private localizationOptions: LocalizationOptions + private originalRootDir: string | null = null constructor(git?: SimpleGit, localizationOptions?: LocalizationOptions) { this.git = git @@ -36,9 +37,22 @@ export class MetadataScanner { * @param sourceName Optional source repository name * @returns Array of discovered items */ - async scanDirectory(rootDir: string, repoUrl: string, sourceName?: string): Promise { + async scanDirectory( + rootDir: string, + repoUrl: string, + sourceName?: string, + isRecursiveCall: boolean = false, + ): Promise { const items: PackageManagerItem[] = [] + console.log("scanDirectory called with rootDir:", rootDir) + console.log("scanDirectory called with repoUrl:", repoUrl) + + // Only set originalRootDir on the first call, not recursive calls + if (!isRecursiveCall && !this.originalRootDir) { + this.originalRootDir = rootDir + } + try { const entries = await fs.readdir(rootDir, { withFileTypes: true }) @@ -46,6 +60,12 @@ export class MetadataScanner { if (!entry.isDirectory()) continue const componentDir = path.join(rootDir, entry.name) + console.log("scanDirectory - entry.name:", entry.name) + console.log("scanDirectory - rootDir:", rootDir) + console.log("scanDirectory - componentDir:", componentDir) + // Always calculate paths relative to the original root directory + const relativePath = path.relative(this.originalRootDir || rootDir, componentDir).replace(/\\/g, "/") + console.log("scanDirectory - relativePath:", relativePath) console.log("Checking directory:", componentDir) const metadata = await this.loadComponentMetadata(componentDir) console.log("Found metadata:", metadata) @@ -53,7 +73,8 @@ export class MetadataScanner { // If no metadata found, or metadata validation fails, try recursing if (!metadata || !this.getLocalizedMetadata(metadata)) { console.log("No valid metadata found, recursing into:", componentDir) - const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName) + // Pass the current directory as the root for this recursive call + const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName, true) items.push(...subItems) continue } @@ -62,7 +83,14 @@ export class MetadataScanner { const localizedMetadata = this.getLocalizedMetadata(metadata) if (!localizedMetadata) continue - const item = await this.createPackageManagerItem(localizedMetadata, componentDir, repoUrl, sourceName) + // Always use the original root directory for path calculations + const item = await this.createPackageManagerItem( + localizedMetadata, + componentDir, + repoUrl, + this.originalRootDir || rootDir, + sourceName, + ) if (item) { // If this is a package, scan for subcomponents if (this.isPackageMetadata(localizedMetadata)) { @@ -71,6 +99,7 @@ export class MetadataScanner { const subcomponents = await Promise.all( localizedMetadata.items.map(async (subItem) => { const subPath = path.join(componentDir, subItem.path) + const subRelativePath = path.relative(rootDir, subPath).replace(/\\/g, "/") const subMetadata = await this.loadComponentMetadata(subPath) // Skip if no metadata found @@ -104,7 +133,12 @@ export class MetadataScanner { // Recursively scan subdirectories only if not in a package if (!metadata || !this.isPackageMetadata(localizedMetadata)) { console.log("Recursing into directory:", componentDir) - const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName) + console.log("Current relativePath:", relativePath) + console.log("Current rootDir:", rootDir) + const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName, true) + console.log("Received subItems:", subItems) + // No need to modify paths for recursive items + // They are already correctly calculated relative to the original root console.log("Found sub items:", subItems) items.push(...subItems) } @@ -210,22 +244,40 @@ export class MetadataScanner { metadata: ComponentMetadata, componentDir: string, repoUrl: string, + rootDir: string, sourceName?: string, ): Promise { // Skip if no type or invalid type if (!metadata.type || !this.isValidComponentType(metadata.type)) { return null } + // Always use the original root directory for path calculations + const effectiveRootDir = this.originalRootDir || rootDir + // Always calculate path relative to the original root directory + const fullPath = path.relative(effectiveRootDir, componentDir).replace(/\\/g, "/") + console.log("createPackageManagerItem - componentDir:", componentDir) + console.log("createPackageManagerItem - effectiveRootDir:", effectiveRootDir) + console.log("createPackageManagerItem - fullPath:", fullPath) + console.log("createPackageManagerItem - path parts:", fullPath.split("/")) + // Don't encode spaces in URL to match test expectations + const urlPath = fullPath + .split("/") + .map((part) => encodeURIComponent(part)) + .join("/") + console.log("createPackageManagerItem - urlPath:", urlPath) + + // Create the item with the correct path and URL return { name: metadata.name, description: metadata.description, type: metadata.type, version: metadata.version, tags: metadata.tags, - url: componentDir, + url: `${repoUrl}/tree/main/${urlPath}`, repoUrl, sourceName, + path: fullPath, lastUpdated: await this.getLastModifiedDate(componentDir), items: [], // Initialize empty items array for all components } @@ -282,7 +334,8 @@ export class MetadataScanner { if (!entry.isDirectory()) continue const subPath = path.join(packageDir, entry.name) - const relativePath = parentPath ? path.join(parentPath, entry.name) : entry.name + // Normalize path to use forward slashes + const relativePath = parentPath ? `${parentPath}/${entry.name}` : entry.name console.log(`Found directory: ${entry.name}, relative path: ${relativePath}`) // Try to load metadata directly diff --git a/src/services/package-manager/__tests__/MetadataScanner.test.ts b/src/services/package-manager/__tests__/MetadataScanner.test.ts index c1b2965622..64314a6d80 100644 --- a/src/services/package-manager/__tests__/MetadataScanner.test.ts +++ b/src/services/package-manager/__tests__/MetadataScanner.test.ts @@ -1,4 +1,7 @@ import * as path from "path" + +// Helper function to normalize paths for test assertions +const normalizePath = (p: string) => p.replace(/\\/g, "/") import * as fs from "fs/promises" import { Dirent } from "fs" import { MetadataScanner } from "../MetadataScanner" @@ -76,16 +79,21 @@ describe("MetadataScanner", () => { describe("Basic Metadata Scanning", () => { it("should discover components with English metadata", async () => { // Mock directory structure - ;(fs.readdir as jest.Mock).mockImplementation((path: any, options?: any) => { - const pathStr = path.toString() - if (pathStr === mockBasePath) { + // Mock fs.readdir to simulate directory structure + ;(fs.readdir as jest.Mock).mockImplementation((dirPath: string) => { + // Normalize path to use forward slashes + const normalizedPath = dirPath.replace(/\\/g, "/") + const relativePath = path.relative(mockBasePath, normalizedPath).replace(/\\/g, "/") + const parts = relativePath.split("/") + + if (normalizedPath === mockBasePath) { return Promise.resolve([ createMockDirent("component1", true), createMockDirent("README.md", false), createMockDirent(".git", true), ]) } - if (pathStr.includes("component1")) { + if (normalizedPath.includes("component1")) { return Promise.resolve([createMockDirent("metadata.en.yml", false)]) } return Promise.resolve([]) @@ -108,6 +116,8 @@ version: 1.0.0 expect(items).toHaveLength(1) expect(items[0].name).toBe("Test Component") expect(items[0].type).toBe("mcp server") + expect(items[0].url).toBe("https://example.com/repo/tree/main/component1") + expect(items[0].path).toBe("component1") }) it("should skip components without English metadata", async () => { @@ -256,25 +266,79 @@ version: 1.0.0 const metadataFile = path.join(fileAnalyzerDir, "metadata.en.yml") const readmeFile = path.join(mockRepo, "README.md") + // Mock fs.stat to handle repository validation and metadata files + ;(fs.stat as jest.Mock).mockImplementation((filePath: string) => { + const normalizedPath = filePath.replace(/\\/g, "/") + if (normalizedPath === fileAnalyzerDir) { + return Promise.resolve({ + mtime: new Date("2025-04-13T10:00:00-07:00"), + isFile: () => false, + isDirectory: () => true, + }) + } + return Promise.resolve({ + mtime: new Date(), + isFile: () => false, + isDirectory: () => true, + }) + }) + // Mock directory structure using createMockDirent helper + // Mock fs.readdir to simulate directory structure + // Mock directory structure + const mockDirs = new Map() + mockDirs.set(mockRepo, [ + createMockDirent("mcp servers", true), + createMockDirent("README.md", false), + createMockDirent(".git", true), + ]) + mockDirs.set(mcpServersDir, [createMockDirent("file-analyzer", true)]) + mockDirs.set(fileAnalyzerDir, [createMockDirent("metadata.en.yml", false)]) ;(fs.readdir as jest.Mock).mockImplementation((dirPath: string) => { - if (dirPath === mockRepo) { - return Promise.resolve([ - createMockDirent("mcp servers", true), - createMockDirent("README.md", false), - createMockDirent(".git", true), - ]) + const normalizedPath = dirPath.replace(/\\/g, "/") + return Promise.resolve(mockDirs.get(normalizedPath) || []) + }) + + // Mock fs.stat to handle repository validation and metadata files + ;(fs.stat as jest.Mock).mockImplementation((filePath: string) => { + const normalizedPath = filePath.replace(/\\/g, "/") + const relativePath = path.relative(mockRepo, normalizedPath).replace(/\\/g, "/") + const parts = relativePath.split("/") + + if (parts[0] === "mcp servers" && parts[1] === "file-analyzer" && parts[2] === "metadata.en.yml") { + return Promise.resolve({ + mtime: new Date("2025-04-13T10:00:00-07:00"), + isFile: () => true, + isDirectory: () => false, + }) } - if (dirPath === mcpServersDir) { - console.log("Reading mcp servers dir:", mcpServersDir) - return Promise.resolve([createMockDirent("file-analyzer", true)]) + + return Promise.resolve({ + mtime: new Date(), + isFile: () => false, + isDirectory: () => true, + }) + }) + + // Mock fs.stat to handle repository validation and metadata files + ;(fs.stat as jest.Mock).mockImplementation((filePath: string) => { + const normalizedPath = filePath.replace(/\\/g, "/") + const relativePath = path.relative(mockRepo, normalizedPath).replace(/\\/g, "/") + const parts = relativePath.split("/") + + if (parts[0] === "mcp servers" && parts[1] === "file-analyzer" && parts[2] === "metadata.en.yml") { + return Promise.resolve({ + mtime: new Date("2025-04-13T10:00:00-07:00"), + isFile: () => true, + isDirectory: () => false, + }) } - console.log("Checking if path matches file analyzer dir:", dirPath, fileAnalyzerDir) - if (dirPath === fileAnalyzerDir) { - console.log("Reading file analyzer dir:", fileAnalyzerDir) - return Promise.resolve([createMockDirent("metadata.en.yml", false)]) - } - return Promise.resolve([]) + + return Promise.resolve({ + mtime: new Date(), + isFile: () => false, + isDirectory: () => true, + }) }) // Mock metadata file content with proper YAML format @@ -302,6 +366,145 @@ tags: []`) expect(items[0].type).toBe("mcp server") expect(items[0].version).toBe("1.0.0") expect(items[0].lastUpdated).toBe("2025-04-13T10:00:00-07:00") + expect(items[0].url).toBe("https://github.com/example/repo/tree/main/mcp%20servers/file-analyzer") + expect(items[0].path).toBe("mcp servers/file-analyzer") + }) + it("should handle nested group directories without path duplication", async () => { + const mockRepo = "/mock/repo" + const groupsDir = path.join(mockRepo, "groups") + const dataEngDir = path.join(groupsDir, "data-engineering") + const modesDir = path.join(dataEngDir, "modes") + const engineerModeDir = path.join(modesDir, "data-engineer-mode") + const metadataFile = path.join(engineerModeDir, "metadata.en.yml") + + // Mock directory structure + const mockDirs = new Map() + mockDirs.set(mockRepo, [createMockDirent("groups", true)]) + mockDirs.set(groupsDir, [createMockDirent("data-engineering", true)]) + mockDirs.set(dataEngDir, [createMockDirent("modes", true)]) + mockDirs.set(modesDir, [createMockDirent("data-engineer-mode", true)]) + mockDirs.set(engineerModeDir, [createMockDirent("metadata.en.yml", false)]) + ;(fs.readdir as jest.Mock).mockImplementation((dirPath: string) => { + const normalizedPath = dirPath.replace(/\\/g, "/") + return Promise.resolve(mockDirs.get(normalizedPath) || []) + }) + + // Mock metadata file content + ;(fs.readFile as jest.Mock).mockImplementation((filePath: string) => { + if (filePath === metadataFile) { + return Promise.resolve(`--- +name: Data Engineer Mode +description: A mode for data engineering +type: mode +version: 1.0.0 +`) + } + return Promise.resolve("") + }) + + const items = await metadataScanner.scanDirectory(mockRepo, "https://github.com/example/repo") + + expect(items).toHaveLength(1) + expect(items[0].name).toBe("Data Engineer Mode") + expect(items[0].type).toBe("mode") + expect(normalizePath(items[0].path!)).toBe("groups/data-engineering/modes/data-engineer-mode") + expect(items[0].url).toBe( + "https://github.com/example/repo/tree/main/groups/data-engineering/modes/data-engineer-mode", + ) + }) + + it("should handle deeply nested directories", async () => { + const mockRepo = "/mock/repo" + const nestedPath = path.join(mockRepo, "mcp servers", "category", "subcategory", "deep-component") + const metadataFile = path.join(nestedPath, "metadata.en.yml") + + // Mock fs.stat to handle repository validation and metadata files + ;(fs.stat as jest.Mock).mockImplementation((filePath: string) => { + const normalizedPath = filePath.replace(/\\/g, "/") + if (normalizedPath === nestedPath) { + return Promise.resolve({ + mtime: new Date("2025-04-13T10:00:00-07:00"), + isFile: () => false, + isDirectory: () => true, + }) + } + return Promise.resolve({ + mtime: new Date(), + isFile: () => false, + isDirectory: () => true, + }) + }) + + // Mock directory structure + const mockDirs = new Map() + const mcpServersDir = path.join(mockRepo, "mcp servers") + const categoryDir = path.join(mcpServersDir, "category") + const subcategoryDir = path.join(categoryDir, "subcategory") + const deepComponentDir = path.join(subcategoryDir, "deep-component") + + mockDirs.set(mockRepo, [createMockDirent("mcp servers", true)]) + mockDirs.set(mcpServersDir, [createMockDirent("category", true)]) + mockDirs.set(categoryDir, [createMockDirent("subcategory", true)]) + mockDirs.set(subcategoryDir, [createMockDirent("deep-component", true)]) + mockDirs.set(deepComponentDir, [createMockDirent("metadata.en.yml", false)]) + ;(fs.readdir as jest.Mock).mockImplementation((dirPath: string) => { + const normalizedPath = dirPath.replace(/\\/g, "/") + return Promise.resolve(mockDirs.get(normalizedPath) || []) + }) + + // Mock fs.stat to handle repository validation and metadata files + ;(fs.stat as jest.Mock).mockImplementation((filePath: string) => { + const normalizedPath = filePath.replace(/\\/g, "/") + const relativePath = path.relative(mockRepo, normalizedPath).replace(/\\/g, "/") + const parts = relativePath.split("/") + + if ( + parts[0] === "mcp servers" && + parts[1] === "category" && + parts[2] === "subcategory" && + parts[3] === "deep-component" && + parts[4] === "metadata.en.yml" + ) { + return Promise.resolve({ + mtime: new Date("2025-04-13T10:00:00-07:00"), + isFile: () => true, + isDirectory: () => false, + }) + } + + return Promise.resolve({ + mtime: new Date(), + isFile: () => false, + isDirectory: () => true, + }) + }) + + // Mock metadata file content + ;(fs.readFile as jest.Mock).mockImplementation((filePath: string) => { + const relativePath = path.relative(mockRepo, filePath) + if ( + relativePath === + path.join("mcp servers", "category", "subcategory", "deep-component", "metadata.en.yml") + ) { + return Promise.resolve(`--- +name: Deep Component +description: A deeply nested component +type: mcp server +version: 1.0.0 +`) + } + return Promise.resolve("") + }) + + const items = await metadataScanner.scanDirectory(mockRepo, "https://github.com/example/repo") + + expect(items).toHaveLength(1) + expect(items[0].name).toBe("Deep Component") + expect(items[0].type).toBe("mcp server") + expect(items[0].url).toBe( + "https://github.com/example/repo/tree/main/mcp%20servers/category/subcategory/deep-component", + ) + expect(items[0].path).toBe("mcp servers/category/subcategory/deep-component") }) it("should parse items from modes directory", async () => { @@ -753,6 +956,9 @@ version: 1.0.0 }, lastUpdated: "2025-04-13T09:00:00-07:00", }) + expect(items[0].url).toBe("https://example.com/tree/main/test-package") + expect(items[0].path).toBe("test-package") + expect(items[0].items![0].path).toBe("subcomponent1") }) it("should load subcomponents from directory structure", async () => { diff --git a/src/services/package-manager/types.ts b/src/services/package-manager/types.ts index cf7bcd1e41..80eb956d1a 100644 --- a/src/services/package-manager/types.ts +++ b/src/services/package-manager/types.ts @@ -76,6 +76,8 @@ export interface PackageManagerItem { version?: string lastUpdated?: string sourceUrl?: string + defaultBranch?: string + path?: string // Add path to main item items?: { type: ComponentType path: string @@ -103,6 +105,7 @@ export interface PackageManagerRepository { items: PackageManagerItem[] url: string error?: string + defaultBranch?: string } /** diff --git a/webview-ui/src/components/package-manager/PackageManagerView.tsx b/webview-ui/src/components/package-manager/PackageManagerView.tsx index 143bfb7ace..087f96a67e 100644 --- a/webview-ui/src/components/package-manager/PackageManagerView.tsx +++ b/webview-ui/src/components/package-manager/PackageManagerView.tsx @@ -115,7 +115,6 @@ const PackageManagerView: React.FC = ({ onDone }) => { } className="p-1 bg-vscode-dropdown-background text-vscode-dropdown-foreground border border-vscode-dropdown-border rounded mr-2"> -