mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-10-07 02:58:15 +00:00
feat: Add Google Researcher MCP Server to marketplace with local fallback
- Add local marketplace data file for development (src/services/marketplace/data/mcps.yaml) - Implement local fallback functionality in RemoteConfigLoader - Add Google Researcher MCP Server configuration with multiple installation methods: - STDIO installation (recommended) - Local installation - HTTP+SSE installation - Include proper parameter configuration for Google APIs - Add comprehensive test coverage for local fallback mechanism Addresses issue #5438: Add Google Researcher MCP Server to marketplace
This commit is contained in:
Roo Code
parent
25be23354a
commit
747ec66816
3 changed files with 306 additions and 10 deletions
|
|
@ -1,6 +1,8 @@
|
|||
import axios from "axios"
|
||||
import * as yaml from "yaml"
|
||||
import { z } from "zod"
|
||||
import * as fs from "fs"
|
||||
import * as path from "path"
|
||||
import { getRooCodeApiUrl } from "@roo-code/cloud"
|
||||
import type { MarketplaceItem, MarketplaceItemType } from "@roo-code/types"
|
||||
import { modeMarketplaceItemSchema, mcpMarketplaceItemSchema } from "@roo-code/types"
|
||||
|
|
@ -57,19 +59,51 @@ export class RemoteConfigLoader {
|
|||
const cached = this.getFromCache(cacheKey)
|
||||
if (cached) return cached
|
||||
|
||||
const data = await this.fetchWithRetry<string>(`${this.apiBaseUrl}/api/marketplace/mcps`)
|
||||
try {
|
||||
const data = await this.fetchWithRetry<string>(`${this.apiBaseUrl}/api/marketplace/mcps`)
|
||||
|
||||
// Parse and validate YAML response
|
||||
const yamlData = yaml.parse(data)
|
||||
const validated = mcpMarketplaceResponse.parse(yamlData)
|
||||
// Parse and validate YAML response
|
||||
const yamlData = yaml.parse(data)
|
||||
const validated = mcpMarketplaceResponse.parse(yamlData)
|
||||
|
||||
const items: MarketplaceItem[] = validated.items.map((item) => ({
|
||||
type: "mcp" as const,
|
||||
...item,
|
||||
}))
|
||||
const items: MarketplaceItem[] = validated.items.map((item) => ({
|
||||
type: "mcp" as const,
|
||||
...item,
|
||||
}))
|
||||
|
||||
this.setCache(cacheKey, items)
|
||||
return items
|
||||
this.setCache(cacheKey, items)
|
||||
return items
|
||||
} catch (error) {
|
||||
// Fallback to local development data if remote fetch fails
|
||||
console.warn("Failed to fetch remote MCP data, falling back to local data:", error)
|
||||
return this.loadLocalMcps()
|
||||
}
|
||||
}
|
||||
|
||||
private async loadLocalMcps(): Promise<MarketplaceItem[]> {
|
||||
try {
|
||||
const localDataPath = path.join(__dirname, "data", "mcps.yaml")
|
||||
|
||||
if (!fs.existsSync(localDataPath)) {
|
||||
console.warn("Local MCP data file not found:", localDataPath)
|
||||
return []
|
||||
}
|
||||
|
||||
const fileContent = fs.readFileSync(localDataPath, "utf-8")
|
||||
const yamlData = yaml.parse(fileContent)
|
||||
const validated = mcpMarketplaceResponse.parse(yamlData)
|
||||
|
||||
const items: MarketplaceItem[] = validated.items.map((item) => ({
|
||||
type: "mcp" as const,
|
||||
...item,
|
||||
}))
|
||||
|
||||
console.log(`Loaded ${items.length} MCP items from local data`)
|
||||
return items
|
||||
} catch (error) {
|
||||
console.error("Failed to load local MCP data:", error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchWithRetry<T>(url: string, maxRetries = 3): Promise<T> {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
// npx vitest services/marketplace/__tests__/RemoteConfigLoader.spec.ts
|
||||
|
||||
import axios from "axios"
|
||||
import * as fs from "fs"
|
||||
import * as path from "path"
|
||||
import { RemoteConfigLoader } from "../RemoteConfigLoader"
|
||||
import type { MarketplaceItemType } from "@roo-code/types"
|
||||
|
||||
|
|
@ -8,6 +10,14 @@ import type { MarketplaceItemType } from "@roo-code/types"
|
|||
vi.mock("axios")
|
||||
const mockedAxios = axios as any
|
||||
|
||||
// Mock fs
|
||||
vi.mock("fs")
|
||||
const mockedFs = fs as any
|
||||
|
||||
// Mock path
|
||||
vi.mock("path")
|
||||
const mockedPath = path as any
|
||||
|
||||
// Mock the cloud config
|
||||
vi.mock("@roo-code/cloud", () => ({
|
||||
getRooCodeApiUrl: () => "https://test.api.com",
|
||||
|
|
@ -332,4 +342,202 @@ describe("RemoteConfigLoader", () => {
|
|||
Date.now = originalDateNow
|
||||
})
|
||||
})
|
||||
|
||||
describe("local fallback functionality", () => {
|
||||
beforeEach(() => {
|
||||
// Reset mocks
|
||||
vi.clearAllMocks()
|
||||
loader.clearCache()
|
||||
// Mock path.join to return a predictable path
|
||||
mockedPath.join.mockReturnValue("/test/path/data/mcps.yaml")
|
||||
})
|
||||
|
||||
it("should fallback to local data when remote API fails", async () => {
|
||||
const localMcpsYaml = `items:
|
||||
- id: "test-mcp"
|
||||
name: "Test MCP"
|
||||
description: "A test MCP"
|
||||
url: "https://github.com/test/test-mcp"
|
||||
content:
|
||||
- name: "Installation"
|
||||
content: '{"command": "test"}'`
|
||||
|
||||
// Mock remote API failure
|
||||
mockedAxios.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modes")) {
|
||||
return Promise.resolve({ data: "items: []" })
|
||||
}
|
||||
if (url.includes("/mcps")) {
|
||||
return Promise.reject(new Error("Network error"))
|
||||
}
|
||||
return Promise.reject(new Error("Unknown URL"))
|
||||
})
|
||||
|
||||
// Mock local file system
|
||||
mockedFs.existsSync.mockReturnValue(true)
|
||||
mockedFs.readFileSync.mockReturnValue(localMcpsYaml)
|
||||
|
||||
const items = await loader.loadAllItems()
|
||||
|
||||
// Should have attempted remote call first
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
"https://test.api.com/api/marketplace/mcps",
|
||||
expect.any(Object),
|
||||
)
|
||||
|
||||
// Should have fallen back to local file
|
||||
expect(mockedFs.existsSync).toHaveBeenCalled()
|
||||
expect(mockedFs.readFileSync).toHaveBeenCalled()
|
||||
|
||||
// Should contain the test MCP
|
||||
expect(items).toHaveLength(1)
|
||||
expect(items[0]).toEqual({
|
||||
type: "mcp",
|
||||
id: "test-mcp",
|
||||
name: "Test MCP",
|
||||
description: "A test MCP",
|
||||
url: "https://github.com/test/test-mcp",
|
||||
content: [
|
||||
{
|
||||
name: "Installation",
|
||||
content: '{"command": "test"}',
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should return empty array when local file doesn't exist", async () => {
|
||||
// Mock remote API failure
|
||||
mockedAxios.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modes")) {
|
||||
return Promise.resolve({ data: "items: []" })
|
||||
}
|
||||
if (url.includes("/mcps")) {
|
||||
return Promise.reject(new Error("Network error"))
|
||||
}
|
||||
return Promise.reject(new Error("Unknown URL"))
|
||||
})
|
||||
|
||||
// Mock local file not existing
|
||||
mockedFs.existsSync.mockReturnValue(false)
|
||||
|
||||
const items = await loader.loadAllItems()
|
||||
|
||||
// Should have attempted remote call first
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
"https://test.api.com/api/marketplace/mcps",
|
||||
expect.any(Object),
|
||||
)
|
||||
|
||||
// Should have checked for local file
|
||||
expect(mockedFs.existsSync).toHaveBeenCalledWith(expect.stringContaining("data/mcps.yaml"))
|
||||
|
||||
// Should not have tried to read the file
|
||||
expect(mockedFs.readFileSync).not.toHaveBeenCalled()
|
||||
|
||||
// Should return empty array (only modes, no MCPs)
|
||||
expect(items).toHaveLength(0)
|
||||
})
|
||||
|
||||
it("should handle local file read errors gracefully", async () => {
|
||||
// Mock remote API failure
|
||||
mockedAxios.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modes")) {
|
||||
return Promise.resolve({ data: "items: []" })
|
||||
}
|
||||
if (url.includes("/mcps")) {
|
||||
return Promise.reject(new Error("Network error"))
|
||||
}
|
||||
return Promise.reject(new Error("Unknown URL"))
|
||||
})
|
||||
|
||||
// Mock local file exists but read fails
|
||||
mockedFs.existsSync.mockReturnValue(true)
|
||||
mockedFs.readFileSync.mockImplementation(() => {
|
||||
throw new Error("File read error")
|
||||
})
|
||||
|
||||
const items = await loader.loadAllItems()
|
||||
|
||||
// Should have attempted to read local file
|
||||
expect(mockedFs.readFileSync).toHaveBeenCalledWith(expect.stringContaining("data/mcps.yaml"), "utf-8")
|
||||
|
||||
// Should return empty array when local fallback fails
|
||||
expect(items).toHaveLength(0)
|
||||
})
|
||||
|
||||
it("should prefer remote data over local when remote is available", async () => {
|
||||
const remoteMcpsYaml = `items:
|
||||
- id: "remote-mcp"
|
||||
name: "Remote MCP"
|
||||
description: "From remote API"
|
||||
url: "https://github.com/remote/mcp"
|
||||
content:
|
||||
- name: "Installation"
|
||||
content: '{"command": "remote"}'`
|
||||
|
||||
// Mock successful remote API
|
||||
mockedAxios.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modes")) {
|
||||
return Promise.resolve({ data: "items: []" })
|
||||
}
|
||||
if (url.includes("/mcps")) {
|
||||
return Promise.resolve({ data: remoteMcpsYaml })
|
||||
}
|
||||
return Promise.reject(new Error("Unknown URL"))
|
||||
})
|
||||
|
||||
const items = await loader.loadAllItems()
|
||||
|
||||
// Should have used remote data
|
||||
expect(items).toHaveLength(1)
|
||||
expect(items[0].id).toBe("remote-mcp")
|
||||
expect(items[0].name).toBe("Remote MCP")
|
||||
|
||||
// Should not have accessed local file system
|
||||
expect(mockedFs.existsSync).not.toHaveBeenCalled()
|
||||
expect(mockedFs.readFileSync).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe("Google Researcher MCP Server integration", () => {
|
||||
it("should find Google Researcher MCP by ID when loaded from local data", async () => {
|
||||
const localMcpsYaml = `items:
|
||||
- id: "google-researcher-mcp"
|
||||
name: "Google Researcher MCP Server"
|
||||
description: "Power your AI agents with Google Search–enhanced research"
|
||||
author: "Zohar Babin"
|
||||
url: "https://github.com/zoharbabin/google-research-mcp"
|
||||
content:
|
||||
- name: "STDIO Installation"
|
||||
content: '{"google-researcher": {"command": "npx", "args": ["google-researcher-mcp@latest"]}}'`
|
||||
|
||||
// Mock remote API failure to trigger local fallback
|
||||
mockedAxios.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modes")) {
|
||||
return Promise.resolve({ data: "items: []" })
|
||||
}
|
||||
if (url.includes("/mcps")) {
|
||||
return Promise.reject(new Error("Network error"))
|
||||
}
|
||||
return Promise.reject(new Error("Unknown URL"))
|
||||
})
|
||||
|
||||
// Mock local file system
|
||||
mockedFs.existsSync.mockReturnValue(true)
|
||||
mockedFs.readFileSync.mockReturnValue(localMcpsYaml)
|
||||
|
||||
const item = await loader.getItem("google-researcher-mcp", "mcp" as MarketplaceItemType)
|
||||
|
||||
expect(item).not.toBeNull()
|
||||
expect(item?.id).toBe("google-researcher-mcp")
|
||||
expect(item?.name).toBe("Google Researcher MCP Server")
|
||||
expect(item?.author).toBe("Zohar Babin")
|
||||
|
||||
// Type guard to ensure we have an MCP item
|
||||
if (item?.type === "mcp") {
|
||||
expect(item.url).toBe("https://github.com/zoharbabin/google-research-mcp")
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
54
src/services/marketplace/data/mcps.yaml
Normal file
54
src/services/marketplace/data/mcps.yaml
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
items:
|
||||
- id: "google-researcher-mcp"
|
||||
name: "Google Researcher MCP Server"
|
||||
description: "Power your AI agents with Google Search–enhanced research via an open-source MCP server. Includes tools for Google Search, YouTube/web scraping, LLM-driven synthesis, persistent caching, and dual transport (STDIO + HTTP SSE) for efficient, flexible integration."
|
||||
author: "Zohar Babin"
|
||||
authorUrl: "https://github.com/zoharbabin"
|
||||
url: "https://github.com/zoharbabin/google-research-mcp"
|
||||
tags: ["research", "google", "search", "web-scraping", "ai-analysis", "caching"]
|
||||
prerequisites:
|
||||
- "Node.js 18.0.0 or higher"
|
||||
- "Google Custom Search API Key"
|
||||
- "Google Custom Search Engine ID"
|
||||
- "Google Gemini API Key"
|
||||
parameters:
|
||||
- name: "Google Custom Search API Key"
|
||||
key: "google_search_api_key"
|
||||
placeholder: "Enter your Google Custom Search API key"
|
||||
- name: "Google Custom Search Engine ID"
|
||||
key: "google_search_engine_id"
|
||||
placeholder: "Enter your Google Custom Search Engine ID"
|
||||
- name: "Google Gemini API Key"
|
||||
key: "google_gemini_api_key"
|
||||
placeholder: "Enter your Google Gemini API key"
|
||||
content:
|
||||
- name: "STDIO Installation (Recommended)"
|
||||
content: '{"google-researcher": {"command": "npx", "args": ["google-researcher-mcp@latest"], "env": {"GOOGLE_SEARCH_API_KEY": "{{google_search_api_key}}", "GOOGLE_SEARCH_ENGINE_ID": "{{google_search_engine_id}}", "GOOGLE_GEMINI_API_KEY": "{{google_gemini_api_key}}"}}}'
|
||||
prerequisites:
|
||||
- "Ensure you have npm installed"
|
||||
- "Get API keys from Google Cloud Console"
|
||||
- name: "Local Installation"
|
||||
content: '{"google-researcher": {"command": "node", "args": ["{{install_path}}/dist/server.js"], "env": {"GOOGLE_SEARCH_API_KEY": "{{google_search_api_key}}", "GOOGLE_SEARCH_ENGINE_ID": "{{google_search_engine_id}}", "GOOGLE_GEMINI_API_KEY": "{{google_gemini_api_key}}"}}}'
|
||||
parameters:
|
||||
- name: "Installation Path"
|
||||
key: "install_path"
|
||||
placeholder: "/path/to/google-research-mcp"
|
||||
prerequisites:
|
||||
- "Clone the repository: git clone https://github.com/zoharbabin/google-research-mcp.git"
|
||||
- "Install dependencies: npm install"
|
||||
- "Build the project: npm run build"
|
||||
- name: "HTTP+SSE Installation"
|
||||
content: '{"google-researcher": {"url": "http://localhost:{{port}}/mcp", "headers": {"Authorization": "Bearer {{oauth_token}}"}}}'
|
||||
parameters:
|
||||
- name: "Server Port"
|
||||
key: "port"
|
||||
placeholder: "3000"
|
||||
optional: true
|
||||
- name: "OAuth Token"
|
||||
key: "oauth_token"
|
||||
placeholder: "Enter your OAuth token"
|
||||
optional: true
|
||||
prerequisites:
|
||||
- "Start the server: npm start"
|
||||
- "Configure OAuth 2.1 provider (optional)"
|
||||
- "Server will be available at http://localhost:3000/mcp"
|
||||
Loading…
Add table
Reference in a new issue