mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-10-10 03:28:03 +00:00
Remove coalescing, background refresh, and cache logging
Address review feedback by removing out-of-scope optimizations: 1. Remove in-flight coalescing infrastructure - Delete inFlightModelFetches and inFlightEndpointFetches maps - Eliminate promise sharing across concurrent requests 2. Remove background refresh on file cache hit - Simplify to synchronous flow: memory → file → network - No more fire-and-forget background updates 3. Remove cache performance logging - Delete console.log statements for cache_hit, file_hit, bg_refresh - Clean up debugging artifacts from development 4. Fix active-provider scoping gap - Include ollama/lmstudio/huggingface in requestRouterModels when active - Prevents empty response that breaks chat flows for local providers Result: Simpler, more maintainable code focused on core goal of reducing unnecessary network requests by scoping to active provider.
This commit is contained in:
daniel-lxs
parent
3f4edff096
commit
97beb1366c
3 changed files with 107 additions and 229 deletions
|
|
@ -26,9 +26,6 @@ import { getRooModels } from "./roo"
|
|||
|
||||
const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
|
||||
|
||||
// Coalesce concurrent fetches per provider within this extension host
|
||||
const inFlightModelFetches = new Map<RouterName, Promise<ModelRecord>>()
|
||||
|
||||
async function writeModels(router: RouterName, data: ModelRecord) {
|
||||
const filename = `${router}_models.json`
|
||||
const cacheDir = await getCacheDirectoryPath(ContextProxy.instance.globalStorageUri.fsPath)
|
||||
|
|
@ -63,160 +60,75 @@ export const getModels = async (options: GetModelsOptions): Promise<ModelRecord>
|
|||
return cached
|
||||
}
|
||||
|
||||
// 2) Try file cache snapshot (Option A), then kick off background refresh
|
||||
// 2) Try file cache snapshot
|
||||
try {
|
||||
const file = await readModels(provider)
|
||||
if (file && Object.keys(file).length > 0) {
|
||||
memoryCache.set(provider, file)
|
||||
|
||||
// Start background refresh if not already in-flight (do not await)
|
||||
if (!inFlightModelFetches.has(provider)) {
|
||||
const signal = AbortSignal.timeout(30_000)
|
||||
const bgPromise = (async (): Promise<ModelRecord> => {
|
||||
let models: ModelRecord = {}
|
||||
switch (provider) {
|
||||
case "openrouter":
|
||||
models = await getOpenRouterModels(undefined, signal)
|
||||
break
|
||||
case "requesty":
|
||||
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "glama":
|
||||
models = await getGlamaModels(signal)
|
||||
break
|
||||
case "unbound":
|
||||
models = await getUnboundModels(options.apiKey, signal)
|
||||
break
|
||||
case "litellm":
|
||||
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
|
||||
break
|
||||
case "ollama":
|
||||
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "lmstudio":
|
||||
models = await getLMStudioModels(options.baseUrl, signal)
|
||||
break
|
||||
case "deepinfra":
|
||||
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
|
||||
break
|
||||
case "io-intelligence":
|
||||
models = await getIOIntelligenceModels(options.apiKey, signal)
|
||||
break
|
||||
case "vercel-ai-gateway":
|
||||
models = await getVercelAiGatewayModels(undefined, signal)
|
||||
break
|
||||
case "huggingface":
|
||||
models = await getHuggingFaceModels(signal)
|
||||
break
|
||||
case "roo": {
|
||||
const rooBaseUrl =
|
||||
options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
|
||||
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
|
||||
break
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown provider: ${provider}`)
|
||||
}
|
||||
|
||||
memoryCache.set(provider, models)
|
||||
await writeModels(provider, models).catch((err) => {
|
||||
console.error(
|
||||
`[modelCache] Error writing ${provider} to file cache during background refresh:`,
|
||||
err instanceof Error ? err.message : String(err),
|
||||
)
|
||||
})
|
||||
return models || {}
|
||||
})()
|
||||
|
||||
inFlightModelFetches.set(provider, bgPromise)
|
||||
Promise.resolve(bgPromise)
|
||||
.catch((err) => {
|
||||
console.error(
|
||||
`[modelCache] Background refresh failed for ${provider}:`,
|
||||
err instanceof Error ? err.message : String(err),
|
||||
)
|
||||
})
|
||||
.finally(() => inFlightModelFetches.delete(provider))
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
} catch {
|
||||
// ignore file read errors; fall through to network/coalesce path
|
||||
// ignore file read errors; fall through to network fetch
|
||||
}
|
||||
|
||||
// 3) Coalesce concurrent fetches
|
||||
const existing = inFlightModelFetches.get(provider)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
|
||||
// 4) Network fetch wrapped as a single in-flight promise for this provider
|
||||
// 3) Network fetch
|
||||
const signal = AbortSignal.timeout(30_000)
|
||||
const fetchPromise = (async (): Promise<ModelRecord> => {
|
||||
let models: ModelRecord = {}
|
||||
switch (provider) {
|
||||
case "openrouter":
|
||||
models = await getOpenRouterModels(undefined, signal)
|
||||
break
|
||||
case "requesty":
|
||||
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "glama":
|
||||
models = await getGlamaModels(signal)
|
||||
break
|
||||
case "unbound":
|
||||
models = await getUnboundModels(options.apiKey, signal)
|
||||
break
|
||||
case "litellm":
|
||||
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
|
||||
break
|
||||
case "ollama":
|
||||
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "lmstudio":
|
||||
models = await getLMStudioModels(options.baseUrl, signal)
|
||||
break
|
||||
case "deepinfra":
|
||||
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
|
||||
break
|
||||
case "io-intelligence":
|
||||
models = await getIOIntelligenceModels(options.apiKey, signal)
|
||||
break
|
||||
case "vercel-ai-gateway":
|
||||
models = await getVercelAiGatewayModels(undefined, signal)
|
||||
break
|
||||
case "huggingface":
|
||||
models = await getHuggingFaceModels(signal)
|
||||
break
|
||||
case "roo": {
|
||||
const rooBaseUrl =
|
||||
options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
|
||||
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
|
||||
break
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unknown provider: ${provider}`)
|
||||
}
|
||||
let models: ModelRecord = {}
|
||||
|
||||
switch (provider) {
|
||||
case "openrouter":
|
||||
models = await getOpenRouterModels(undefined, signal)
|
||||
break
|
||||
case "requesty":
|
||||
models = await getRequestyModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "glama":
|
||||
models = await getGlamaModels(signal)
|
||||
break
|
||||
case "unbound":
|
||||
models = await getUnboundModels(options.apiKey, signal)
|
||||
break
|
||||
case "litellm":
|
||||
models = await getLiteLLMModels(options.apiKey as string, options.baseUrl as string, signal)
|
||||
break
|
||||
case "ollama":
|
||||
models = await getOllamaModels(options.baseUrl, options.apiKey, signal)
|
||||
break
|
||||
case "lmstudio":
|
||||
models = await getLMStudioModels(options.baseUrl, signal)
|
||||
break
|
||||
case "deepinfra":
|
||||
models = await getDeepInfraModels(options.apiKey, options.baseUrl, signal)
|
||||
break
|
||||
case "io-intelligence":
|
||||
models = await getIOIntelligenceModels(options.apiKey, signal)
|
||||
break
|
||||
case "vercel-ai-gateway":
|
||||
models = await getVercelAiGatewayModels(undefined, signal)
|
||||
break
|
||||
case "huggingface":
|
||||
models = await getHuggingFaceModels(signal)
|
||||
break
|
||||
case "roo": {
|
||||
const rooBaseUrl = options.baseUrl ?? process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy"
|
||||
models = await getRooModels(rooBaseUrl, options.apiKey, signal)
|
||||
break
|
||||
}
|
||||
default: {
|
||||
throw new Error(`Unknown provider: ${provider}`)
|
||||
}
|
||||
memoryCache.set(provider, models)
|
||||
|
||||
await writeModels(provider, models).catch((err) => {
|
||||
console.error(
|
||||
`[modelCache] Error writing ${provider} to file cache after network fetch:`,
|
||||
err instanceof Error ? err.message : String(err),
|
||||
)
|
||||
})
|
||||
|
||||
return models || {}
|
||||
})()
|
||||
|
||||
inFlightModelFetches.set(provider, fetchPromise)
|
||||
try {
|
||||
return await fetchPromise
|
||||
} finally {
|
||||
inFlightModelFetches.delete(provider)
|
||||
}
|
||||
|
||||
memoryCache.set(provider, models)
|
||||
|
||||
await writeModels(provider, models).catch((err) => {
|
||||
console.error(
|
||||
`[modelCache] Error writing ${provider} to file cache after network fetch:`,
|
||||
err instanceof Error ? err.message : String(err),
|
||||
)
|
||||
})
|
||||
|
||||
return models || {}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -14,9 +14,6 @@ import { getOpenRouterModelEndpoints } from "./openrouter"
|
|||
|
||||
const memoryCache = new NodeCache({ stdTTL: 5 * 60, checkperiod: 5 * 60 })
|
||||
|
||||
// Coalesce concurrent endpoint fetches per (router,modelId)
|
||||
const inFlightEndpointFetches = new Map<string, Promise<ModelRecord>>()
|
||||
|
||||
const getCacheKey = (router: RouterName, modelId: string) => sanitize(`${router}_${modelId}`)
|
||||
|
||||
async function writeModelEndpoints(key: string, data: ModelRecord) {
|
||||
|
|
@ -53,107 +50,47 @@ export const getModelEndpoints = async ({
|
|||
// 1) Try memory cache
|
||||
const cached = memoryCache.get<ModelRecord>(key)
|
||||
if (cached) {
|
||||
// Using console.log for cache layer logging (no provider access in utility functions)
|
||||
console.log(`[endpointCache] cache_hit: ${key} (${Object.keys(cached).length} endpoints)`)
|
||||
return cached
|
||||
}
|
||||
|
||||
// 2) Try file cache snapshot (Option A), then kick off background refresh
|
||||
// 2) Try file cache snapshot
|
||||
try {
|
||||
const file = await readModelEndpoints(key)
|
||||
if (file && Object.keys(file).length > 0) {
|
||||
// Using console.log for cache layer logging (no provider access in utility functions)
|
||||
console.log(`[endpointCache] file_hit: ${key} (${Object.keys(file).length} endpoints, bg_refresh queued)`)
|
||||
// Populate memory cache immediately
|
||||
memoryCache.set(key, file)
|
||||
|
||||
// Start background refresh if not already in-flight (do not await)
|
||||
if (!inFlightEndpointFetches.has(key)) {
|
||||
const signal = AbortSignal.timeout(30_000)
|
||||
const bgPromise = (async (): Promise<ModelRecord> => {
|
||||
const modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
|
||||
if (Object.keys(modelProviders).length > 0) {
|
||||
console.log(
|
||||
`[endpointCache] bg_refresh_done: ${key} (${Object.keys(modelProviders).length} endpoints)`,
|
||||
)
|
||||
memoryCache.set(key, modelProviders)
|
||||
try {
|
||||
await writeModelEndpoints(key, modelProviders)
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[endpointCache] Error writing ${key} to file cache during background refresh:`,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
)
|
||||
}
|
||||
return modelProviders
|
||||
}
|
||||
return {}
|
||||
})()
|
||||
|
||||
inFlightEndpointFetches.set(key, bgPromise)
|
||||
Promise.resolve(bgPromise)
|
||||
.catch((err) => {
|
||||
// Log background refresh failures for monitoring
|
||||
console.error(
|
||||
`[endpointCache] Background refresh failed for ${key}:`,
|
||||
err instanceof Error ? err.message : String(err),
|
||||
)
|
||||
})
|
||||
.finally(() => inFlightEndpointFetches.delete(key))
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
} catch {
|
||||
// ignore file read errors; fall through
|
||||
// ignore file read errors; fall through to network fetch
|
||||
}
|
||||
|
||||
// 3) Coalesce concurrent fetches
|
||||
const inFlight = inFlightEndpointFetches.get(key)
|
||||
if (inFlight) {
|
||||
// Using console.log for cache layer logging (no provider access in utility functions)
|
||||
console.log(`[endpointCache] coalesced_wait: ${key}`)
|
||||
return inFlight
|
||||
}
|
||||
|
||||
// 4) Single network fetch for this key
|
||||
// 3) Network fetch
|
||||
const signal = AbortSignal.timeout(30_000)
|
||||
const fetchPromise = (async (): Promise<ModelRecord> => {
|
||||
let modelProviders: ModelRecord = {}
|
||||
modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
|
||||
let modelProviders: ModelRecord = {}
|
||||
|
||||
if (Object.keys(modelProviders).length > 0) {
|
||||
console.log(`[endpointCache] network_fetch_done: ${key} (${Object.keys(modelProviders).length} endpoints)`)
|
||||
// Update memory cache first
|
||||
memoryCache.set(key, modelProviders)
|
||||
modelProviders = await getOpenRouterModelEndpoints(modelId, undefined, signal)
|
||||
|
||||
// Best-effort persist
|
||||
try {
|
||||
await writeModelEndpoints(key, modelProviders)
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[endpointCache] Error writing ${key} to file cache after network fetch:`,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
)
|
||||
}
|
||||
if (Object.keys(modelProviders).length > 0) {
|
||||
memoryCache.set(key, modelProviders)
|
||||
|
||||
return modelProviders
|
||||
}
|
||||
|
||||
// Fallback to file cache if network returned empty (rare)
|
||||
try {
|
||||
const file = await readModelEndpoints(key)
|
||||
return file ?? {}
|
||||
} catch {
|
||||
return {}
|
||||
await writeModelEndpoints(key, modelProviders)
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[endpointCache] Error writing ${key} to file cache after network fetch:`,
|
||||
error instanceof Error ? error.message : String(error),
|
||||
)
|
||||
}
|
||||
})()
|
||||
|
||||
inFlightEndpointFetches.set(key, fetchPromise)
|
||||
return modelProviders
|
||||
}
|
||||
|
||||
// Fallback to file cache if network returned empty (rare)
|
||||
try {
|
||||
return await fetchPromise
|
||||
} finally {
|
||||
inFlightEndpointFetches.delete(key)
|
||||
const file = await readModelEndpoints(key)
|
||||
return file ?? {}
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -832,6 +832,35 @@ export const webviewMessageHandler = async (
|
|||
},
|
||||
]
|
||||
|
||||
// Include local providers (ollama, lmstudio, huggingface) when they are the active provider
|
||||
if (activeProvider === "ollama") {
|
||||
allFetches.push({
|
||||
key: "ollama",
|
||||
options: {
|
||||
provider: "ollama",
|
||||
baseUrl: apiConfiguration.ollamaBaseUrl,
|
||||
apiKey: apiConfiguration.ollamaApiKey,
|
||||
},
|
||||
})
|
||||
}
|
||||
if (activeProvider === "lmstudio") {
|
||||
allFetches.push({
|
||||
key: "lmstudio",
|
||||
options: {
|
||||
provider: "lmstudio",
|
||||
baseUrl: apiConfiguration.lmStudioBaseUrl,
|
||||
},
|
||||
})
|
||||
}
|
||||
if (activeProvider === "huggingface") {
|
||||
allFetches.push({
|
||||
key: "huggingface",
|
||||
options: {
|
||||
provider: "huggingface",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// IO Intelligence (optional)
|
||||
if (apiConfiguration.ioIntelligenceApiKey) {
|
||||
allFetches.push({
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue