scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-09-07 08:26:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Configure Qdrant to use on-disk storage for memory efficiency

Browse source 
- Add memory optimization configuration to QdrantVectorStore
- Configure vectors and HNSW indexes to use on-disk storage by default
- Add memory-mapped file support for segments larger than 50k vectors
- Reduce HNSW search parameter (ef) from 128 to 64 for memory efficiency
- Add configuration options for memory optimization settings
- Update type definitions to include new configuration options
- Update all affected tests to handle new constructor parameters

Fixes #6262
This commit is contained in:
Roo Code 2025-07-27 01:10:08 +00:00
parent 91c21a1bba
commit dab65f3164
8 changed files with 309 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/types/src/codebase-index.ts
View file

@ -34,6 +34,10 @@ export const codebaseIndexConfigSchema = z.object({
// OpenAI Compatible specific fields
codebaseIndexOpenAiCompatibleBaseUrl: z.string().optional(),
codebaseIndexOpenAiCompatibleModelDimension: z.number().optional(),
// Memory optimization settings
codebaseIndexUseOnDiskStorage: z.boolean().optional(),
codebaseIndexMemoryMapThreshold: z.number().optional(),
codebaseIndexHnswEfSearch: z.number().optional(),
})
export type CodebaseIndexConfig = z.infer<typeof codebaseIndexConfigSchema>

40
src/services/code-index/__tests__/config-manager.spec.ts
View file

@ -104,11 +104,18 @@ describe("CodeIndexConfigManager", () => {
isConfigured: false,
embedderProvider: "openai",
modelId: undefined,
modelDimension: undefined,
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
openAiCompatibleOptions: undefined,
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://localhost:6333",
qdrantApiKey: "",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
expect(result.requiresRestart).toBe(false)
})
@ -135,11 +142,18 @@ describe("CodeIndexConfigManager", () => {
isConfigured: true,
embedderProvider: "openai",
modelId: "text-embedding-3-large",
modelDimension: undefined,
openAiOptions: { openAiNativeApiKey: "test-openai-key" },
ollamaOptions: { ollamaBaseUrl: "" },
openAiCompatibleOptions: undefined,
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})
@ -168,15 +182,21 @@ describe("CodeIndexConfigManager", () => {
isConfigured: true,
embedderProvider: "openai-compatible",
modelId: "text-embedding-3-large",
modelDimension: undefined,
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
openAiCompatibleOptions: {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
},
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})
@ -212,9 +232,14 @@ describe("CodeIndexConfigManager", () => {
baseUrl: "https://api.example.com/v1",
apiKey: "test-openai-compatible-key",
},
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})
@ -243,6 +268,7 @@ describe("CodeIndexConfigManager", () => {
isConfigured: true,
embedderProvider: "openai-compatible",
modelId: "custom-model",
modelDimension: undefined,
openAiOptions: { openAiNativeApiKey: "" },
ollamaOptions: { ollamaBaseUrl: "" },
openAiCompatibleOptions: {
@ -250,9 +276,14 @@ describe("CodeIndexConfigManager", () => {
apiKey: "test-openai-compatible-key",
// modelDimension is undefined when not set
},
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})
@ -289,9 +320,13 @@ describe("CodeIndexConfigManager", () => {
apiKey: "test-openai-compatible-key",
},
geminiOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})
@ -1292,14 +1327,19 @@ describe("CodeIndexConfigManager", () => {
isConfigured: true,
embedderProvider: "openai",
modelId: "text-embedding-3-large",
modelDimension: undefined,
openAiOptions: { openAiNativeApiKey: "test-openai-key" },
ollamaOptions: { ollamaBaseUrl: undefined },
geminiOptions: undefined,
openAiCompatibleOptions: undefined,
mistralOptions: undefined,
qdrantUrl: "http://qdrant.local",
qdrantApiKey: "test-qdrant-key",
searchMinScore: 0.4,
searchMaxResults: 50,
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
})
})

50
src/services/code-index/__tests__/service-factory.spec.ts
View file

@ -49,6 +49,11 @@ describe("CodeIndexServiceFactory", () => {
mockConfigManager = {
getConfig: vitest.fn(),
memoryOptimizationConfig: {
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
}
mockCacheManager = {}
@ -367,6 +372,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
3072,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -392,6 +402,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
768,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -417,6 +432,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
3072,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -449,6 +469,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
modelDimension, // Should use model's built-in dimension, not manual
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -480,6 +505,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
manualDimension, // Should use manual dimension as fallback
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -509,6 +539,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
768,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -578,6 +613,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
3072,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -603,6 +643,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
3072,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})
@ -627,6 +672,11 @@ describe("CodeIndexServiceFactory", () => {
"http://localhost:6333",
1536,
"test-key",
{
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
},
)
})

37
src/services/code-index/config-manager.ts
View file

@ -23,6 +23,10 @@ export class CodeIndexConfigManager {
private qdrantApiKey?: string
private searchMinScore?: number
private searchMaxResults?: number
// Memory optimization settings
private useOnDiskStorage?: boolean
private memoryMapThreshold?: number
private hnswEfSearch?: number
constructor(private readonly contextProxy: ContextProxy) {
// Initialize with current configuration to avoid false restart triggers
@ -50,6 +54,9 @@ export class CodeIndexConfigManager {
codebaseIndexEmbedderModelId: "",
codebaseIndexSearchMinScore: undefined,
codebaseIndexSearchMaxResults: undefined,
codebaseIndexUseOnDiskStorage: true, // Default to true for memory efficiency
codebaseIndexMemoryMapThreshold: 50000, // Default 50k vectors
codebaseIndexHnswEfSearch: 64, // Default 64 for balance
}
const {
@ -60,6 +67,9 @@ export class CodeIndexConfigManager {
codebaseIndexEmbedderModelId,
codebaseIndexSearchMinScore,
codebaseIndexSearchMaxResults,
codebaseIndexUseOnDiskStorage,
codebaseIndexMemoryMapThreshold,
codebaseIndexHnswEfSearch,
} = codebaseIndexConfig
const openAiKey = this.contextProxy?.getSecret("codeIndexOpenAiKey") ?? ""
@ -76,6 +86,9 @@ export class CodeIndexConfigManager {
this.qdrantApiKey = qdrantApiKey ?? ""
this.searchMinScore = codebaseIndexSearchMinScore
this.searchMaxResults = codebaseIndexSearchMaxResults
this.useOnDiskStorage = codebaseIndexUseOnDiskStorage ?? true
this.memoryMapThreshold = codebaseIndexMemoryMapThreshold ?? 50000
this.hnswEfSearch = codebaseIndexHnswEfSearch ?? 64
// Validate and set model dimension
const rawDimension = codebaseIndexConfig.codebaseIndexEmbedderModelDimension
@ -144,6 +157,9 @@ export class CodeIndexConfigManager {
qdrantUrl?: string
qdrantApiKey?: string
searchMinScore?: number
useOnDiskStorage?: boolean
memoryMapThreshold?: number
hnswEfSearch?: number
}
requiresRestart: boolean
}> {
@ -187,6 +203,9 @@ export class CodeIndexConfigManager {
qdrantUrl: this.qdrantUrl,
qdrantApiKey: this.qdrantApiKey,
searchMinScore: this.currentSearchMinScore,
useOnDiskStorage: this.useOnDiskStorage,
memoryMapThreshold: this.memoryMapThreshold,
hnswEfSearch: this.hnswEfSearch,
},
requiresRestart,
}
@ -379,6 +398,9 @@ export class CodeIndexConfigManager {
qdrantApiKey: this.qdrantApiKey,
searchMinScore: this.currentSearchMinScore,
searchMaxResults: this.currentSearchMaxResults,
useOnDiskStorage: this.useOnDiskStorage,
memoryMapThreshold: this.memoryMapThreshold,
hnswEfSearch: this.hnswEfSearch,
}
}
@ -413,6 +435,21 @@ export class CodeIndexConfigManager {
}
}
/**
* Gets the memory optimization settings
*/
public get memoryOptimizationConfig(): {
useOnDiskStorage?: boolean
memoryMapThreshold?: number
hnswEfSearch?: number
} {
return {
useOnDiskStorage: this.useOnDiskStorage,
memoryMapThreshold: this.memoryMapThreshold,
hnswEfSearch: this.hnswEfSearch,
}
}
/**
* Gets the current model ID being used for embeddings.
*/

8
src/services/code-index/interfaces/config.ts
View file

@ -18,6 +18,10 @@ export interface CodeIndexConfig {
qdrantApiKey?: string
searchMinScore?: number
searchMaxResults?: number
// Memory optimization settings
useOnDiskStorage?: boolean // Enable on-disk storage for vectors and indexes
memoryMapThreshold?: number // Number of vectors before using memory-mapped files
hnswEfSearch?: number // HNSW search parameter (lower = less memory, faster)
}
/**
@ -37,4 +41,8 @@ export type PreviousConfigSnapshot = {
mistralApiKey?: string
qdrantUrl?: string
qdrantApiKey?: string
// Memory optimization settings
useOnDiskStorage?: boolean
memoryMapThreshold?: number
hnswEfSearch?: number
}

13
src/services/code-index/service-factory.ts
View file

@ -136,8 +136,17 @@ export class CodeIndexServiceFactory {
throw new Error(t("embeddings:serviceFactory.qdrantUrlMissing"))
}
// Assuming constructor is updated: new QdrantVectorStore(workspacePath, url, vectorSize, apiKey?)
return new QdrantVectorStore(this.workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey)
// Get memory optimization config from config manager
const memoryOptimization = this.configManager.memoryOptimizationConfig
// Create QdrantVectorStore with memory optimization settings
return new QdrantVectorStore(
this.workspacePath,
config.qdrantUrl,
vectorSize,
config.qdrantApiKey,
memoryOptimization,
)
}
/**

92
src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts
View file

@ -528,6 +528,25 @@ describe("QdrantVectorStore", () => {
vectors: {
size: mockVectorSize,
distance: "Cosine", // Assuming 'Cosine' is the DISTANCE_METRIC
on_disk: true, // Default memory optimization
},
hnsw_config: {
m: 16,
ef_construct: 100,
full_scan_threshold: 10000,
max_indexing_threads: 0,
on_disk: true,
payload_m: null,
},
optimizers_config: {
deleted_threshold: 0.2,
vacuum_min_vector_number: 1000,
default_segment_number: 2,
max_segment_size: null,
memmap_threshold: 50000,
indexing_threshold: 20000,
flush_interval_sec: 5,
max_optimization_threads: 0,
},
})
expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled()
@ -606,6 +625,25 @@ describe("QdrantVectorStore", () => {
vectors: {
size: mockVectorSize, // Should use the new, correct vector size
distance: "Cosine",
on_disk: true, // Default memory optimization
},
hnsw_config: {
m: 16,
ef_construct: 100,
full_scan_threshold: 10000,
max_indexing_threads: 0,
on_disk: true,
payload_m: null,
},
optimizers_config: {
deleted_threshold: 0.2,
vacuum_min_vector_number: 1000,
default_segment_number: 2,
max_segment_size: null,
memmap_threshold: 50000,
indexing_threshold: 20000,
flush_interval_sec: 5,
max_optimization_threads: 0,
},
})
@ -899,6 +937,25 @@ describe("QdrantVectorStore", () => {
vectors: {
size: newVectorSize, // Should create with new 768 dimensions
distance: "Cosine",
on_disk: true, // Default memory optimization
},
hnsw_config: {
m: 16,
ef_construct: 100,
full_scan_threshold: 10000,
max_indexing_threads: 0,
on_disk: true,
payload_m: null,
},
optimizers_config: {
deleted_threshold: 0.2,
vacuum_min_vector_number: 1000,
default_segment_number: 2,
max_segment_size: null,
memmap_threshold: 50000,
indexing_threshold: 20000,
flush_interval_sec: 5,
max_optimization_threads: 0,
},
})
expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(5)
@ -1244,8 +1301,13 @@ describe("QdrantVectorStore", () => {
score_threshold: DEFAULT_SEARCH_MIN_SCORE,
limit: DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
hnsw_ef: 64, // Default memory optimized value
exact: false,
quantization: {
ignore: false,
rescore: true,
oversampling: 2.0,
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],
@ -1295,8 +1357,13 @@ describe("QdrantVectorStore", () => {
score_threshold: DEFAULT_SEARCH_MIN_SCORE,
limit: DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
hnsw_ef: 64, // Default memory optimized value
exact: false,
quantization: {
ignore: false,
rescore: true,
oversampling: 2.0,
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],
@ -1321,8 +1388,13 @@ describe("QdrantVectorStore", () => {
score_threshold: customMinScore,
limit: DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
hnsw_ef: 64, // Default memory optimized value
exact: false,
quantization: {
ignore: false,
rescore: true,
oversampling: 2.0,
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],
@ -1345,8 +1417,13 @@ describe("QdrantVectorStore", () => {
score_threshold: DEFAULT_SEARCH_MIN_SCORE,
limit: customMaxResults,
params: {
hnsw_ef: 128,
hnsw_ef: 64, // Default memory optimized value
exact: false,
quantization: {
ignore: false,
rescore: true,
oversampling: 2.0,
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],
@ -1492,8 +1569,13 @@ describe("QdrantVectorStore", () => {
score_threshold: DEFAULT_SEARCH_MIN_SCORE,
limit: DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
hnsw_ef: 64, // Default memory optimized value
exact: false,
quantization: {
ignore: false,
rescore: true,
oversampling: 2.0,
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],

74
src/services/code-index/vector-store/qdrant-client.ts
View file

@ -17,13 +17,31 @@ export class QdrantVectorStore implements IVectorStore {
private client: QdrantClient
private readonly collectionName: string
private readonly qdrantUrl: string = "http://localhost:6333"
private readonly memoryOptimization: {
useOnDiskStorage?: boolean
memoryMapThreshold?: number
hnswEfSearch?: number
}
/**
* Creates a new Qdrant vector store
* @param workspacePath Path to the workspace
* @param url Optional URL to the Qdrant server
* @param vectorSize Size of the vectors
* @param apiKey Optional API key for authentication
* @param memoryOptimization Optional memory optimization settings
*/
constructor(workspacePath: string, url: string, vectorSize: number, apiKey?: string) {
constructor(
workspacePath: string,
url: string,
vectorSize: number,
apiKey?: string,
memoryOptimization?: {
useOnDiskStorage?: boolean
memoryMapThreshold?: number
hnswEfSearch?: number
},
) {
// Parse the URL to determine the appropriate QdrantClient configuration
const parsedUrl = this.parseQdrantUrl(url)
@ -79,6 +97,11 @@ export class QdrantVectorStore implements IVectorStore {
const hash = createHash("sha256").update(workspacePath).digest("hex")
this.vectorSize = vectorSize
this.collectionName = `ws-${hash.substring(0, 16)}`
this.memoryOptimization = memoryOptimization || {
useOnDiskStorage: true,
memoryMapThreshold: 50000,
hnswEfSearch: 64,
}
}
/**
@ -155,6 +178,27 @@ export class QdrantVectorStore implements IVectorStore {
vectors: {
size: this.vectorSize,
distance: this.DISTANCE_METRIC,
on_disk: this.memoryOptimization.useOnDiskStorage ?? true, // Store vectors on disk instead of RAM
},
// Configure HNSW index for memory efficiency
hnsw_config: {
m: 16, // Number of bi-directional links created for each node
ef_construct: 100, // Size of the dynamic list during construction
full_scan_threshold: 10000, // Use full scan for small collections
max_indexing_threads: 0, // Use all available CPU cores
on_disk: this.memoryOptimization.useOnDiskStorage ?? true, // Store HNSW index on disk
payload_m: null, // Use default payload index
},
// Enable memory-mapped storage for better memory management
optimizers_config: {
deleted_threshold: 0.2, // Trigger optimization when 20% vectors are deleted
vacuum_min_vector_number: 1000, // Minimum vectors before vacuum
default_segment_number: 2, // Number of segments to create
max_segment_size: null, // No limit on segment size
memmap_threshold: this.memoryOptimization.memoryMapThreshold ?? 50000, // Use mmap for segments larger than this
indexing_threshold: 20000, // Start indexing after 20k vectors
flush_interval_sec: 5, // Flush to disk every 5 seconds
max_optimization_threads: 0, // Use all available CPU cores
},
})
created = true
@ -244,6 +288,27 @@ export class QdrantVectorStore implements IVectorStore {
vectors: {
size: this.vectorSize,
distance: this.DISTANCE_METRIC,
on_disk: this.memoryOptimization.useOnDiskStorage ?? true, // Store vectors on disk instead of RAM
},
// Configure HNSW index for memory efficiency
hnsw_config: {
m: 16, // Number of bi-directional links created for each node
ef_construct: 100, // Size of the dynamic list during construction
full_scan_threshold: 10000, // Use full scan for small collections
max_indexing_threads: 0, // Use all available CPU cores
on_disk: this.memoryOptimization.useOnDiskStorage ?? true, // Store HNSW index on disk
payload_m: null, // Use default payload index
},
// Enable memory-mapped storage for better memory management
optimizers_config: {
deleted_threshold: 0.2, // Trigger optimization when 20% vectors are deleted
vacuum_min_vector_number: 1000, // Minimum vectors before vacuum
default_segment_number: 2, // Number of segments to create
max_segment_size: null, // No limit on segment size
memmap_threshold: this.memoryOptimization.memoryMapThreshold ?? 50000, // Use mmap for segments larger than this
indexing_threshold: 20000, // Start indexing after 20k vectors
flush_interval_sec: 5, // Flush to disk every 5 seconds
max_optimization_threads: 0, // Use all available CPU cores
},
})
console.log(`[QdrantVectorStore] Successfully created new collection ${this.collectionName}`)
@ -391,8 +456,13 @@ export class QdrantVectorStore implements IVectorStore {
score_threshold: minScore ?? DEFAULT_SEARCH_MIN_SCORE,
limit: maxResults ?? DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
hnsw_ef: this.memoryOptimization.hnswEfSearch ?? 64, // Use configured value or default
exact: false,
quantization: {
ignore: false, // Enable quantization for memory efficiency
rescore: true, // Rescore with original vectors for accuracy
oversampling: 2.0, // Oversample to maintain quality
},
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],