From 2eb26fcd9674129249dbcbcde8ee300a4d403509 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Fri, 8 Aug 2025 14:19:23 +0000 Subject: [PATCH] fix: prevent Ollama indexing from freezing at 69% - Add dynamic timeout for batch embedding operations based on batch size - Improve error handling and recovery in batch processing - Ensure errors are properly thrown to prevent silent failures - Add immediate error state updates when batch processing fails - Add better logging for debugging timeout and retry scenarios Fixes #6849 --- src/i18n/locales/en/embeddings.json | 3 +- src/services/code-index/embedders/ollama.ts | 14 +++++++-- src/services/code-index/orchestrator.ts | 4 +++ src/services/code-index/processors/scanner.ts | 31 +++++++++++++++++-- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/i18n/locales/en/embeddings.json b/src/i18n/locales/en/embeddings.json index 66465d8c35..390d843416 100644 --- a/src/i18n/locales/en/embeddings.json +++ b/src/i18n/locales/en/embeddings.json @@ -15,7 +15,8 @@ "serviceUnavailable": "Ollama service is unavailable (status: {{status}})", "modelNotFound": "Ollama model not found: {{modelId}}", "modelNotEmbeddingCapable": "Ollama model is not embedding capable: {{modelId}}", - "hostNotFound": "Ollama host not found: {{baseUrl}}" + "hostNotFound": "Ollama host not found: {{baseUrl}}", + "batchTimeoutError": "Ollama embedding timed out after {{timeout}} seconds while processing {{count}} texts. Consider reducing batch size or increasing timeout." }, "scanner": { "unknownErrorProcessingFile": "Unknown error processing file {{filePath}}", diff --git a/src/services/code-index/embedders/ollama.ts b/src/services/code-index/embedders/ollama.ts index 9688a15ff0..d05ac78645 100644 --- a/src/services/code-index/embedders/ollama.ts +++ b/src/services/code-index/embedders/ollama.ts @@ -69,8 +69,10 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { // Implementing based on user's specific request structure. // Add timeout to prevent indefinite hanging + // Use a longer timeout for batch operations as they can take more time + const batchTimeout = Math.max(OLLAMA_EMBEDDING_TIMEOUT_MS, texts.length * 2000) // At least 2 seconds per text const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const timeoutId = setTimeout(() => controller.abort(), batchTimeout) const response = await fetch(url, { method: "POST", @@ -125,7 +127,15 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { // Handle specific error types with better messages if (error.name === "AbortError") { - throw new Error(t("embeddings:validation.connectionFailed")) + // More specific timeout error message for batch operations + const timeoutMessage = + texts.length > 1 + ? t("embeddings:ollama.batchTimeoutError", { + count: texts.length, + timeout: Math.round(Math.max(OLLAMA_EMBEDDING_TIMEOUT_MS, texts.length * 2000) / 1000), + }) + : t("embeddings:validation.connectionFailed") + throw new Error(timeoutMessage) } else if (error.message?.includes("fetch failed") || error.code === "ECONNREFUSED") { throw new Error(t("embeddings:ollama.serviceNotRunning", { baseUrl: this.baseUrl })) } else if (error.code === "ENOTFOUND") { diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index fbc4a24118..55619dfe9f 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -138,11 +138,13 @@ export class CodeIndexOrchestrator { const handleFileParsed = (fileBlockCount: number) => { cumulativeBlocksFoundSoFar += fileBlockCount + // Update progress immediately when blocks are found this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } const handleBlocksIndexed = (indexedCount: number) => { cumulativeBlocksIndexed += indexedCount + // Update progress immediately when blocks are indexed this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } @@ -154,6 +156,8 @@ export class CodeIndexOrchestrator { batchError, ) batchErrors.push(batchError) + // Update state to show error immediately + this.stateManager.setSystemState("Error", `Indexing error: ${batchError.message}`) }, handleBlocksIndexed, handleFileParsed, diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 27362b8b74..db976598e4 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -385,8 +385,29 @@ export class DirectoryScanner implements IDirectoryScanner { } // --- End Deletion Step --- - // Create embeddings for batch - const { embeddings } = await this.embedder.createEmbeddings(batchTexts) + // Create embeddings for batch with better error context + let embeddings: number[][] + try { + const response = await this.embedder.createEmbeddings(batchTexts) + embeddings = response.embeddings + } catch (embeddingError: any) { + // Log specific details about the embedding failure + console.error( + `[DirectoryScanner] Embedding creation failed for batch of ${batchTexts.length} texts:`, + embeddingError, + ) + + // Check if it's a timeout error and provide more context + if (embeddingError.message?.includes("timed out") || embeddingError.message?.includes("timeout")) { + throw new Error( + `Embedding timeout for batch of ${batchTexts.length} texts. This may indicate the Ollama service is overloaded or the model is too slow. ${embeddingError.message}`, + { cause: embeddingError }, + ) + } + + // Re-throw with additional context + throw embeddingError + } // Prepare points for Qdrant const points = batchBlocks.map((block, index) => { @@ -420,7 +441,7 @@ export class DirectoryScanner implements IDirectoryScanner { } catch (error) { lastError = error as Error console.error( - `[DirectoryScanner] Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}:`, + `[DirectoryScanner] Error processing batch (attempt ${attempts}/${MAX_BATCH_RETRIES}) in workspace ${scanWorkspace}:`, error, ) TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, { @@ -433,6 +454,7 @@ export class DirectoryScanner implements IDirectoryScanner { if (attempts < MAX_BATCH_RETRIES) { const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1) + console.log(`[DirectoryScanner] Retrying batch processing in ${delay}ms...`) await new Promise((resolve) => setTimeout(resolve, delay)) } } @@ -454,6 +476,9 @@ export class DirectoryScanner implements IDirectoryScanner { ), ) } + // CRITICAL: Throw the error to ensure it's caught by the orchestrator + // This prevents the indexing from appearing to succeed when it actually failed + throw lastError } } }