From d2c85760fb575bd735eec68cf64830786ef4fe1e Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:20:33 -0800 Subject: [PATCH] Add a block anchor fallback match to diff search --- src/core/assistant-message/diff.ts | 227 ++++++++++++++++++++--------- 1 file changed, 161 insertions(+), 66 deletions(-) diff --git a/src/core/assistant-message/diff.ts b/src/core/assistant-message/diff.ts index f492c2f461..125e111bcc 100644 --- a/src/core/assistant-message/diff.ts +++ b/src/core/assistant-message/diff.ts @@ -1,64 +1,3 @@ -/** - * This function reconstructs the file content by applying a streamed diff (in a - * specialized SEARCH/REPLACE block format) to the original file content. It is designed - * to handle both incremental updates and the final resulting file after all chunks have - * been processed. - * - * The diff format is a custom structure that uses three markers to define changes: - * - * <<<<<<< SEARCH - * [Exact content to find in the original file] - * ======= - * [Content to replace with] - * >>>>>>> REPLACE - * - * Behavior and Assumptions: - * 1. The file is processed chunk-by-chunk. Each chunk of `diffContent` may contain - * partial or complete SEARCH/REPLACE blocks. By calling this function with each - * incremental chunk (with `isFinal` indicating the last chunk), the final reconstructed - * file content is produced. - * - * 2. Exact Matching with Fallback: - * - For each SEARCH block, the exact text must appear in the original file after - * `lastProcessedIndex`. If it does, that portion of the original file will be replaced. - * - If no exact match is found using a plain `indexOf`, we fall back to a line-by-line - * comparison that ignores leading/trailing whitespace. This is useful if the AI-generated - * search content differs slightly in indentation or spacing from the original code. - * - If neither exact nor trimmed line-based match is found, an error is thrown. - * - * 3. Empty SEARCH Section: - * - If SEARCH is empty and the original file is empty, this indicates creating a new file - * (pure insertion). - * - If SEARCH is empty and the original file is not empty, this indicates a complete - * file replacement (the entire original content is considered matched and replaced). - * - * 4. Applying Changes: - * - Before encountering the "=======" marker, lines are accumulated as search content. - * - After "=======" and before ">>>>>>> REPLACE", lines are accumulated as replacement content. - * - Once the block is complete (">>>>>>> REPLACE"), the matched section in the original - * file is replaced with the accumulated replacement lines, and the position in the original - * file is advanced. - * - * 5. Incremental Output: - * - As soon as the match location is found and we are in the REPLACE section, each new - * replacement line is appended to the result so that partial updates can be viewed - * incrementally. - * - * 6. Partial Markers: - * - If the final line of the chunk looks like it might be part of a marker but is not one - * of the known markers, it is removed. This prevents incomplete or partial markers - * from corrupting the output. - * - * 7. Finalization: - * - Once all chunks have been processed (when `isFinal` is true), any remaining original - * content after the last replaced section is appended to the result. - * - Trailing newlines are not forcibly added. The code tries to output exactly what is specified. - * - * Errors: - * - If the search block cannot be matched exactly or with the line-trimmed fallback approach, - * an error is thrown. - */ - /** * Attempts a line-trimmed fallback match for the given search content in the original content. * It tries to match `searchContent` lines against a block of lines in `originalContent` starting @@ -125,6 +64,152 @@ function lineTrimmedFallbackMatch( return false } +/** + * Attempts to match blocks of code by using the first and last lines as anchors. + * This is a third-tier fallback strategy that helps match blocks where we can identify + * the correct location by matching the beginning and end, even if the exact content + * differs slightly. + * + * The matching strategy: + * 1. Only attempts to match blocks of 3 or more lines to avoid false positives + * 2. Extracts from the search content: + * - First line as the "start anchor" + * - Last line as the "end anchor" + * 3. For each position in the original content: + * - Checks if the next line matches the start anchor + * - If it does, jumps ahead by the search block size + * - Checks if that line matches the end anchor + * - All comparisons are done after trimming whitespace + * + * This approach is particularly useful for matching blocks of code where: + * - The exact content might have minor differences + * - The beginning and end of the block are distinctive enough to serve as anchors + * - The overall structure (number of lines) remains the same + * + * @param originalContent - The full content of the original file + * @param searchContent - The content we're trying to find in the original file + * @param startIndex - The character index in originalContent where to start searching + * @returns A tuple of [startIndex, endIndex] if a match is found, false otherwise + */ +function blockAnchorFallbackMatch( + originalContent: string, + searchContent: string, + startIndex: number, +): [number, number] | false { + const originalLines = originalContent.split("\n") + const searchLines = searchContent.split("\n") + + // Only use this approach for blocks of 3+ lines + if (searchLines.length < 3) { + return false + } + + // Trim trailing empty line if exists + if (searchLines[searchLines.length - 1] === "") { + searchLines.pop() + } + + const firstLineSearch = searchLines[0].trim() + const lastLineSearch = searchLines[searchLines.length - 1].trim() + const searchBlockSize = searchLines.length + + // Find the line number where startIndex falls + let startLineNum = 0 + let currentIndex = 0 + while (currentIndex < startIndex && startLineNum < originalLines.length) { + currentIndex += originalLines[startLineNum].length + 1 + startLineNum++ + } + + // Look for matching start and end anchors + for (let i = startLineNum; i <= originalLines.length - searchBlockSize; i++) { + // Check if first line matches + if (originalLines[i].trim() !== firstLineSearch) { + continue + } + + // Check if last line matches at the expected position + if (originalLines[i + searchBlockSize - 1].trim() !== lastLineSearch) { + continue + } + + // Calculate exact character positions + let matchStartIndex = 0 + for (let k = 0; k < i; k++) { + matchStartIndex += originalLines[k].length + 1 + } + + let matchEndIndex = matchStartIndex + for (let k = 0; k < searchBlockSize; k++) { + matchEndIndex += originalLines[i + k].length + 1 + } + + return [matchStartIndex, matchEndIndex] + } + + return false +} + +/** + * This function reconstructs the file content by applying a streamed diff (in a + * specialized SEARCH/REPLACE block format) to the original file content. It is designed + * to handle both incremental updates and the final resulting file after all chunks have + * been processed. + * + * The diff format is a custom structure that uses three markers to define changes: + * + * <<<<<<< SEARCH + * [Exact content to find in the original file] + * ======= + * [Content to replace with] + * >>>>>>> REPLACE + * + * Behavior and Assumptions: + * 1. The file is processed chunk-by-chunk. Each chunk of `diffContent` may contain + * partial or complete SEARCH/REPLACE blocks. By calling this function with each + * incremental chunk (with `isFinal` indicating the last chunk), the final reconstructed + * file content is produced. + * + * 2. Exact Matching with Fallback: + * - For each SEARCH block, the exact text must appear in the original file after + * `lastProcessedIndex`. If it does, that portion of the original file will be replaced. + * - If no exact match is found using a plain `indexOf`, we fall back to a line-by-line + * comparison that ignores leading/trailing whitespace. This is useful if the AI-generated + * search content differs slightly in indentation or spacing from the original code. + * - If neither exact nor trimmed line-based match is found, an error is thrown. + * + * 3. Empty SEARCH Section: + * - If SEARCH is empty and the original file is empty, this indicates creating a new file + * (pure insertion). + * - If SEARCH is empty and the original file is not empty, this indicates a complete + * file replacement (the entire original content is considered matched and replaced). + * + * 4. Applying Changes: + * - Before encountering the "=======" marker, lines are accumulated as search content. + * - After "=======" and before ">>>>>>> REPLACE", lines are accumulated as replacement content. + * - Once the block is complete (">>>>>>> REPLACE"), the matched section in the original + * file is replaced with the accumulated replacement lines, and the position in the original + * file is advanced. + * + * 5. Incremental Output: + * - As soon as the match location is found and we are in the REPLACE section, each new + * replacement line is appended to the result so that partial updates can be viewed + * incrementally. + * + * 6. Partial Markers: + * - If the final line of the chunk looks like it might be part of a marker but is not one + * of the known markers, it is removed. This prevents incomplete or partial markers + * from corrupting the output. + * + * 7. Finalization: + * - Once all chunks have been processed (when `isFinal` is true), any remaining original + * content after the last replaced section is appended to the result. + * - Trailing newlines are not forcibly added. The code tries to output exactly what is specified. + * + * Errors: + * - If the search block cannot be matched exactly or with the line-trimmed fallback approach, + * an error is thrown. + */ export async function constructNewFileContent( diffContent: string, originalContent: string, @@ -196,17 +281,27 @@ export async function constructNewFileContent( searchEndIndex = exactIndex + currentSearchContent.length } else { // Attempt fallback line-trimmed matching - const fallbackMatch = lineTrimmedFallbackMatch( + const lineMatch = lineTrimmedFallbackMatch( originalContent, currentSearchContent, lastProcessedIndex, ) - if (fallbackMatch) { - ;[searchMatchIndex, searchEndIndex] = fallbackMatch + if (lineMatch) { + ;[searchMatchIndex, searchEndIndex] = lineMatch } else { - throw new Error( - `The SEARCH block:\n${currentSearchContent.trimEnd()}\n...does not match anything in the file.\nThis is likely due to differences in whitespace or line endings between the SEARCH block and the actual file. Try again with a more precise SEARCH block.\n(If you keep running into this error, you may use the write_to_file tool as a workaround.)`, + // Try block anchor fallback for larger blocks + const blockMatch = blockAnchorFallbackMatch( + originalContent, + currentSearchContent, + lastProcessedIndex, ) + if (blockMatch) { + ;[searchMatchIndex, searchEndIndex] = blockMatch + } else { + throw new Error( + `The SEARCH block:\n${currentSearchContent.trimEnd()}\n...does not match anything in the file.\nThis is likely due to differences in whitespace or line endings between the SEARCH block and the actual file. Try again with a more precise SEARCH block.\n(If you keep running into this error, you may use the write_to_file tool as a workaround.)`, + ) + } } } }