From 141de12b62ab10e315834e40417028b9758ad756 Mon Sep 17 00:00:00 2001
From: Algorithm5838 <108630393+Algorithm5838@users.noreply.github.com>
Date: Thu, 9 Apr 2026 14:23:33 +0300
Subject: [PATCH] perf: pre-lex details inner content at tokenize time

---
 .../chat/Messages/Markdown/MarkdownTokens.svelte    |  6 +++---
 src/lib/utils/marked/extension.ts                   | 13 +++++++++++--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte
index 961b731924..e060428290 100644
--- a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte
+++ b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte
@@ -6,7 +6,7 @@
 	import fileSaver from 'file-saver';
 	const { saveAs } = fileSaver;
 
-	import { marked, type Token } from 'marked';
+	import { type Token } from 'marked';
 	import { copyToClipboard, unescapeHtml } from '$lib/utils';
 
 	import { WEBUI_BASE_URL } from '$lib/constants';
@@ -396,7 +396,7 @@
 							<div class="mb-1.5" slot="content">
 								<svelte:self
 									id={`${id}-${tokenIdx}-${detailIdx}-d`}
-									tokens={marked.lexer(decode(detailToken.text))}
+									tokens={detailToken.tokens ?? []}
 									attributes={detailToken?.attributes}
 									{done}
 									{editCodeBlock}
@@ -443,7 +443,7 @@
 				<div class=" mb-1.5" slot="content">
 					<svelte:self
 						id={`${id}-${tokenIdx}-d`}
-						tokens={marked.lexer(decode(token.text))}
+						tokens={token.tokens ?? []}
 						attributes={token?.attributes}
 						{done}
 						{editCodeBlock}
diff --git a/src/lib/utils/marked/extension.ts b/src/lib/utils/marked/extension.ts
index eec17adbd0..1558c05fa4 100644
--- a/src/lib/utils/marked/extension.ts
+++ b/src/lib/utils/marked/extension.ts
@@ -1,3 +1,6 @@
+import { decode } from 'html-entities';
+import { type TokenizerThis } from 'marked';
+
 // Helper function to find matching closing tag
 function findMatchingClosingTag(src: string, openTag: string, closeTag: string): number {
 	let depth = 1;
@@ -26,7 +29,7 @@ function parseAttributes(tag: string): { [key: string]: string } {
 	return attributes;
 }
 
-function detailsTokenizer(src: string) {
+function detailsTokenizer(this: TokenizerThis, src: string) {
 	// Updated regex to capture attributes inside <details>
 	const detailsRegex = /^<details(\s+[^>]*)?>\n/;
 	const summaryRegex = /^<summary>(.*?)<\/summary>\n/;
@@ -49,11 +52,17 @@ function detailsTokenizer(src: string) {
 			content = content.slice(summaryMatch[0].length).trim();
 		}
 
+		const tokens: any[] = [];
+		if (attributes?.type !== 'tool_calls') {
+			this.lexer.blockTokens(decode(content), tokens);
+		}
+
 		return {
 			type: 'details',
 			raw: fullMatch,
 			summary: summary,
 			text: content,
+			tokens,
 			attributes: attributes // Include extracted attributes from <details>
 		};
 	}
@@ -89,6 +98,6 @@ function detailsExtension() {
 
 export default function (options = {}) {
 	return {
-		extensions: [detailsExtension(options)]
+		extensions: [detailsExtension()]
 	};
 }