diff --git a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte index 11281526fe..4e61aeab18 100644 --- a/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte +++ b/src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte @@ -6,7 +6,7 @@ import fileSaver from 'file-saver'; const { saveAs } = fileSaver; - import { marked, type Token } from 'marked'; + import { type Token } from 'marked'; import { copyToClipboard, unescapeHtml } from '$lib/utils'; import { resolveChatMessageToolCall } from '$lib/apis/chats'; @@ -470,7 +470,7 @@ id={`${id}-${tokenIdx}-${detailIdx}-d`} {chatId} {messageId} - tokens={marked.lexer(decode(detailToken.text))} + tokens={detailToken.tokens ?? []} attributes={detailToken?.attributes} {done} {allowEmbeds} @@ -531,7 +531,7 @@ id={`${id}-${tokenIdx}-d`} {chatId} {messageId} - tokens={marked.lexer(decode(token.text))} + tokens={token.tokens ?? []} attributes={token?.attributes} {done} {allowEmbeds} diff --git a/src/lib/utils/marked/extension.ts b/src/lib/utils/marked/extension.ts index d82f70578e..37b6346ab9 100644 --- a/src/lib/utils/marked/extension.ts +++ b/src/lib/utils/marked/extension.ts @@ -1,3 +1,6 @@ +import { decode } from 'html-entities'; +import { type Token, type TokenizerThis } from 'marked'; + // Helper function to find matching closing tag function findMatchingClosingTag(src: string, openTag: string, closeTag: string): number { let depth = 1; @@ -26,7 +29,7 @@ function parseAttributes(tag: string): { [key: string]: string } { return attributes; } -function detailsTokenizer(src: string) { +function detailsTokenizer(this: TokenizerThis, src: string) { // Updated regex to capture attributes inside
const detailsRegex = /^]*)?>\n/; const summaryRegex = /^(.*?)<\/summary>\n/; @@ -49,11 +52,17 @@ function detailsTokenizer(src: string) { content = content.slice(summaryMatch[0].length).trim(); } + const tokens: Token[] = []; + if (attributes.type !== 'tool_calls') { + this.lexer.blockTokens(decode(content), tokens); + } + return { type: 'details', raw: fullMatch, summary: summary, text: content, + tokens, attributes: attributes // Include extracted attributes from
}; }