scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-10-09 03:18:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

perf: pre-lex details inner content at tokenize time

Browse source
This commit is contained in:
Algorithm5838 2026-04-09 14:23:33 +03:00
parent cb0fd6ed41
commit 141de12b62
2 changed files with 14 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte
View file

@ -6,7 +6,7 @@
import fileSaver from 'file-saver';
const { saveAs } = fileSaver;
import { marked, type Token } from 'marked';
import { type Token } from 'marked';
import { copyToClipboard, unescapeHtml } from '$lib/utils';
import { WEBUI_BASE_URL } from '$lib/constants';
@ -396,7 +396,7 @@
<div class="mb-1.5" slot="content">
<svelte:self
id={`${id}-${tokenIdx}-${detailIdx}-d`}
tokens={marked.lexer(decode(detailToken.text))}
tokens={detailToken.tokens ?? []}
attributes={detailToken?.attributes}
{done}
{editCodeBlock}
@ -443,7 +443,7 @@
<div class=" mb-1.5" slot="content">
<svelte:self
id={`${id}-${tokenIdx}-d`}
tokens={marked.lexer(decode(token.text))}
tokens={token.tokens ?? []}
attributes={token?.attributes}
{done}
{editCodeBlock}

13
src/lib/utils/marked/extension.ts
View file

@ -1,3 +1,6 @@
import { decode } from 'html-entities';
import { type TokenizerThis } from 'marked';
// Helper function to find matching closing tag
function findMatchingClosingTag(src: string, openTag: string, closeTag: string): number {
let depth = 1;
@ -26,7 +29,7 @@ function parseAttributes(tag: string): { [key: string]: string } {
return attributes;
}
function detailsTokenizer(src: string) {
function detailsTokenizer(this: TokenizerThis, src: string) {
// Updated regex to capture attributes inside <details>
const detailsRegex = /^<details(\s+[^>]*)?>\n/;
const summaryRegex = /^<summary>(.*?)<\/summary>\n/;
@ -49,11 +52,17 @@ function detailsTokenizer(src: string) {
content = content.slice(summaryMatch[0].length).trim();
}
const tokens: any[] = [];
if (attributes?.type !== 'tool_calls') {
this.lexer.blockTokens(decode(content), tokens);
}
return {
type: 'details',
raw: fullMatch,
summary: summary,
text: content,
tokens,
attributes: attributes // Include extracted attributes from <details>
};
}
@ -89,6 +98,6 @@ function detailsExtension() {
export default function (options = {}) {
return {
extensions: [detailsExtension(options)]
extensions: [detailsExtension()]
};
}