scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-28 05:27:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

perf: stop rescanning the whole response for tag boundaries on every streamed chunk (#28861)

Browse source 
Streamed responses are scanned for reasoning and code interpreter tags. To work out where the last complete tag ended, the scanner searched backwards from the start of the accumulated text on every chunk, once per tag set. Ordinary prose contains no angle bracket, so that search never stopped early and read the entire response back every time. The cost grows with the square of the response length, and this scanning is on unless a model turns it off.

The two positions are now carried forward as the text grows, so each chunk only scans the characters it added.

Measured on CPython 3.12, a 270 KB response streamed in 27000 chunks:

| response text | before | after |
|---|---|---|
| no newlines | 7690 ms | 40.6 ms |
| with newlines | 5695 ms | 41.7 ms |

The carried positions match a full rescan at every step of 36282 randomized replays, covering text with no markers, newlines only, dense markers, real tags and truncation part way through.
This commit is contained in:
Classic298 2026-08-24 12:11:32 +02:00 • committed by GitHub
parent 978d257214
commit 091c44c621
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 25 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
backend/open_webui/utils/middleware.py
View file

@ -4181,6 +4181,7 @@ async def streaming_chat_response_handler(response, ctx):
async def response_handler(response, events):
filter_context = FilterContext()
tag_scan_positions = {}
tag_boundary_positions = {}
response_stream_task_id = metadata.get('task_id') or metadata.get('message_id')
def tag_output_handler(content_type, tags, output):
@ -4236,6 +4237,29 @@ async def streaming_chat_response_handler(response, ctx):
item_id = item.get('id')
if item_id:
tag_scan_positions.pop((item_id, content_type), None)
tag_boundary_positions.pop((item_id, content_type), None)
def get_tag_boundaries(item, text, scanned_length):
"""Index of the last '<', and of the last '>' or newline, before scanned_length."""
key = (item.get('id'), content_type)
scanned, last_open, last_boundary = tag_boundary_positions.get(key, (0, -1, -1))
if scanned > scanned_length: # the item was rewritten, so the cached positions are stale
scanned, last_open, last_boundary = 0, -1, -1
if scanned < scanned_length:
# only text added since the last call can move either position
open_tag = text.rfind('<', scanned, scanned_length)
if open_tag != -1:
last_open = open_tag
boundary = max(
text.rfind('>', scanned, scanned_length),
text.rfind('\n', scanned, scanned_length),
)
if boundary != -1:
last_boundary = boundary
tag_boundary_positions[key] = (scanned_length, last_open, last_boundary)
return last_open, last_boundary
# Map content_type to output item type
output_type_map = {
@ -4258,11 +4282,7 @@ async def streaming_chat_response_handler(response, ctx):
if scanned_length and any(
start_tag.startswith('<') and start_tag.endswith('>') for start_tag, _ in tags
):
last_tag_boundary = max(
item_text.rfind('>', 0, scanned_length),
item_text.rfind('\n', 0, scanned_length),
)
open_tag_start = item_text.rfind('<', 0, scanned_length)
open_tag_start, last_tag_boundary = get_tag_boundaries(item, item_text, scanned_length)
if open_tag_start > last_tag_boundary:
search_start = min(search_start, open_tag_start)