mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-16 09:41:23 +00:00
perf: stop resending the whole message on every token when continuing a response
Continue Response made Open WebUI send the entire message so far to the browser on every token and, with Redis, rewrite the copy kept for page reloads each time, so continuing a long reply moved hundreds of megabytes for a few kilobytes of new text. The text updates of a continued reply now carry only the new text, like those of a normal reply, and the finished message is the same as before. Measured against a simulated provider continuing a 4,000-word reply: | | before | after | |---|---|---| | Data sent to the browser | 209.3 MB | 1.3 MB | | Redis writes for the reload copy | 4,002 | 13 |
This commit is contained in:
parent
8957ac1397
commit
9d432d659f
1 changed files with 22 additions and 16 deletions
|
|
@ -4549,15 +4549,18 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
finish_reason = None
|
||||
last_response_id = None
|
||||
|
||||
def full_output():
|
||||
if (
|
||||
def continues_prior_message() -> bool:
|
||||
return bool(
|
||||
continuing
|
||||
and prior_output
|
||||
and output
|
||||
and prior_output[-1].get('type') == output[0].get('type') == 'message'
|
||||
and prior_output[-1].get('role', 'assistant') == 'assistant'
|
||||
and output[0].get('role', 'assistant') == 'assistant'
|
||||
):
|
||||
)
|
||||
|
||||
def full_output():
|
||||
if continues_prior_message():
|
||||
return [
|
||||
*prior_output[:-1],
|
||||
{
|
||||
|
|
@ -4587,11 +4590,9 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
nonlocal stream_size
|
||||
if not stream_size or not request.app.state.redis:
|
||||
return
|
||||
# Continue merges items; code blocks write to `code`, not the event's text field.
|
||||
if (
|
||||
continuing
|
||||
or not isinstance(delta.get('delta'), str)
|
||||
or any(item.get('type') == 'open_webui:code_interpreter' for item in output)
|
||||
# Code blocks write to `code`, not the event's text field.
|
||||
if not isinstance(delta.get('delta'), str) or any(
|
||||
item.get('type') == 'open_webui:code_interpreter' for item in output
|
||||
):
|
||||
stream_size = 0
|
||||
return
|
||||
|
|
@ -4660,6 +4661,10 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
reasoning_tags = DEFAULT_REASONING_TAGS
|
||||
|
||||
try:
|
||||
# deltas of a continued reply assume the client already holds the prior output
|
||||
if prior_output:
|
||||
await event_emitter({'type': 'chat:completion', 'data': {'output': full_output()}})
|
||||
|
||||
for event in events:
|
||||
await event_emitter(
|
||||
{
|
||||
|
|
@ -4703,6 +4708,14 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
|
||||
def get_response_data_with_full_output_index(response_data: dict):
|
||||
if prior_output and isinstance(response_data.get('output_index'), int):
|
||||
if continues_prior_message():
|
||||
output_index = response_data['output_index']
|
||||
content_offset = len(prior_output[-1].get('content', [])) if output_index == 0 else 0
|
||||
return {
|
||||
**response_data,
|
||||
'output_index': output_index + len(prior_output) - 1,
|
||||
'content_index': response_data.get('content_index', 0) + content_offset,
|
||||
}
|
||||
return {
|
||||
**response_data,
|
||||
'output_index': response_data['output_index'] + len(prior_output),
|
||||
|
|
@ -4724,14 +4737,7 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
nonlocal last_delta_key
|
||||
|
||||
if delta_count >= threshold and last_delta_data:
|
||||
await event_emitter(
|
||||
{
|
||||
'type': 'chat:completion' if continuing else 'response:completion',
|
||||
'data': {'output': full_output(), 'type': last_delta_data.get('type')}
|
||||
if continuing
|
||||
else last_delta_data,
|
||||
}
|
||||
)
|
||||
await event_emitter({'type': 'response:completion', 'data': last_delta_data})
|
||||
delta_count = 0
|
||||
last_delta_data = None
|
||||
last_delta_type = None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue