This commit is contained in:
Timothy Jaeryang Baek 2026-10-29 17:39:31 +09:00
parent df61370456
commit 44a9d499db
6 changed files with 130 additions and 104 deletions

View file

@ -58,6 +58,51 @@ def chat_search_terms(text: str) -> list[str]:
return list(dict.fromkeys(re.findall(r'[a-z0-9]+', text.lower())))
def chat_message_list(chat: dict) -> list:
if not isinstance(chat, dict):
return []
history = chat.get('history')
messages = history.get('messages') if isinstance(history, dict) else None
if not isinstance(messages, (dict, list)) or not messages:
messages = chat.get('messages')
if isinstance(messages, dict):
return list(messages.values())
return messages if isinstance(messages, list) else []
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
search_text = chat_search_content_query(search_text)
if not search_text:
return None
messages = chat_message_list(chat)
needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)]))
for needle in needles:
for message in messages:
if not isinstance(message, dict):
continue
content = message.get('content')
if not isinstance(content, str):
continue
index = content.lower().find(needle)
if index == -1:
continue
start = max(index - max_length // 2, 0)
end = min(start + max_length, len(content))
if index + len(needle) > end:
end = min(index + len(needle), len(content))
start = max(end - max_length, 0)
snippet = ' '.join(content[start:end].split())
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
return None
def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
if dialect_name == 'sqlite':
return f"""
@ -65,12 +110,14 @@ def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
EXISTS (
SELECT 1
FROM json_each(Chat.chat, '$.history.messages') AS history_message
WHERE LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
WHERE LOWER(CASE WHEN history_message.type = 'object'
THEN history_message.value->>'content' END) LIKE '%' || :{key} || '%'
)
OR EXISTS (
SELECT 1
FROM json_each(Chat.chat, '$.messages') AS legacy_message
WHERE LOWER(legacy_message.value->>'content') LIKE '%' || :{key} || '%'
WHERE LOWER(CASE WHEN legacy_message.type = 'object'
THEN legacy_message.value->>'content' END) LIKE '%' || :{key} || '%'
)
)
"""
@ -88,13 +135,15 @@ def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
)
OR EXISTS (
SELECT 1
FROM json_each(Chat.chat#>'{{history,messages}}') AS history_message
FROM json_each(CASE WHEN json_typeof(Chat.chat#>'{{history,messages}}') = 'object'
THEN Chat.chat#>'{{history,messages}}' ELSE '{{}}'::json END) AS history_message
WHERE json_typeof(history_message.value->'content') = 'string'
AND LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
)
OR EXISTS (
SELECT 1
FROM json_array_elements(Chat.chat->'messages') AS legacy_message
FROM json_array_elements(CASE WHEN json_typeof(Chat.chat->'messages') = 'array'
THEN Chat.chat->'messages' ELSE '[]'::json END) AS legacy_message
WHERE json_typeof(legacy_message->'content') = 'string'
AND LOWER(legacy_message->>'content') LIKE '%' || :{key} || '%'
)
@ -2171,9 +2220,9 @@ class ChatTable:
skip: int = 0,
limit: int = 60,
db: AsyncSession | None = None,
) -> list[ChatModel]:
) -> list[ChatTitleIdResponse]:
"""
Filters chats based on a search query using Python, allowing pagination using skip and limit.
Search chats without loading their bodies, with pagination applied in SQL.
"""
search_text = sanitize_text_for_db(search_text).lower().strip()
@ -2219,7 +2268,9 @@ class ChatTable:
search_terms = [term for term in chat_search_terms(phrase_query) if term != phrase_query]
async with get_async_db_context(db) as session:
stmt = select(Chat).filter(Chat.user_id == user_id)
stmt = select(
Chat.id, Chat.title, Chat.updated_at, Chat.created_at, Chat.last_read_at, Chat.archived
).filter(Chat.user_id == user_id)
stmt = stmt.where(Chat.meta['internal'].as_boolean().is_not(True))
if is_archived is not None:
@ -2347,12 +2398,21 @@ class ChatTable:
# Perform pagination at the SQL level
stmt = stmt.offset(skip).limit(limit)
result = await session.execute(stmt)
all_chats = result.scalars().all()
all_chats = result.mappings().all()
log.info('The number of chats: %s', len(all_chats))
# Validate and return chats
return [ChatModel.model_validate(chat) for chat in all_chats]
return [ChatTitleIdResponse.model_validate(chat) for chat in all_chats]
async def get_chat_search_snippets(
self, ids: list[str], user_id: str, search_text: str, db: AsyncSession | None = None
) -> dict[str, str | None]:
if not ids or not chat_search_content_query(search_text):
return {}
async with get_async_db_context(db) as session:
result = await session.execute(select(Chat.id, Chat.chat).where(Chat.id.in_(ids), Chat.user_id == user_id))
return {id: chat_search_snippet(body, search_text) for id, body in result.all()}
async def get_chats_by_folder_id_and_user_id(
self,

View file

@ -30,8 +30,6 @@ from open_webui.models.chats import (
ChatTitleIdResponse,
ChatUsageStatsListResponse,
MessageStats,
chat_search_content_query,
chat_search_terms,
)
from open_webui.models.config import Config
from open_webui.models.folders import Folders
@ -227,47 +225,6 @@ class CompactChatForm(BaseModel):
model: str | None = None
def chat_search_content_text(text: str) -> str:
return chat_search_content_query(text)
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
if not search_text:
return None
history = chat.get('history', {})
messages = history.get('messages') if isinstance(history, dict) else None
if not messages:
messages = chat.get('messages', []) or []
if isinstance(messages, dict):
messages = messages.values()
needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)]))
for needle in needles:
for message in messages:
if not isinstance(message, dict):
continue
content = message.get('content')
if not isinstance(content, str):
continue
index = content.lower().find(needle)
if index == -1:
continue
start = max(index - max_length // 2, 0)
end = min(start + max_length, len(content))
if index + len(needle) > end:
end = min(index + len(needle), len(content))
start = max(end - max_length, 0)
snippet = ' '.join(content[start:end].split())
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
return None
async def get_chat_config_values() -> dict:
values = await Config.get_many(*CHAT_CONFIG_KEYS.values())
config = {field: values[storage_key] for field, storage_key in CHAT_CONFIG_KEYS.items() if storage_key in values}
@ -954,21 +911,7 @@ async def search_user_chats(
limit = 60
skip = (page - 1) * limit
search_text = chat_search_content_text(text)
chat_list = []
for chat in await Chats.get_chats_by_user_id_and_search_text(user.id, text, skip=skip, limit=limit, db=db):
# Explicit fields: model_dump() would deep-copy the entire chat blob per row
chat_list.append(
ChatTitleIdResponse(
id=chat.id,
title=chat.title,
updated_at=chat.updated_at,
created_at=chat.created_at,
last_read_at=chat.last_read_at,
snippet=chat_search_snippet(chat.chat, search_text),
archived=chat.archived,
)
)
chat_list = await Chats.get_chats_by_user_id_and_search_text(user.id, text, skip=skip, limit=limit, db=db)
# Delete tag if no chat is found
words = text.strip().split(' ')
@ -982,6 +925,17 @@ async def search_user_chats(
return await add_active_state_to_chat_list(request, chat_list)
@router.get('/{id}/search/snippet', response_model=str | None)
async def get_chat_search_snippet(
id: str,
text: str,
user=Depends(get_verified_user),
db: AsyncSession = Depends(get_async_session),
):
snippets = await Chats.get_chat_search_snippets([id], user.id, text, db=db)
return snippets.get(id)
############################
# GetChatsByFolderId
############################

View file

@ -7,7 +7,7 @@ from fastapi import APIRouter, Depends, Query
from open_webui.internal.db import get_async_session
from open_webui.models.access_grants import AccessGrant, AccessGrants
from open_webui.models.channels import ChannelMember, Channels
from open_webui.models.chats import Chat, chat_search_content_query
from open_webui.models.chats import Chat, chat_message_list, chat_search_content_query
from open_webui.models.config import Config
from open_webui.models.folders import Folder, Folders
from open_webui.models.groups import Groups
@ -73,9 +73,7 @@ def excerpt(content: str, query: str, length: int = 200) -> str:
def chat_passage(data: dict, query: str) -> tuple[str, str | None]:
history = (data.get('history') or {}).get('messages') or {}
messages = list(history.values()) if history else data.get('messages') or []
messages = [m for m in messages if isinstance(m, dict) and isinstance(m.get('content'), str)]
messages = [m for m in chat_message_list(data) if isinstance(m, dict) and isinstance(m.get('content'), str)]
terms = terms_for(query)
match = next((m for m in messages if terms and all(t in m['content'].lower() for t in terms)), None)
if match is None:
@ -97,13 +95,19 @@ def message_match_sql(table: str, dialect: str, key: str):
return text(
'('
+ ' OR '.join(
f"EXISTS (SELECT 1 FROM {source} AS m WHERE json_type(m.value, '$.content') = 'text' "
f"AND lower(m.value ->> 'content') LIKE :{key} ESCAPE '\\')"
f"EXISTS (SELECT 1 FROM {source} AS m WHERE lower(CASE WHEN m.type = 'object' "
f"AND json_type(m.value, '$.content') = 'text' THEN m.value ->> 'content' END) "
f"LIKE :{key} ESCAPE '\\')"
for source in sources
)
+ ')'
)
sources = [f"json_each({table}.chat->'history'->'messages')", f"json_array_elements({table}.chat->'messages')"]
sources = [
f"json_each(CASE WHEN json_typeof({table}.chat->'history'->'messages') = 'object' "
f"THEN {table}.chat->'history'->'messages' ELSE '{{}}'::json END)",
f"json_array_elements(CASE WHEN json_typeof({table}.chat->'messages') = 'array' "
f"THEN {table}.chat->'messages' ELSE '[]'::json END)",
]
return text(
'('
+ ' OR '.join(

View file

@ -1676,7 +1676,7 @@ async def search_chats(
limit=count * 3, # Fetch more for filtering
)
results = []
matched = []
for chat in chats:
# Skip the current chat to avoid showing it in search results
if __chat_id__ and chat.id == __chat_id__:
@ -1688,31 +1688,16 @@ async def search_chats(
if end_timestamp and chat.updated_at > end_timestamp:
continue
# Find a matching message snippet
snippet = ''
messages = (getattr(chat, 'chat', None) or {}).get('history', {}).get('messages', {})
if not messages:
messages = (getattr(chat, 'chat', None) or {}).get('messages', {}) or {}
if isinstance(messages, list):
messages = {str(idx): message for idx, message in enumerate(messages)}
lower_query = chat_search_content_query(query)
needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else []
for needle in needles:
for msg_id, msg in messages.items():
content = msg.get('content', '') if isinstance(msg, dict) else ''
if isinstance(content, str) and needle in content.lower():
idx = content.lower().find(needle)
start = max(0, idx - 50)
end = min(len(content), idx + len(needle) + 100)
snippet = (
('...' if start > 0 else '') + content[start:end] + ('...' if end < len(content) else '')
)
break
if snippet:
break
matched.append(chat)
if len(matched) >= count:
break
snippets = await Chats.get_chat_search_snippets([chat.id for chat in matched], user_id, query)
lower_query = chat_search_content_query(query)
needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else []
results = []
for chat in matched:
snippet = snippets.get(chat.id) or ''
title = chat.title or ''
if not snippet and any(needle in title.lower() for needle in needles):
snippet = f'Title match: {title}'
@ -1726,9 +1711,6 @@ async def search_chats(
}
)
if len(results) >= count:
break
return JSONCodec.dumps(results, ensure_ascii=False)
except Exception as e:
log.exception(f'search_chats error: {e}')

View file

@ -481,6 +481,18 @@ export const getAllChats = async (token: string) => {
return chats;
};
export const getChatSearchSnippet = async (token: string, id: string, text: string) => {
const searchParams = new URLSearchParams({ text });
const res = await fetch(`${WEBUI_API_BASE_URL}/chats/${id}/search/snippet?${searchParams}`, {
headers: {
Accept: 'application/json',
...(token && { authorization: `Bearer ${token}` })
}
});
if (!res.ok) throw getErrorDetail(await res.json());
return (await res.json()) as string | null;
};
export const getChatListBySearchText = async (token: string, text: string, page: number = 1) => {
let error = null;

View file

@ -3,7 +3,7 @@
import { onDestroy, onMount, tick, getContext } from 'svelte';
import { decodeString } from '$lib/utils';
import { getChatList, getChatListBySearchText } from '$lib/apis/chats';
import { getChatList, getChatListBySearchText, getChatSearchSnippet } from '$lib/apis/chats';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import Spinner from '$lib/components/common/Spinner.svelte';
@ -17,7 +17,7 @@
let loaded = false;
let items = [];
let items: { id: string; name: string; updated_at: number; snippet?: string | null }[] = [];
let selectedIdx = 0;
let query = '';
@ -29,6 +29,18 @@
let searchDebounceTimer: ReturnType<typeof setTimeout>;
let requestId = 0;
const loadSnippet = async (item: { id: string; snippet?: string | null }) => {
if (!searchedQuery.trim() || item.snippet !== undefined) return;
item.snippet = null;
try {
item.snippet = await getChatSearchSnippet(localStorage.token, item.id, searchedQuery);
items = items;
} catch (error) {
item.snippet = undefined;
console.error(error);
}
};
// Only re-run the search when the query actually changes. Flipping `initialized`
// after the initial load would otherwise trigger a second, identical request.
$: if (initialized && query !== searchedQuery) {
@ -82,6 +94,7 @@
.map((item) => {
return {
...item,
snippet: undefined,
type: 'chat',
name: item.title
};
@ -145,6 +158,7 @@
.locale($i18n.language)
.fromNow()}
placement="top-start"
tippyOptions={{ onShown: () => loadSnippet(item) }}
>
<div class="line-clamp-1 flex-1">
{decodeString(item?.name)}