mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-16 09:41:23 +00:00
refac
This commit is contained in:
parent
df61370456
commit
44a9d499db
6 changed files with 130 additions and 104 deletions
|
|
@ -58,6 +58,51 @@ def chat_search_terms(text: str) -> list[str]:
|
|||
return list(dict.fromkeys(re.findall(r'[a-z0-9]+', text.lower())))
|
||||
|
||||
|
||||
def chat_message_list(chat: dict) -> list:
|
||||
if not isinstance(chat, dict):
|
||||
return []
|
||||
history = chat.get('history')
|
||||
messages = history.get('messages') if isinstance(history, dict) else None
|
||||
if not isinstance(messages, (dict, list)) or not messages:
|
||||
messages = chat.get('messages')
|
||||
if isinstance(messages, dict):
|
||||
return list(messages.values())
|
||||
return messages if isinstance(messages, list) else []
|
||||
|
||||
|
||||
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
|
||||
search_text = chat_search_content_query(search_text)
|
||||
if not search_text:
|
||||
return None
|
||||
|
||||
messages = chat_message_list(chat)
|
||||
|
||||
needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)]))
|
||||
for needle in needles:
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
|
||||
content = message.get('content')
|
||||
if not isinstance(content, str):
|
||||
continue
|
||||
|
||||
index = content.lower().find(needle)
|
||||
if index == -1:
|
||||
continue
|
||||
|
||||
start = max(index - max_length // 2, 0)
|
||||
end = min(start + max_length, len(content))
|
||||
if index + len(needle) > end:
|
||||
end = min(index + len(needle), len(content))
|
||||
start = max(end - max_length, 0)
|
||||
|
||||
snippet = ' '.join(content[start:end].split())
|
||||
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
|
||||
if dialect_name == 'sqlite':
|
||||
return f"""
|
||||
|
|
@ -65,12 +110,14 @@ def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
|
|||
EXISTS (
|
||||
SELECT 1
|
||||
FROM json_each(Chat.chat, '$.history.messages') AS history_message
|
||||
WHERE LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||
WHERE LOWER(CASE WHEN history_message.type = 'object'
|
||||
THEN history_message.value->>'content' END) LIKE '%' || :{key} || '%'
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM json_each(Chat.chat, '$.messages') AS legacy_message
|
||||
WHERE LOWER(legacy_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||
WHERE LOWER(CASE WHEN legacy_message.type = 'object'
|
||||
THEN legacy_message.value->>'content' END) LIKE '%' || :{key} || '%'
|
||||
)
|
||||
)
|
||||
"""
|
||||
|
|
@ -88,13 +135,15 @@ def chat_search_message_content_match_sql(dialect_name: str, key: str) -> str:
|
|||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM json_each(Chat.chat#>'{{history,messages}}') AS history_message
|
||||
FROM json_each(CASE WHEN json_typeof(Chat.chat#>'{{history,messages}}') = 'object'
|
||||
THEN Chat.chat#>'{{history,messages}}' ELSE '{{}}'::json END) AS history_message
|
||||
WHERE json_typeof(history_message.value->'content') = 'string'
|
||||
AND LOWER(history_message.value->>'content') LIKE '%' || :{key} || '%'
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM json_array_elements(Chat.chat->'messages') AS legacy_message
|
||||
FROM json_array_elements(CASE WHEN json_typeof(Chat.chat->'messages') = 'array'
|
||||
THEN Chat.chat->'messages' ELSE '[]'::json END) AS legacy_message
|
||||
WHERE json_typeof(legacy_message->'content') = 'string'
|
||||
AND LOWER(legacy_message->>'content') LIKE '%' || :{key} || '%'
|
||||
)
|
||||
|
|
@ -2171,9 +2220,9 @@ class ChatTable:
|
|||
skip: int = 0,
|
||||
limit: int = 60,
|
||||
db: AsyncSession | None = None,
|
||||
) -> list[ChatModel]:
|
||||
) -> list[ChatTitleIdResponse]:
|
||||
"""
|
||||
Filters chats based on a search query using Python, allowing pagination using skip and limit.
|
||||
Search chats without loading their bodies, with pagination applied in SQL.
|
||||
"""
|
||||
search_text = sanitize_text_for_db(search_text).lower().strip()
|
||||
|
||||
|
|
@ -2219,7 +2268,9 @@ class ChatTable:
|
|||
search_terms = [term for term in chat_search_terms(phrase_query) if term != phrase_query]
|
||||
|
||||
async with get_async_db_context(db) as session:
|
||||
stmt = select(Chat).filter(Chat.user_id == user_id)
|
||||
stmt = select(
|
||||
Chat.id, Chat.title, Chat.updated_at, Chat.created_at, Chat.last_read_at, Chat.archived
|
||||
).filter(Chat.user_id == user_id)
|
||||
stmt = stmt.where(Chat.meta['internal'].as_boolean().is_not(True))
|
||||
|
||||
if is_archived is not None:
|
||||
|
|
@ -2347,12 +2398,21 @@ class ChatTable:
|
|||
# Perform pagination at the SQL level
|
||||
stmt = stmt.offset(skip).limit(limit)
|
||||
result = await session.execute(stmt)
|
||||
all_chats = result.scalars().all()
|
||||
all_chats = result.mappings().all()
|
||||
|
||||
log.info('The number of chats: %s', len(all_chats))
|
||||
|
||||
# Validate and return chats
|
||||
return [ChatModel.model_validate(chat) for chat in all_chats]
|
||||
return [ChatTitleIdResponse.model_validate(chat) for chat in all_chats]
|
||||
|
||||
async def get_chat_search_snippets(
|
||||
self, ids: list[str], user_id: str, search_text: str, db: AsyncSession | None = None
|
||||
) -> dict[str, str | None]:
|
||||
if not ids or not chat_search_content_query(search_text):
|
||||
return {}
|
||||
async with get_async_db_context(db) as session:
|
||||
result = await session.execute(select(Chat.id, Chat.chat).where(Chat.id.in_(ids), Chat.user_id == user_id))
|
||||
return {id: chat_search_snippet(body, search_text) for id, body in result.all()}
|
||||
|
||||
async def get_chats_by_folder_id_and_user_id(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -30,8 +30,6 @@ from open_webui.models.chats import (
|
|||
ChatTitleIdResponse,
|
||||
ChatUsageStatsListResponse,
|
||||
MessageStats,
|
||||
chat_search_content_query,
|
||||
chat_search_terms,
|
||||
)
|
||||
from open_webui.models.config import Config
|
||||
from open_webui.models.folders import Folders
|
||||
|
|
@ -227,47 +225,6 @@ class CompactChatForm(BaseModel):
|
|||
model: str | None = None
|
||||
|
||||
|
||||
def chat_search_content_text(text: str) -> str:
|
||||
return chat_search_content_query(text)
|
||||
|
||||
|
||||
def chat_search_snippet(chat: dict, search_text: str, max_length: int = 200) -> str | None:
|
||||
if not search_text:
|
||||
return None
|
||||
|
||||
history = chat.get('history', {})
|
||||
messages = history.get('messages') if isinstance(history, dict) else None
|
||||
if not messages:
|
||||
messages = chat.get('messages', []) or []
|
||||
if isinstance(messages, dict):
|
||||
messages = messages.values()
|
||||
|
||||
needles = list(dict.fromkeys([search_text, *chat_search_terms(search_text)]))
|
||||
for needle in needles:
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
|
||||
content = message.get('content')
|
||||
if not isinstance(content, str):
|
||||
continue
|
||||
|
||||
index = content.lower().find(needle)
|
||||
if index == -1:
|
||||
continue
|
||||
|
||||
start = max(index - max_length // 2, 0)
|
||||
end = min(start + max_length, len(content))
|
||||
if index + len(needle) > end:
|
||||
end = min(index + len(needle), len(content))
|
||||
start = max(end - max_length, 0)
|
||||
|
||||
snippet = ' '.join(content[start:end].split())
|
||||
return f'{"..." if start else ""}{snippet}{"..." if end < len(content) else ""}'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def get_chat_config_values() -> dict:
|
||||
values = await Config.get_many(*CHAT_CONFIG_KEYS.values())
|
||||
config = {field: values[storage_key] for field, storage_key in CHAT_CONFIG_KEYS.items() if storage_key in values}
|
||||
|
|
@ -954,21 +911,7 @@ async def search_user_chats(
|
|||
limit = 60
|
||||
skip = (page - 1) * limit
|
||||
|
||||
search_text = chat_search_content_text(text)
|
||||
chat_list = []
|
||||
for chat in await Chats.get_chats_by_user_id_and_search_text(user.id, text, skip=skip, limit=limit, db=db):
|
||||
# Explicit fields: model_dump() would deep-copy the entire chat blob per row
|
||||
chat_list.append(
|
||||
ChatTitleIdResponse(
|
||||
id=chat.id,
|
||||
title=chat.title,
|
||||
updated_at=chat.updated_at,
|
||||
created_at=chat.created_at,
|
||||
last_read_at=chat.last_read_at,
|
||||
snippet=chat_search_snippet(chat.chat, search_text),
|
||||
archived=chat.archived,
|
||||
)
|
||||
)
|
||||
chat_list = await Chats.get_chats_by_user_id_and_search_text(user.id, text, skip=skip, limit=limit, db=db)
|
||||
|
||||
# Delete tag if no chat is found
|
||||
words = text.strip().split(' ')
|
||||
|
|
@ -982,6 +925,17 @@ async def search_user_chats(
|
|||
return await add_active_state_to_chat_list(request, chat_list)
|
||||
|
||||
|
||||
@router.get('/{id}/search/snippet', response_model=str | None)
|
||||
async def get_chat_search_snippet(
|
||||
id: str,
|
||||
text: str,
|
||||
user=Depends(get_verified_user),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
snippets = await Chats.get_chat_search_snippets([id], user.id, text, db=db)
|
||||
return snippets.get(id)
|
||||
|
||||
|
||||
############################
|
||||
# GetChatsByFolderId
|
||||
############################
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from fastapi import APIRouter, Depends, Query
|
|||
from open_webui.internal.db import get_async_session
|
||||
from open_webui.models.access_grants import AccessGrant, AccessGrants
|
||||
from open_webui.models.channels import ChannelMember, Channels
|
||||
from open_webui.models.chats import Chat, chat_search_content_query
|
||||
from open_webui.models.chats import Chat, chat_message_list, chat_search_content_query
|
||||
from open_webui.models.config import Config
|
||||
from open_webui.models.folders import Folder, Folders
|
||||
from open_webui.models.groups import Groups
|
||||
|
|
@ -73,9 +73,7 @@ def excerpt(content: str, query: str, length: int = 200) -> str:
|
|||
|
||||
|
||||
def chat_passage(data: dict, query: str) -> tuple[str, str | None]:
|
||||
history = (data.get('history') or {}).get('messages') or {}
|
||||
messages = list(history.values()) if history else data.get('messages') or []
|
||||
messages = [m for m in messages if isinstance(m, dict) and isinstance(m.get('content'), str)]
|
||||
messages = [m for m in chat_message_list(data) if isinstance(m, dict) and isinstance(m.get('content'), str)]
|
||||
terms = terms_for(query)
|
||||
match = next((m for m in messages if terms and all(t in m['content'].lower() for t in terms)), None)
|
||||
if match is None:
|
||||
|
|
@ -97,13 +95,19 @@ def message_match_sql(table: str, dialect: str, key: str):
|
|||
return text(
|
||||
'('
|
||||
+ ' OR '.join(
|
||||
f"EXISTS (SELECT 1 FROM {source} AS m WHERE json_type(m.value, '$.content') = 'text' "
|
||||
f"AND lower(m.value ->> 'content') LIKE :{key} ESCAPE '\\')"
|
||||
f"EXISTS (SELECT 1 FROM {source} AS m WHERE lower(CASE WHEN m.type = 'object' "
|
||||
f"AND json_type(m.value, '$.content') = 'text' THEN m.value ->> 'content' END) "
|
||||
f"LIKE :{key} ESCAPE '\\')"
|
||||
for source in sources
|
||||
)
|
||||
+ ')'
|
||||
)
|
||||
sources = [f"json_each({table}.chat->'history'->'messages')", f"json_array_elements({table}.chat->'messages')"]
|
||||
sources = [
|
||||
f"json_each(CASE WHEN json_typeof({table}.chat->'history'->'messages') = 'object' "
|
||||
f"THEN {table}.chat->'history'->'messages' ELSE '{{}}'::json END)",
|
||||
f"json_array_elements(CASE WHEN json_typeof({table}.chat->'messages') = 'array' "
|
||||
f"THEN {table}.chat->'messages' ELSE '[]'::json END)",
|
||||
]
|
||||
return text(
|
||||
'('
|
||||
+ ' OR '.join(
|
||||
|
|
|
|||
|
|
@ -1676,7 +1676,7 @@ async def search_chats(
|
|||
limit=count * 3, # Fetch more for filtering
|
||||
)
|
||||
|
||||
results = []
|
||||
matched = []
|
||||
for chat in chats:
|
||||
# Skip the current chat to avoid showing it in search results
|
||||
if __chat_id__ and chat.id == __chat_id__:
|
||||
|
|
@ -1688,31 +1688,16 @@ async def search_chats(
|
|||
if end_timestamp and chat.updated_at > end_timestamp:
|
||||
continue
|
||||
|
||||
# Find a matching message snippet
|
||||
snippet = ''
|
||||
messages = (getattr(chat, 'chat', None) or {}).get('history', {}).get('messages', {})
|
||||
if not messages:
|
||||
messages = (getattr(chat, 'chat', None) or {}).get('messages', {}) or {}
|
||||
if isinstance(messages, list):
|
||||
messages = {str(idx): message for idx, message in enumerate(messages)}
|
||||
|
||||
lower_query = chat_search_content_query(query)
|
||||
needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else []
|
||||
|
||||
for needle in needles:
|
||||
for msg_id, msg in messages.items():
|
||||
content = msg.get('content', '') if isinstance(msg, dict) else ''
|
||||
if isinstance(content, str) and needle in content.lower():
|
||||
idx = content.lower().find(needle)
|
||||
start = max(0, idx - 50)
|
||||
end = min(len(content), idx + len(needle) + 100)
|
||||
snippet = (
|
||||
('...' if start > 0 else '') + content[start:end] + ('...' if end < len(content) else '')
|
||||
)
|
||||
break
|
||||
if snippet:
|
||||
break
|
||||
matched.append(chat)
|
||||
if len(matched) >= count:
|
||||
break
|
||||
|
||||
snippets = await Chats.get_chat_search_snippets([chat.id for chat in matched], user_id, query)
|
||||
lower_query = chat_search_content_query(query)
|
||||
needles = list(dict.fromkeys([lower_query, *chat_search_terms(lower_query)])) if lower_query else []
|
||||
results = []
|
||||
for chat in matched:
|
||||
snippet = snippets.get(chat.id) or ''
|
||||
title = chat.title or ''
|
||||
if not snippet and any(needle in title.lower() for needle in needles):
|
||||
snippet = f'Title match: {title}'
|
||||
|
|
@ -1726,9 +1711,6 @@ async def search_chats(
|
|||
}
|
||||
)
|
||||
|
||||
if len(results) >= count:
|
||||
break
|
||||
|
||||
return JSONCodec.dumps(results, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
log.exception(f'search_chats error: {e}')
|
||||
|
|
|
|||
|
|
@ -481,6 +481,18 @@ export const getAllChats = async (token: string) => {
|
|||
return chats;
|
||||
};
|
||||
|
||||
export const getChatSearchSnippet = async (token: string, id: string, text: string) => {
|
||||
const searchParams = new URLSearchParams({ text });
|
||||
const res = await fetch(`${WEBUI_API_BASE_URL}/chats/${id}/search/snippet?${searchParams}`, {
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
...(token && { authorization: `Bearer ${token}` })
|
||||
}
|
||||
});
|
||||
if (!res.ok) throw getErrorDetail(await res.json());
|
||||
return (await res.json()) as string | null;
|
||||
};
|
||||
|
||||
export const getChatListBySearchText = async (token: string, text: string, page: number = 1) => {
|
||||
let error = null;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
import { onDestroy, onMount, tick, getContext } from 'svelte';
|
||||
|
||||
import { decodeString } from '$lib/utils';
|
||||
import { getChatList, getChatListBySearchText } from '$lib/apis/chats';
|
||||
import { getChatList, getChatListBySearchText, getChatSearchSnippet } from '$lib/apis/chats';
|
||||
|
||||
import Tooltip from '$lib/components/common/Tooltip.svelte';
|
||||
import Spinner from '$lib/components/common/Spinner.svelte';
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
let loaded = false;
|
||||
|
||||
let items = [];
|
||||
let items: { id: string; name: string; updated_at: number; snippet?: string | null }[] = [];
|
||||
let selectedIdx = 0;
|
||||
let query = '';
|
||||
|
||||
|
|
@ -29,6 +29,18 @@
|
|||
let searchDebounceTimer: ReturnType<typeof setTimeout>;
|
||||
let requestId = 0;
|
||||
|
||||
const loadSnippet = async (item: { id: string; snippet?: string | null }) => {
|
||||
if (!searchedQuery.trim() || item.snippet !== undefined) return;
|
||||
item.snippet = null;
|
||||
try {
|
||||
item.snippet = await getChatSearchSnippet(localStorage.token, item.id, searchedQuery);
|
||||
items = items;
|
||||
} catch (error) {
|
||||
item.snippet = undefined;
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
// Only re-run the search when the query actually changes. Flipping `initialized`
|
||||
// after the initial load would otherwise trigger a second, identical request.
|
||||
$: if (initialized && query !== searchedQuery) {
|
||||
|
|
@ -82,6 +94,7 @@
|
|||
.map((item) => {
|
||||
return {
|
||||
...item,
|
||||
snippet: undefined,
|
||||
type: 'chat',
|
||||
name: item.title
|
||||
};
|
||||
|
|
@ -145,6 +158,7 @@
|
|||
.locale($i18n.language)
|
||||
.fromNow()}
|
||||
placement="top-start"
|
||||
tippyOptions={{ onShown: () => loadSnippet(item) }}
|
||||
>
|
||||
<div class="line-clamp-1 flex-1">
|
||||
{decodeString(item?.name)}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue