diff --git a/backend/open_webui/models/notes.py b/backend/open_webui/models/notes.py index 40ca4ef95e..b73457646e 100644 --- a/backend/open_webui/models/notes.py +++ b/backend/open_webui/models/notes.py @@ -1,17 +1,389 @@ +import json +import re import time import uuid -from functools import lru_cache -from typing import Optional +from copy import deepcopy +from html import escape +from typing import Literal, Optional +import markdown +from bs4 import BeautifulSoup from open_webui.internal.db import Base, get_async_db_context from open_webui.models.access_grants import AccessGrantModel, AccessGrants from open_webui.models.groups import Groups -from open_webui.models.users import User, UserModel, UserResponse, Users +from open_webui.models.users import User, UserModel, UserResponse from open_webui.utils.json_codec import JSONCodec from pydantic import BaseModel, ConfigDict, Field, field_validator -from sqlalchemy import JSON, BigInteger, Boolean, Column, ForeignKey, Text, cast, delete, func, or_, select, update +from sqlalchemy import JSON, BigInteger, Column, ForeignKey, Text, cast, delete, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession + +class NotePageConflict(ValueError): + pass + + +class NotePageOperation(BaseModel): + action: Literal['create', 'rename', 'duplicate', 'reorder', 'move', 'delete'] + page_id: str | None = None + new_id: str | None = None + parent_id: str | None = None + before_id: str | None = None + title: str | None = Field(default=None, max_length=200) + order: list[str] | None = None + revision: int = Field(ge=0) + + +def order_pages(pages: list[dict]) -> list[dict]: + """Keep sibling order while storing each subtree together, parent first.""" + ids = {page['id'] for page in pages} + if len(ids) != len(pages): + raise ValueError('Page IDs must be unique') + children: dict[str | None, list[dict]] = {} + for page in pages: + if page['id'] == 'main': + page['parent_id'] = None + else: + page.setdefault('parent_id', None) + parent = page['parent_id'] + if parent is not None and parent not in ids: + raise ValueError('Parent page no longer exists') + children.setdefault(parent, []).append(page) + ordered = [] + roots = children.get(None, []) + roots.sort(key=lambda page: page['id'] != 'main') + stack = list(reversed(roots)) + while stack: + page = stack.pop() + ordered.append(page) + stack.extend(reversed(children.get(page['id'], []))) + if len(ordered) != len(pages): + raise ValueError('A page cannot be nested inside itself or its subpages') + return ordered + + +def subtree_ids(pages: list[dict], page_id: str) -> set[str]: + # Tabs are in preorder, so all ancestors precede their descendants. + result = {page_id} + for page in pages: + if page.get('parent_id') in result: + result.add(page['id']) + return result + + +def page_paths(pages: list[dict]) -> dict[str, str]: + paths = {} + for page in pages: + parent_path = paths.get(page.get('parent_id')) + paths[page['id']] = f'{parent_path} / {page["title"]}' if parent_path else page['title'] + return paths + + +def get_content_files(content: dict | None) -> list[dict]: + """Attachment blocks are authoritative; HTML/Markdown support external note writers.""" + if not isinstance(content, dict): + return [] + document = content.get('json') + files = {} + if isinstance(document, dict): + stack = [document] + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + if node.get('type') == 'fileAttachment': + attrs = node.get('attrs') or {} + file_id = attrs.get('id') + if isinstance(file_id, str) and re.fullmatch(r'[\w-]+', file_id): + files[file_id] = { + 'type': 'file', + 'id': file_id, + 'name': attrs.get('name') or file_id, + 'size': attrs.get('size', 0), + } + stack.extend(reversed(node.get('content') or [])) + else: + html = content.get('html') or markdown.markdown(content.get('md') or '') + for link in BeautifulSoup(html, 'html.parser').find_all('a', href=True): + match = re.fullmatch(r'/api/v1/files/([\w-]+)/content', link['href']) + if match: + file_id = match[1] + files[file_id] = {'type': 'file', 'id': file_id, 'name': link.get_text() or file_id, 'size': 0} + return list(files.values()) + + +def migrate_page_files(page: dict, files: list[dict]) -> None: + """Consume legacy attachment lists once, preserving existing content and inline images.""" + content = page['content'] + existing = {file['id'] for file in get_content_files(content)} + changed = False + for file in files: + if not isinstance(file, dict) or not file.get('id'): + continue + if file.get('type') == 'image': + reference = f'data://{file["id"]}' + url = file.get('url') or '' + if not url: + continue + # These assets also back images in older versions of the page. + for snapshot in [content, *(page.get('versions') or [])]: + if not isinstance(snapshot, dict): + continue + encoded = json.dumps(snapshot) + if reference in encoded: + snapshot.update(json.loads(encoded.replace(reference, json.dumps(url)[1:-1]))) + changed = True + elif file['id'] not in existing: + attrs = {'id': file['id'], 'name': file.get('name') or file['id'], 'size': file.get('size') or 0} + href = f'/api/v1/files/{file["id"]}/content' + block = f'
' + content['html'] = (content.get('html') or markdown.markdown(content.get('md') or '')) + block + content['md'] = ( + (content.get('md') or '') + '\n\n' + f'[{attrs["name"].replace("[", "").replace("]", "")}]({href})' + ) + if isinstance(content.get('json'), dict): + content['json'].setdefault('content', []).append({'type': 'fileAttachment', 'attrs': attrs}) + existing.add(file['id']) + changed = True + if changed: + # Old collaboration documents must not overwrite the migrated content. + page['generation'] = page.get('generation', 0) + 1 + + +def get_note_files(data: dict | None) -> list[dict]: + return [file for page in normalize_pages(data)['pages'] for file in get_content_files(page['content'])] + + +def normalize_pages(data: dict | None, note_title: str | None = None) -> dict: + data = deepcopy(data or {}) + if not data.get('pages'): + data['pages'] = [ + { + 'id': 'main', + 'title': note_title or 'Untitled', + 'content': data.get('content') or {'json': None, 'html': '', 'md': ''}, + 'versions': data.get('versions') or [], + 'revision': 0, + 'generation': 0, + } + ] + if not any(page['id'] == 'main' for page in data['pages']): + data['pages'].insert( + 0, + { + 'id': 'main', + 'title': note_title or 'Untitled', + 'content': {'json': None, 'html': '', 'md': ''}, + 'versions': [], + 'revision': 0, + 'generation': 0, + }, + ) + if note_title is not None: + for page in data['pages']: + if page['id'] == 'main' and page.get('inherit_note_title', True): + page['title'] = note_title or 'Untitled' + data['pages'] = order_pages(data['pages']) + legacy_files = data.pop('files', None) or [] + for page in data['pages']: + files = page.pop('files', None) + if files is None: + # Old whole-note files belong to main; subpages only retain referenced images. + references = json.dumps([page.get('content'), page.get('versions')]) + files = ( + legacy_files + if page['id'] == 'main' + else [ + file + for file in legacy_files + if isinstance(file, dict) + and file.get('type') == 'image' + and f'data://{file.get("id")}' in references + ] + ) + if files: + migrate_page_files(page, files) + data.setdefault('pages_revision', 0) + data.pop('versions', None) + # A derived projection keeps existing SQL search and whole-note retrieval cheap. + pages = data['pages'] + paths = page_paths(pages) + data['content'] = ( + deepcopy(pages[0]['content']) + if len(pages) == 1 + else { + 'json': None, + 'md': '\n\n'.join(f'## {paths[p["id"]]}\n\n{p["content"].get("md", "")}' for p in pages), + 'html': '', + } + ) + data['search_text'] = '\n\n'.join(f'{paths[p["id"]]}\n{p["content"].get("md", "")}' for p in pages) + return data + + +def get_page(data: dict, page_id: str | None = None) -> dict: + pages = normalize_pages(data)['pages'] + if page_id is None: + if len(pages) != 1: + raise NotePageConflict('Select a page before editing this note') + return pages[0] + page = next((page for page in pages if page['id'] == page_id), None) + if page is None: + raise NotePageConflict('Page no longer exists') + return page + + +def document_page(data: dict, document_id: str) -> dict: + parts = document_id.split(':') + if len(parts) == 2: + page = get_page(data) + if page.get('generation', 0) != 0: + raise NotePageConflict('Reload this page before editing') + return page + if len(parts) != 5 or parts[2] != 'page': + raise ValueError('Invalid note document ID') + page = get_page(data, parts[3]) + if str(page.get('generation', 0)) != parts[4]: + raise NotePageConflict('Page changed; reload before editing') + return page + + +def apply_page_operation(data: dict, operation: NotePageOperation) -> dict: + data = normalize_pages(data) + pages = data['pages'] + if operation.action in ('create', 'duplicate'): + if not operation.new_id: + raise ValueError('A new page ID is required') + uuid.UUID(operation.new_id) + if any(p['id'] == operation.new_id for p in pages): + return data + if operation.revision != data['pages_revision']: + raise NotePageConflict('Pages changed. Refresh and try again') + if operation.action == 'create': + title = (operation.title or '').strip() + if not title: + raise ValueError('Page title is required') + pages.append( + { + 'id': operation.new_id, + 'parent_id': operation.parent_id, + 'title': title, + 'content': {'json': None, 'html': '', 'md': ''}, + 'versions': [], + 'revision': 0, + 'generation': 0, + } + ) + elif operation.action == 'reorder': + order = operation.order or [] + if len(order) != len(pages) or set(order) != {p['id'] for p in pages}: + raise ValueError('Order must contain each page exactly once') + if order[0] != 'main': + raise ValueError('The default page must stay first') + lookup = {p['id']: p for p in pages} + data['pages'] = [lookup[id] for id in order] + else: + page = next((p for p in pages if p['id'] == operation.page_id), None) + if page is None: + raise NotePageConflict('Page no longer exists') + if page['id'] == 'main' and operation.action in ('move', 'delete'): + raise ValueError('The default page cannot be moved or deleted') + if operation.action == 'rename': + title = (operation.title or '').strip() + if not title: + raise ValueError('Page title is required') + page['title'] = title + if page['id'] == 'main': + page['inherit_note_title'] = False + elif operation.action == 'move': + parent_id = operation.parent_id + if operation.before_id == 'main': + raise ValueError('The default page must stay first') + branch_ids = subtree_ids(pages, page['id']) + if parent_id in branch_ids: + raise ValueError('A page cannot be nested inside itself or its subpages') + if parent_id is not None and not any(p['id'] == parent_id for p in pages): + raise ValueError('Parent page no longer exists') + branch = [p for p in pages if p['id'] in branch_ids] + remaining = [p for p in pages if p['id'] not in branch_ids] + position = len(remaining) + if operation.before_id is not None: + anchor = next((p for p in remaining if p['id'] == operation.before_id), None) + if anchor is None or anchor['parent_id'] != parent_id: + raise ValueError('Move target must be a sibling in the destination') + position = remaining.index(anchor) + page['parent_id'] = parent_id + data['pages'] = remaining[:position] + branch + remaining[position:] + elif operation.action == 'delete': + if len(pages) == 1: + raise ValueError('A note must have at least one page') + # Deleting one tab never silently deletes its children's content. + for child in pages: + if child['parent_id'] == page['id']: + child['parent_id'] = page['parent_id'] + pages.remove(page) + elif operation.action == 'duplicate': + branch_ids = subtree_ids(pages, page['id']) + branch = [p for p in pages if p['id'] in branch_ids] + ids = {p['id']: str(uuid.uuid5(uuid.UUID(operation.new_id), p['id'])) for p in branch} + ids[page['id']] = operation.new_id + if set(ids.values()) & {p['id'] for p in pages}: + raise ValueError('A duplicated page ID already exists') + copies = deepcopy(branch) + for copy in copies: + copy.update( + id=ids[copy['id']], + parent_id=ids.get(copy['parent_id'], copy['parent_id']), + versions=[], + revision=0, + generation=0, + ) + copies[0]['title'] = page['title'][:193] + ' (Copy)' + position = pages.index(branch[-1]) + 1 + pages[position:position] = copies + data['pages_revision'] += 1 + return normalize_pages(data) + + +def update_page_content( + data: dict, page_id: str | None, changes: dict, generation: int | None = None, revision: int | None = None +) -> dict: + data = normalize_pages(data) + target = get_page(data, page_id) + page = next(p for p in data['pages'] if p['id'] == target['id']) + if generation is not None and generation != page.get('generation', 0): + raise NotePageConflict('Page changed; reload before editing') + if revision is not None and revision != page.get('revision', 0): + raise NotePageConflict('Page content changed. Read it again before editing') + if 'content' in changes: + content = changes['content'] + if not isinstance(content, dict) or any( + key in content and not isinstance(content[key], str) for key in ('md', 'html') + ): + raise ValueError('Page content must contain text markdown and HTML') + page['content'] = {'json': None, 'html': '', 'md': '', **deepcopy(content)} + page['revision'] = page.get('revision', 0) + 1 + # Socket snapshots retain their generation. External replacements invalidate old documents. + if generation is None: + page['generation'] = page.get('generation', 0) + 1 + if 'versions' in changes: + page['versions'] = deepcopy(changes['versions']) + if 'files' in changes: + raise ValueError('Attach files through page content') + return normalize_pages(data) + + +def matching_page(data: dict, query: str = '') -> dict: + pages = normalize_pages(data)['pages'] + paths = page_paths(pages) + terms = query.lower().split() + for require_all in (True, False): + for page in pages: + text = f'{paths[page["id"]]} {page["content"].get("md", "")}'.lower() + if terms and (all(t in text for t in terms) if require_all else any(t in text for t in terms)): + return page + return pages[0] + + #################### # Note DB Schema #################### @@ -102,6 +474,8 @@ class NoteForm(BaseModel): @field_validator('data', mode='before') @classmethod def sanitize_data(cls, data): + if isinstance(data, dict) and 'pages' in data: + raise ValueError('Create pages using page operations') return sanitize_note_data(data) @@ -109,6 +483,10 @@ class NoteUpdateForm(BaseModel): title: Optional[str] = None data: Optional[dict] = None meta: Optional[dict] = None + page_id: Optional[str] = None + page_revision: Optional[int] = None + page_generation: Optional[int] = None + page_operation: Optional[NotePageOperation] = None @field_validator('data', mode='before') @classmethod @@ -147,7 +525,7 @@ class NoteTable: ) -> NoteModel: # We exclude access_grants to inject them note_model = NoteModel.model_validate(note) - note_model.data = note_model.data or {} + note_model.data = normalize_pages(note_model.data, note_model.title) note_model.access_grants = ( access_grants if access_grants is not None else await self._get_access_grants(note_model.id, db=db) ) @@ -178,6 +556,7 @@ class NoteTable: } ) + note.data = normalize_pages(note.data, note.title) new_note = Note(**note.model_dump(exclude={'access_grants', 'is_pinned'})) db.add(new_note) @@ -221,7 +600,13 @@ class NoteTable: or_( func.replace(func.replace(Note.title, '-', ''), ' ', '').ilike(f'%{word}%'), func.replace( - func.replace(Note.data['content']['md'].as_string(), '-', ''), + func.replace( + func.coalesce( + Note.data['search_text'].as_string(), Note.data['content']['md'].as_string() + ), + '-', + '', + ), ' ', '', ).ilike(f'%{word}%'), @@ -331,7 +716,7 @@ class NoteTable: async def get_note_by_id(self, id: str, db: Optional[AsyncSession] = None) -> Optional[NoteModel]: async with get_async_db_context(db) as db: - result = await db.execute(select(Note).filter(Note.id == id)) + result = await db.execute(select(Note).filter(Note.id == id).execution_options(populate_existing=True)) note = result.scalars().first() return await self._to_note_model(note, db=db) if note else None @@ -349,39 +734,58 @@ class NoteTable: note_id for note_id, data in result.all() if isinstance(data, dict) - and isinstance(data.get('files'), list) and any( isinstance(item, dict) and item.get('type') == 'file' and item.get('id') == file_id - for item in data['files'] + for item in get_note_files(data) ) ] async def update_note_by_id( self, id: str, form_data: NoteUpdateForm, db: Optional[AsyncSession] = None ) -> Optional[NoteModel]: + changes = form_data.model_dump(exclude_unset=True) + incoming = changes.get('data') or {} + if 'pages' in incoming or 'pages_revision' in incoming: + raise ValueError('Use page operations instead of replacing all pages') async with get_async_db_context(db) as db: - result = await db.execute(select(Note).filter(Note.id == id)) - note = result.scalars().first() - if not note: - return None - - form_data = form_data.model_dump(exclude_unset=True) - note.data = sanitize_note_data(note.data) or {} - - if 'title' in form_data: - note.title = form_data['title'] - if 'data' in form_data: - note.data = {**(note.data or {}), **(form_data['data'] or {})} - if 'meta' in form_data: - note.meta = {**(note.meta or {}), **(form_data['meta'] or {})} - - if not db.is_modified(note): - return await self._to_note_model(note, db=db) - - note.updated_at = int(time.time_ns()) - - await db.commit() - return await self._to_note_model(note, db=db) if note else None + # CAS protects the entire JSON from writers on other pages and other workers. + for _ in range(8): + result = await db.execute(select(Note).where(Note.id == id).execution_options(populate_existing=True)) + note = result.scalars().first() + if not note: + return None + previous_updated_at = note.updated_at + data = normalize_pages(sanitize_note_data(note.data), note.title) + if form_data.page_operation: + data = apply_page_operation(data, form_data.page_operation) + if any(key in incoming for key in ('content', 'versions', 'files')): + data = update_page_content( + data, + form_data.page_id or ('main' if not {'content', 'versions'} & incoming.keys() else None), + incoming, + form_data.page_generation, + form_data.page_revision, + ) + data = normalize_pages( + {**data, **{k: v for k, v in incoming.items() if k not in ('content', 'versions', 'files')}}, + changes.get('title', note.title), + ) + values = {'data': data, 'updated_at': max(time.time_ns(), (previous_updated_at or 0) + 1)} + if 'title' in changes: + values['title'] = changes['title'] + if 'meta' in changes: + values['meta'] = {**(note.meta or {}), **(changes['meta'] or {})} + result = await db.execute( + update(Note) + .where(Note.id == id, Note.updated_at == previous_updated_at) + .values(**values) + .execution_options(synchronize_session=False) + ) + if result.rowcount: + await db.commit() + return await self.get_note_by_id(id, db=db) + await db.rollback() + raise NotePageConflict('Note changed while saving. Please try again') async def toggle_note_pinned_by_id( self, id: str, user_id: str, db: Optional[AsyncSession] = None diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 9577b22b53..e98a1c2a30 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -1461,8 +1461,20 @@ async def get_sources_from_items( ): # User has access to the note query_result = { - 'documents': [[note.data.get('content', {}).get('md', '')]], - 'metadatas': [[{'file_id': note.id, 'name': note.title}]], + 'documents': [[page['content'].get('md', '') for page in note.data['pages']]], + 'metadatas': [ + [ + { + 'file_id': note.id, + 'note_id': note.id, + 'page_id': page['id'], + 'parent_id': page.get('parent_id'), + 'name': f'{note.title} / {page["title"]}', + 'source': f'/notes/{note.id}?page={page["id"]}', + } + for page in note.data['pages'] + ] + ], } elif item.get('type') == 'chat': diff --git a/backend/open_webui/routers/notes.py b/backend/open_webui/routers/notes.py index c21b71cdf0..7be91b2e61 100644 --- a/backend/open_webui/routers/notes.py +++ b/backend/open_webui/routers/notes.py @@ -19,12 +19,18 @@ from open_webui.models.notes import ( NoteForm, NoteListResponse, NoteModel, + NotePageConflict, + NotePageOperation, Notes, NoteUpdateForm, NoteUserResponse, + get_content_files, + get_note_files, + matching_page, + normalize_pages, ) from open_webui.models.users import UserModel, UserResponse, Users -from open_webui.socket.main import leave_room_for_users, sio +from open_webui.socket.main import close_changed_note_documents, leave_room_for_users, sio from open_webui.utils.access_control import ( filter_allowed_access_grants, has_permission, @@ -56,11 +62,17 @@ async def check_notes_access(user: UserModel, db: AsyncSession): ) -def _truncate_note_data(data: Optional[dict], max_length: int = 1000) -> Optional[dict]: +def _truncate_note_data(data: Optional[dict], max_length: int = 1000, query: str = '') -> Optional[dict]: if not data: return data - md = (data.get('content') or {}).get('md') or '' - return {'content': {'md': md[:max_length]}} + page = matching_page(data, query) + md = page['content'].get('md') or '' + return { + 'content': {'md': md[:max_length]}, + 'page_count': len(data.get('pages') or [None]), + 'page_id': page['id'], + 'page_title': page['title'], + } async def leave_note_rooms_for_revoked_users( @@ -76,7 +88,10 @@ async def leave_note_rooms_for_revoked_users( users = await Users.get_users_by_user_ids(list(revoked_user_ids), db=db) # Admins retain access to notes regardless of grants. user_ids = [user.id for user in users if user.role != 'admin'] - for room in [f'note:{note.id}', f'doc_note:{note.id}']: + page_rooms = [ + f'doc_note:{note.id}:page:{p["id"]}:{p.get("generation", 0)}' for p in normalize_pages(note.data)['pages'] + ] + for room in [f'note:{note.id}', f'doc_note:{note.id}', *page_rooms]: await leave_room_for_users(room, user_ids) @@ -210,7 +225,7 @@ async def search_notes( pinned_note_ids = await Notes.get_pinned_note_ids(user.id, db=db) for note in result.items: note.is_pinned = note.id in pinned_note_ids - note.data = _truncate_note_data(note.data) + note.data = _truncate_note_data(note.data, query=query or '') return result @@ -237,6 +252,9 @@ async def create_new_note( db=db, ) + if not await has_access_to_note_files(None, get_note_files(form_data.data), user, db=db): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) + try: note = await Notes.insert_new_note(user.id, form_data, db=db) await publish_event( @@ -536,13 +554,16 @@ async def update_note_by_id( if ( form_data.data - and 'files' in form_data.data - and not await has_access_to_note_files(note, form_data.data['files'], user, db=db) + and not await has_access_to_note_files(note, get_content_files(form_data.data.get('content')), user, db=db) ): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) try: + if form_data.page_generation is not None: + raise ValueError('Page generation is managed by collaboration') + previous = note note = await Notes.update_note_by_id(id, form_data, db=db) + await close_changed_note_documents(previous, note) pinned_note_ids = await Notes.get_pinned_note_ids(user.id, db=db) note.is_pinned = note.id in pinned_note_ids @@ -552,6 +573,8 @@ async def update_note_by_id( event_data['data'] = { key: note.data.get(key) for key in form_data.data.keys() if note.data is not None and key in note.data } + if form_data.page_operation or (form_data.data and {'content', 'files'} & form_data.data.keys()): + event_data['data'] = note.data await sio.emit( 'events:note', @@ -567,11 +590,26 @@ async def update_note_by_id( data={'title': note.title}, ) return note + except NotePageConflict as e: + raise HTTPException(status_code=409, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) except Exception as e: log.exception(e) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()) +@router.post('/{id}/pages/update', response_model=NoteModel) +async def update_note_pages( + request: Request, + id: str, + form_data: NotePageOperation, + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + return await update_note_by_id(request, id, NoteUpdateForm(page_operation=form_data), user, db) + + ############################ # UpdateNoteAccessById ############################ @@ -704,8 +742,13 @@ async def delete_note_by_id( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) try: + previous = note + page_rooms = [ + f'doc_note:{id}:page:{p["id"]}:{p.get("generation", 0)}' for p in normalize_pages(note.data)['pages'] + ] note = await Notes.delete_note_by_id(id, db=db) - for room in [f'note:{id}', f'doc_note:{id}']: + await close_changed_note_documents(previous, None) + for room in [f'note:{id}', f'doc_note:{id}', *page_rooms]: await sio.close_room(room) await publish_event( diff --git a/backend/open_webui/routers/search.py b/backend/open_webui/routers/search.py index 1091d7581f..c5850bdd05 100644 --- a/backend/open_webui/routers/search.py +++ b/backend/open_webui/routers/search.py @@ -12,7 +12,7 @@ from open_webui.models.config import Config from open_webui.models.folders import Folder, Folders from open_webui.models.groups import Groups from open_webui.models.messages import Message -from open_webui.models.notes import Note, PinnedNote +from open_webui.models.notes import Note, PinnedNote, matching_page from open_webui.models.shared_chats import SharedChat from open_webui.models.users import User from open_webui.utils.access_control import has_permission @@ -338,7 +338,9 @@ async def search_workspace( # noqa: C901 — keep permission scoping and source db, select(Note, pinned, note_shared), Note, {'user_id': user.id, 'group_ids': groups}, 'note', 'read' ) stmt = stmt.where(sharing_filter(Note.user_id)) - stmt = apply_match(stmt, Note, Note.data['content']['md'].as_string()) + stmt = apply_match( + stmt, Note, func.coalesce(Note.data['search_text'].as_string(), Note.data['content']['md'].as_string()) + ) rank = title_rank(Note.title, query) if query else case((pinned, 0), else_=3) sources['note'] = (Note, stmt, rank, cast(Note.updated_at.op('/')(literal(1_000_000_000)), BigInteger)) @@ -421,14 +423,15 @@ async def search_workspace( # noqa: C901 — keep permission scoping and source match.rank, ) elif match.source == 'note': - content = ((item.data or {}).get('content') or {}).get('md') or '' + page = matching_page(item.data, query) + content = page['content'].get('md', '') append_result( 'note:' + item.id, 'note', item.title, 'Shared note' if item.user_id != user.id or row[2] else 'Personal note', content if isinstance(content, str) else '', - '/notes/' + item.id + ('?' + urlencode({'search': query}) if query else ''), + '/notes/' + item.id + '?' + urlencode({'page': page['id'], 'search': query}), match.updated_at, item.user_id != user.id or row[2], row[1], diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index 2d235175cf..99931b4e06 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -40,7 +40,7 @@ from open_webui.models.channels import Channels from open_webui.models.chats import Chats from open_webui.models.config import Config from open_webui.models.folders import Folders -from open_webui.models.notes import Notes, NoteUpdateForm +from open_webui.models.notes import Notes, NoteUpdateForm, document_page, get_content_files, get_page, normalize_pages from open_webui.models.users import UserNameResponse, Users from open_webui.socket.encryption import Relay from open_webui.socket.redis_room_channels import AsyncRedisRoomChannelManager @@ -695,7 +695,7 @@ async def join_channel(sid, data): await sio.enter_room(sid, f'channel:{channel.id}') -@sio.on('join-note') +@sio.on('note:join') async def join_note(sid, data): auth = data['auth'] if 'auth' in data else None if not auth or 'token' not in auth: @@ -736,6 +736,13 @@ async def join_note(sid, data): await sio.enter_room(sid, f'note:{note.id}') +@sio.on('note:leave') +async def leave_note(sid, data): + note_id = data.get('note_id') + if isinstance(note_id, str): + await sio.leave_room(sid, f'note:{note_id}') + + @sio.on('events:channel') async def channel_events(sid, data): room = f'channel:{data["channel_id"]}' @@ -886,6 +893,23 @@ def normalize_document_id(document_id: str) -> str: return document_id +async def close_changed_note_documents(previous, current): + remaining = {p['id']: p.get('generation', 0) for p in normalize_pages(current.data)['pages']} if current else {} + for page in normalize_pages(previous.data)['pages']: + if remaining.get(page['id']) == page.get('generation', 0): + continue + ids = [f'note:{previous.id}:page:{page["id"]}:{page.get("generation", 0)}'] + if len(previous.data['pages']) == 1: + ids.append(f'note:{previous.id}') + for document_id in ids: + async with YDOC_MANAGER.lock(document_id): + await stop_item_tasks(REDIS, document_id) + await sio.close_room(f'doc_{document_id}') + for sid in list(await YDOC_MANAGER.get_users(document_id)): + await YDOC_MANAGER.remove_user(document_id, sid) + await YDOC_MANAGER.clear_document(document_id) + + def with_document_lock(handler): @wraps(handler) async def wrapped(sid, data): @@ -920,6 +944,7 @@ async def ydoc_document_join(sid, data): if not note: log.error(f'Note {note_id} not found') return + page = document_page(note.data, document_id) if ( user.get('role') != 'admin' @@ -964,7 +989,7 @@ async def ydoc_document_join(sid, data): { 'document_id': document_id, 'state': list(state_update), # Convert bytes to list for JSON - 'content': note.data.get('content') if document_id.startswith('note:') and note.data else None, + 'content': page['content'] if document_id.startswith('note:') else None, }, room=sid, ) @@ -1012,7 +1037,31 @@ async def document_save_handler(document_id, data, user): log.error(f'User {user.get("id")} does not have write access to note {note_id}') return - return await Notes.update_note_by_id(note_id, NoteUpdateForm(data=data)) + page = document_page(note.data, document_id) + if not isinstance(data.get('content'), dict): + return None + editor = await Users.get_user_by_id(user['id']) + if not editor or not await has_access_to_note_files(note, get_content_files(data['content']), editor): + raise ValueError('Cannot attach files you cannot access') + saved = await Notes.update_note_by_id( + note_id, + NoteUpdateForm( + data={'content': data['content']}, + page_id=page['id'], + page_generation=page.get('generation', 0), + ), + ) + if saved: + await sio.emit( + 'events:note', + { + 'id': note_id, + 'page': get_page(saved.data, page['id']), + 'updated_at': saved.updated_at, + }, + to=f'note:{note_id}', + ) + return saved @sio.on('ydoc:document:state') @@ -1084,6 +1133,7 @@ async def yjs_document_update(sid, data): if not note: log.error(f'Note {note_id} not found') return + page = document_page(note.data, document_id) if ( user.get('role') != 'admin' @@ -1099,9 +1149,9 @@ async def yjs_document_update(sid, data): return snapshot = data.get('data') - if isinstance(snapshot, dict) and 'files' in snapshot: + if isinstance(snapshot, dict) and 'content' in snapshot: editor = await Users.get_user_by_id(user['id']) - if not editor or not await has_access_to_note_files(note, snapshot['files'], editor): + if not editor or not await has_access_to_note_files(note, get_content_files(snapshot['content']), editor): await sio.emit('error', {'message': 'Cannot attach files you cannot access'}, room=sid) return @@ -1143,6 +1193,11 @@ async def yjs_document_update(sid, data): if data.get('initialize'): return update if stored else None + if data.get('flush'): + await stop_item_tasks(REDIS, document_id) + saved = await document_save_handler(document_id, data.get('data', {}), user) + return {'ok': bool(saved)} + async def debounced_save(): await asyncio.sleep(0.5) async with YDOC_MANAGER.lock(document_id): @@ -1167,6 +1222,7 @@ async def yjs_document_update(sid, data): except Exception as e: log.error(f'Error in yjs_document_update: {e}') + return {'error': str(e)} @sio.on('ydoc:document:leave') diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 72dd8422a7..c2d18366a4 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -26,8 +26,8 @@ from open_webui.models.config import Config from open_webui.models.groups import Groups from open_webui.models.memories import Memories from open_webui.models.messages import Message, Messages -from open_webui.models.notes import Notes -from open_webui.models.users import UserModel +from open_webui.models.notes import NotePageConflict, Notes, get_content_files, get_page, matching_page +from open_webui.models.users import UserModel, Users from open_webui.retrieval.utils import filter_source_metadata, get_content_from_url from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.routers.images import ( @@ -61,9 +61,9 @@ from open_webui.routers.memories import ( update_memories as _update_memories, ) from open_webui.routers.retrieval import search_web as _search_web -from open_webui.socket.main import sio -from open_webui.tasks import stop_item_tasks +from open_webui.socket.main import close_changed_note_documents, sio from open_webui.tools.knowledge_fs import kb_exec # noqa: F401 — re-exported +from open_webui.utils.access_control.files import has_access_to_note_files from open_webui.utils.chat_id import is_saved_chat_id from open_webui.utils.json_codec import JSONCodec from open_webui.utils.notifications import notify_target @@ -1202,10 +1202,11 @@ async def search_notes( if end_ts and note.updated_at > end_ts: continue - # Extract a snippet from the markdown content + # Extract a snippet from the matching page. + page = matching_page(note.data, query) content_snippet = '' - if note.data and note.data.get('content', {}).get('md'): - md_content = note.data['content']['md'] + if page['content'].get('md'): + md_content = page['content']['md'] content_lower = md_content.lower() # Find the first matching word to center the snippet around. @@ -1235,6 +1236,9 @@ async def search_notes( 'id': note.id, 'title': note.title, 'snippet': content_snippet, + 'page_id': page['id'], + 'page_title': page['title'], + 'url': f'/notes/{note.id}?page={page["id"]}', 'updated_at': note.updated_at, } ) @@ -1250,6 +1254,7 @@ async def search_notes( async def view_note( note_id: str, + page_id: Optional[str] = None, __request__: Request = None, __user__: dict = None, ) -> str: @@ -1257,6 +1262,7 @@ async def view_note( Get the full content of a note by its ID. :param note_id: The ID of the note to retrieve + :param page_id: Page to read; omit to inspect the complete note and its page directory :return: JSON with the note's id, title, and full markdown content """ if __request__ is None: @@ -1290,16 +1296,24 @@ async def view_note( ): return JSONCodec.dumps({'error': 'Access denied'}) - # Extract markdown content - content = '' - if note.data and note.data.get('content', {}).get('md'): - content = note.data['content']['md'] + page = get_page(note.data, page_id) if page_id or len(note.data['pages']) == 1 else None + content = page['content'].get('md', '') if page else note.data['content'].get('md', '') + # Keep the version read by this tool run out of the model-facing interface. + if not hasattr(__request__.state, 'note_page_revisions'): + __request__.state.note_page_revisions = {} + for read_page in [page] if page else note.data['pages']: + __request__.state.note_page_revisions[(user_id, note.id, read_page['id'])] = read_page.get('revision', 0) return JSONCodec.dumps( { 'id': note.id, 'title': note.title, 'content': content, + 'page_id': page['id'] if page else None, + 'page_title': page['title'] if page else None, + 'pages': [ + {'id': p['id'], 'title': p['title'], 'parent_id': p.get('parent_id')} for p in note.data['pages'] + ], 'updated_at': note.updated_at, 'created_at': note.created_at, }, @@ -1340,6 +1354,9 @@ async def write_note( access_grants=[], # Private by default - only owner can access ) + editor = await Users.get_user_by_id(user_id) + if not editor or not await has_access_to_note_files(None, get_content_files(form.data['content']), editor): + return JSONCodec.dumps({'error': 'Cannot attach files you cannot access'}) new_note = await Notes.insert_new_note(user_id, form) if not new_note: @@ -1364,12 +1381,14 @@ async def replace_note_content( content: Optional[str] = None, operations: Optional[list[dict]] = None, title: Optional[str] = None, + page_id: Optional[str] = None, __request__: Request = None, __user__: dict = None, ) -> str: """ Update an existing note by replacing the whole markdown content, exact text, or character ranges. + Read the note with view_note before editing. If it changes meanwhile, read it again before retrying. Prefer "replace_text" when only part of the note changes. Supply either find or start/end, not both. find is non-empty, literal, and case-sensitive. Text matches are non-overlapping. Multiple matches require a 0-indexed occurrence or replace_all=true; @@ -1389,6 +1408,7 @@ async def replace_note_content( - {"action": "replace_text", "start": 0, "end": 10, "content": "...", "expected": "..."} - {"action": "replace_text", "find": "old text", "content": "new text", "occurrence": 0, "replace_all": false} :param title: Optional new title for the note + :param page_id: Page to edit; required for notes with multiple pages. Use the active page_id supplied in the request unless the user selects another page :return: JSON with success status and updated note info """ if __request__ is None: @@ -1409,7 +1429,16 @@ async def replace_note_content( if __user__.get('role') != 'admin' and not await _has_write_access_to_note(note, user_id): return JSONCodec.dumps({'error': 'Write access denied', 'code': 'write_access_denied'}) - current_content = ((note.data or {}).get('content') or {}).get('md') or '' + page = get_page(note.data, page_id) + page_id = page['id'] + read_revisions = getattr(__request__.state, 'note_page_revisions', {}) + read_key = (user_id, note.id, page_id) + if read_key not in read_revisions: + return JSONCodec.dumps({'error': 'Read this page with view_note before editing', 'code': 'read_required'}) + read_revision = read_revisions[read_key] + if read_revision != page.get('revision', 0): + raise NotePageConflict('Page changed since you read it. Use view_note again before editing') + current_content = page['content'].get('md') or '' applied_operation_count = 0 range_operations = [] if operations is not None: @@ -1557,37 +1586,38 @@ async def replace_note_content( elif content is None: return JSONCodec.dumps({'error': 'content or operations is required', 'code': 'content_required'}) - try: - await stop_item_tasks(__request__.app.state.redis, f'note:{note_id}') - except Exception: - pass - update_data = { + 'page_id': page_id, + 'page_revision': read_revision, 'data': { - **(note.data or {}), 'content': { - **((note.data or {}).get('content') or {}), 'json': None, 'html': '', 'md': content, }, - } + }, } if title: update_data['title'] = title + editor = await Users.get_user_by_id(__user__['id']) + if not editor or not await has_access_to_note_files(note, get_content_files(update_data['data']['content']), editor): + return JSONCodec.dumps({'error': 'Cannot attach files you cannot access'}) form = NoteUpdateForm(**update_data) updated_note = await Notes.update_note_by_id(note_id, form) if not updated_note: return JSONCodec.dumps({'error': 'Failed to update note', 'code': 'update_failed'}) + read_revisions[read_key] = get_page(updated_note.data, page_id).get('revision', 0) + await close_changed_note_documents(note, updated_note) await _emit_note_updated(__request__, __user__, updated_note) return JSONCodec.dumps( { 'status': 'success', 'id': updated_note.id, + 'page_id': page_id, 'title': updated_note.title, 'updated_at': updated_note.updated_at, 'applied_operation_count': applied_operation_count, @@ -1595,6 +1625,8 @@ async def replace_note_content( }, ensure_ascii=False, ) + except NotePageConflict as e: + return JSONCodec.dumps({'error': str(e), 'code': 'page_conflict'}) except Exception as e: log.exception(f'replace_note_content error: {e}') return JSONCodec.dumps({'error': str(e), 'code': 'unexpected_error'}) @@ -3624,6 +3656,7 @@ async def view_skill( return JSONCodec.dumps(skill, ensure_ascii=False) from types import SimpleNamespace + from open_webui.models.skills import get_skill_snapshot from open_webui.routers.skills import authorized_skill from open_webui.utils.skill_files import file_summaries @@ -3671,6 +3704,7 @@ async def read_skill_file( try: from types import SimpleNamespace from urllib.parse import urlencode + from open_webui.models.skills import get_skill_snapshot from open_webui.routers.skills import authorized_skill from open_webui.utils.skill_files import file_summaries @@ -3777,6 +3811,7 @@ async def grep_skill_files( files = [{'path': 'SKILL.md', 'content': skill['content']}] else: from types import SimpleNamespace + from open_webui.models.skills import get_skill_snapshot from open_webui.routers.skills import authorized_skill @@ -3827,10 +3862,11 @@ async def create_skill( """ try: from types import SimpleNamespace - from open_webui.routers.skills import create_new_skill + from open_webui.models.skills import SkillForm - from open_webui.utils.skill_files import frontmatter + from open_webui.routers.skills import create_new_skill from open_webui.utils.access_control import has_permission + from open_webui.utils.skill_files import frontmatter if __request__ is None or not __user__: raise ValueError('Request and user context required') @@ -3877,9 +3913,10 @@ async def update_skill_files( """ try: from types import SimpleNamespace - from open_webui.routers.skills import authorized_skill - from open_webui.models.skills import Skills + from open_webui.events import EVENTS, publish_event + from open_webui.models.skills import Skills + from open_webui.routers.skills import authorized_skill if __request__ is None or not __user__: raise ValueError('Request and user context required') @@ -4147,7 +4184,7 @@ async def create_automation( return JSONCodec.dumps({'error': 'User context not available'}) try: - from open_webui.models.automations import AutomationData, AutomationForm, AutomationTarget, Automations + from open_webui.models.automations import AutomationData, AutomationForm, Automations, AutomationTarget from open_webui.models.users import Users from open_webui.routers.automations import check_automation_limits from open_webui.utils.automations import next_n_runs_ns, next_run_ns, validate_rrule @@ -4246,7 +4283,7 @@ async def update_automation( return JSONCodec.dumps({'error': 'User context not available'}) try: - from open_webui.models.automations import AutomationData, AutomationForm, AutomationTarget, Automations + from open_webui.models.automations import AutomationData, AutomationForm, Automations, AutomationTarget from open_webui.models.users import Users from open_webui.routers.automations import check_automation_limits from open_webui.utils.automations import next_n_runs_ns, next_run_ns, validate_rrule diff --git a/backend/open_webui/utils/access_control/files.py b/backend/open_webui/utils/access_control/files.py index c8a10910da..4052bd0798 100644 --- a/backend/open_webui/utils/access_control/files.py +++ b/backend/open_webui/utils/access_control/files.py @@ -8,7 +8,7 @@ from open_webui.models.folders import FolderModel from open_webui.models.groups import Groups from open_webui.models.knowledge import Knowledges from open_webui.models.models import Models -from open_webui.models.notes import NoteModel, Notes +from open_webui.models.notes import NoteModel, Notes, get_note_files from open_webui.models.users import UserModel, Users from sqlalchemy.ext.asyncio import AsyncSession @@ -28,14 +28,14 @@ def get_note_file_ids(entries: list[dict] | None) -> set[str]: async def has_access_to_note_files( - note: NoteModel, entries: list[dict] | None, user: UserModel, db: AsyncSession | None = None + note: NoteModel | None, entries: list[dict] | None, user: UserModel, db: AsyncSession | None = None ) -> bool: """Check the editor's read access to newly attached files.""" if entries is not None and not isinstance(entries, list): return False # Existing attachments may belong to another editor; only additions need permission. - added_ids = get_note_file_ids(entries) - get_note_file_ids((note.data or {}).get('files')) + added_ids = get_note_file_ids(entries) - get_note_file_ids(get_note_files(note.data) if note else []) if not added_ids or user.role == 'admin': return True diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 8a4e2ba902..934a078f2c 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -52,7 +52,6 @@ from open_webui.models.chats import Chats from open_webui.models.config import Config from open_webui.models.folders import Folders from open_webui.models.models import Models -from open_webui.models.notes import Notes from open_webui.models.users import UserModel, Users from open_webui.retrieval.utils import filter_source_metadata, get_sources_from_items from open_webui.routers.images import ( @@ -2486,28 +2485,6 @@ async def process_chat_payload(request, form_data, user, metadata, model): activated_skill_ids = mentioned_skill_ids available_skills = [] terminal_skills = [] - if is_note_chat: - note_id = (chat.meta or {}).get('note_id') - note = await Notes.get_note_by_id(note_id) if note_id else None - if note and ( - user.role == 'admin' - or note.user_id == user.id - or await AccessGrants.has_access( - user_id=user.id, - resource_type='note', - resource_id=note.id, - permission='read', - ) - ): - note_files = [ - file - for file in ((note.data or {}).get('files') or []) - if isinstance(file, dict) - and file.get('type') != 'image' - and not (file.get('content_type') or '').startswith('image/') - ] - if note_files: - files = [*(files or []), *note_files] model_builtin_tools = model.get('info', {}).get('meta', {}).get('builtinTools') or {} diff --git a/src/lib/apis/notes/index.ts b/src/lib/apis/notes/index.ts index 71626953f3..d8b00d7184 100644 --- a/src/lib/apis/notes/index.ts +++ b/src/lib/apis/notes/index.ts @@ -8,6 +8,39 @@ type NoteItem = { access_grants?: object[]; }; +export type NoteContent = { json: any; html: string; md: string }; +export type NotePage = { + id: string; + parent_id?: string | null; + inherit_note_title?: boolean; + title: string; + content: NoteContent; + versions: NoteContent[]; + revision: number; + generation: number; +}; +export type NotePageOperation = { + action: 'create' | 'rename' | 'duplicate' | 'reorder' | 'move' | 'delete'; + revision: number; + page_id?: string; + new_id?: string; + parent_id?: string | null; + before_id?: string | null; + title?: string; + order?: string[]; +}; + +export const updateNotePages = async (token: string, id: string, operation: NotePageOperation) => { + const response = await fetch(`${WEBUI_API_BASE_URL}/notes/${id}/pages/update`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', authorization: `Bearer ${token}` }, + body: JSON.stringify(operation) + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || 'Failed to update pages'); + return result; +}; + export const createNewNote = async (token: string, note: NoteItem) => { let error = null; @@ -318,7 +351,7 @@ export const createNoteChatById = async (token: string, id: string) => { export const updateNoteById = async ( token: string, id: string, - note: Omit+ {$i18n.t('Its subpages will be kept and moved up one level.')} +
{/if} +