From 1f72e1aa47af5d5fca125797f35f8b0ffcc1ccf1 Mon Sep 17 00:00:00 2001 From: Scorpoon <22363852+Scorpoon@users.noreply.github.com> Date: Mon, 10 Aug 2026 03:10:51 +0200 Subject: [PATCH] fix: handle non-string note content and multi-line :param docstrings --- .../0f47c530da11_fix_corrupt_note_md.py | 82 +++++++++++++++++++ backend/open_webui/models/notes.py | 46 ++++++++++- backend/open_webui/routers/notes.py | 3 +- backend/open_webui/tools/builtin.py | 54 +++++++++--- backend/open_webui/utils/notes.py | 28 +++++++ backend/open_webui/utils/tools.py | 43 ++++++++-- 6 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 backend/open_webui/migrations/versions/0f47c530da11_fix_corrupt_note_md.py create mode 100644 backend/open_webui/utils/notes.py diff --git a/backend/open_webui/migrations/versions/0f47c530da11_fix_corrupt_note_md.py b/backend/open_webui/migrations/versions/0f47c530da11_fix_corrupt_note_md.py new file mode 100644 index 0000000000..422fc558a5 --- /dev/null +++ b/backend/open_webui/migrations/versions/0f47c530da11_fix_corrupt_note_md.py @@ -0,0 +1,82 @@ +"""fix corrupt note md content + +Revision ID: 0f47c530da11 +Revises: 1ce6ade7d93b +Create Date: 2026-08-05 21:06:03.932000 + +""" + +from typing import Sequence, Union + +import json +import sqlalchemy as sa +from alembic import op +from sqlalchemy.sql import column, select, table, update + +# revision identifiers, used by Alembic. +revision: str = '0f47c530da11' +down_revision: Union[str, None] = '1ce6ade7d93b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _ensure_md_string(value): + """Inline copy of ensure_md_string for migration self-containment.""" + if isinstance(value, str): + return value + if value is None: + return '' + return '```text\n' + json.dumps(value, indent=2, ensure_ascii=False) + '\n```' + + +def upgrade() -> None: + conn = op.get_bind() + note_table = table( + 'note', + column('id', sa.Text), + column('data', sa.JSON), + ) + + result = conn.execute(select(note_table.c.id, note_table.c.data)).yield_per(100) + fixed_count = 0 + + for note_id, data in result: + if data is None or not isinstance(data, dict): + continue + + content = data.get('content') + + # Handle non-dict content (e.g. list, str, int) + if not isinstance(content, dict): + fixed_md = _ensure_md_string(content) + fixed_data = {**data, 'content': {'md': fixed_md}} + conn.execute( + update(note_table) + .where(note_table.c.id == note_id) + .values(data=fixed_data) + ) + fixed_count += 1 + continue + + md = content.get('md') + if md is None or isinstance(md, str): + continue + + fixed_md = _ensure_md_string(md) + fixed_data = { + **data, + 'content': {**content, 'md': fixed_md}, + } + conn.execute( + update(note_table) + .where(note_table.c.id == note_id) + .values(data=fixed_data) + ) + fixed_count += 1 + + print(f'fix_corrupt_note_md: fixed {fixed_count} note(s)') + + +def downgrade() -> None: + # Irreversible: original non-string md content cannot be recovered + pass diff --git a/backend/open_webui/models/notes.py b/backend/open_webui/models/notes.py index c651c6f51e..30203eb1e6 100644 --- a/backend/open_webui/models/notes.py +++ b/backend/open_webui/models/notes.py @@ -7,7 +7,8 @@ from open_webui.internal.db import Base, get_async_db_context from open_webui.models.access_grants import AccessGrantModel, AccessGrants from open_webui.models.groups import Groups from open_webui.models.users import User, UserModel, UserResponse, Users -from pydantic import BaseModel, ConfigDict, Field +from open_webui.utils.notes import ensure_md_string +from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy import JSON, BigInteger, Boolean, Column, ForeignKey, Text, delete, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession @@ -61,12 +62,50 @@ class PinnedNote(Base): #################### +def _coerce_note_data_md(data: Optional[dict]) -> Optional[dict]: + """Normalize data['content']['md'] through ensure_md_string. + + Closes the REST API boundary gap: even if a client sends a dict/list + as md, or a non-dict value as content, it gets coerced to a safe + markdown string before hitting the DB. + """ + if not data or not isinstance(data, dict): + return data + + content = data.get('content') + + # Handle non-dict content (e.g. list, str, int) — coerce the whole value + if not isinstance(content, dict): + coerced = ensure_md_string(content) + data = dict(data) + data['content'] = {'md': coerced} + return data + + md = content.get('md') + if md is None: + return data + + coerced = ensure_md_string(md) + if coerced != md: + # Only mutate if something actually changed + data = dict(data) + content = dict(content) + content['md'] = coerced + data['content'] = content + return data + + class NoteForm(BaseModel): title: str data: Optional[dict] = None meta: Optional[dict] = None access_grants: Optional[list[dict]] = None + @field_validator('data', mode='before') + @classmethod + def validate_data(cls, v: Optional[dict]) -> Optional[dict]: + return _coerce_note_data_md(v) + class NoteUpdateForm(BaseModel): title: Optional[str] = None @@ -74,6 +113,11 @@ class NoteUpdateForm(BaseModel): meta: Optional[dict] = None access_grants: Optional[list[dict]] = None + @field_validator('data', mode='before') + @classmethod + def validate_data(cls, v: Optional[dict]) -> Optional[dict]: + return _coerce_note_data_md(v) + class NoteUserResponse(NoteModel): user: Optional[UserResponse] = None diff --git a/backend/open_webui/routers/notes.py b/backend/open_webui/routers/notes.py index 3f598c69c4..752ed77ac3 100644 --- a/backend/open_webui/routers/notes.py +++ b/backend/open_webui/routers/notes.py @@ -31,6 +31,7 @@ from open_webui.utils.access_control import ( has_public_write_access_grant, ) from open_webui.utils.auth import get_admin_user, get_verified_user +from open_webui.utils.notes import get_note_md from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession @@ -42,7 +43,7 @@ router = APIRouter() def _truncate_note_data(data: Optional[dict], max_length: int = 1000) -> Optional[dict]: if not data: return data - md = (data.get('content') or {}).get('md') or '' + md = get_note_md(data) return {'content': {'md': md[:max_length]}} diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index c71b305898..13e62fbbee 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -66,6 +66,7 @@ from open_webui.tasks import stop_item_tasks from open_webui.tools.knowledge_fs import kb_exec # noqa: F401 — re-exported from open_webui.utils.chat_id import is_saved_chat_id from open_webui.utils.json_codec import JSONCodec +from open_webui.utils.notes import get_note_md from open_webui.utils.notifications import notify_target from open_webui.utils.sanitize import sanitize_code @@ -1050,8 +1051,8 @@ async def search_notes( # Extract a snippet from the markdown content content_snippet = '' - if note.data and note.data.get('content', {}).get('md'): - md_content = note.data['content']['md'] + md_content = get_note_md(note.data) + if md_content: content_lower = md_content.lower() # Find the first matching word to center the snippet around. @@ -1137,9 +1138,7 @@ async def view_note( return JSONCodec.dumps({'error': 'Access denied'}) # Extract markdown content - content = '' - if note.data and note.data.get('content', {}).get('md'): - content = note.data['content']['md'] + content = get_note_md(note.data) return JSONCodec.dumps( { @@ -1165,8 +1164,18 @@ async def write_note( """ Create a new note with the given title and content. + IMPORTANT: The 'content' parameter must ALWAYS be a plain text string (markdown). + If the note contains JSON, code, or structured data, wrap it in a markdown code block. + Never pass raw JSON objects or dicts as content -- they will be rejected. + + Examples: + - Plain text: "Hello world" + - JSON content: "```json\n{\"name\": \"test\", \"version\": \"1.0\"}\n```" + - Code content: "```python\nprint('hello')\n```" + :param title: The title of the new note - :param content: The markdown content for the note + :param content: The markdown content for the note. MUST be a string. + If the content is JSON or code, wrap it in a markdown code block (e.g., ```json ... ```). :return: JSON with success status and new note id """ if __request__ is None: @@ -1180,6 +1189,12 @@ async def write_note( user_id = __user__.get('id') + if not isinstance(content, str): + return JSONCodec.dumps({ + 'error': f'content must be a string (markdown text), got {type(content).__name__}', + 'code': 'invalid_content_type', + }) + form = NoteForm( title=title, data={'content': {'md': content}}, @@ -1216,8 +1231,18 @@ async def replace_note_content( """ Update an existing note by replacing the whole markdown content or applying range operations. + IMPORTANT: The 'content' parameter must ALWAYS be a plain text string (markdown). + If the note contains JSON, code, or structured data, wrap it in a markdown code block. + Never pass raw JSON objects or dicts as content -- they will be rejected. + + Examples: + - Plain text: "Hello world" + - JSON content: "```json\n{\"name\": \"test\", \"version\": \"1.0\"}\n```" + - Code content: "```python\nprint('hello')\n```" + :param note_id: The ID of the note to update - :param content: The new markdown content for a whole-note update + :param content: The new markdown content for a whole-note update. MUST be a string. + If the content is JSON or code, wrap it in a markdown code block (e.g., ```json ... ```). :param operations: Optional note operations: - {"action": "replace", "content": "..."} - {"action": "replace_range", "start": 0, "end": 10, "content": "...", "expected": "..."} @@ -1242,7 +1267,7 @@ async def replace_note_content( if __user__.get('role') != 'admin' and not await _has_write_access_to_note(note, user_id): return JSONCodec.dumps({'error': 'Write access denied', 'code': 'write_access_denied'}) - current_content = ((note.data or {}).get('content') or {}).get('md') or '' + current_content = get_note_md(note.data) applied_operation_count = 0 if operations is not None: if not isinstance(operations, list) or len(operations) == 0: @@ -1327,16 +1352,25 @@ async def replace_note_content( elif content is None: return JSONCodec.dumps({'error': 'content or operations is required', 'code': 'content_required'}) + if content is not None and not isinstance(content, str): + return JSONCodec.dumps({ + 'error': f'content must be a string (markdown text), got {type(content).__name__}', + 'code': 'invalid_content_type', + }) + try: await stop_item_tasks(__request__.app.state.redis, f'note:{note_id}') except Exception: pass + # Safely extract existing content dict — guard against non-dict content + existing_content = note.data.get('content') if isinstance(note.data, dict) else None + content_base = existing_content if isinstance(existing_content, dict) else {} update_data = { 'data': { **(note.data or {}), 'content': { - **((note.data or {}).get('content') or {}), + **content_base, 'json': None, 'html': '', 'md': content, @@ -3124,7 +3158,7 @@ async def query_knowledge_files( permission='read', ) ): - content = note.data.get('content', {}).get('md', '') + content = get_note_md(note.data) note_results.append( { 'content': content, diff --git a/backend/open_webui/utils/notes.py b/backend/open_webui/utils/notes.py new file mode 100644 index 0000000000..36bdbe830c --- /dev/null +++ b/backend/open_webui/utils/notes.py @@ -0,0 +1,28 @@ +import json +from typing import Any, Optional + + +def ensure_md_string(value: Any) -> str: + if isinstance(value, str): + return value + if value is None: + return '' + return '```text\n' + json.dumps(value, indent=2, ensure_ascii=False) + '\n```' + + +def get_note_md(data: Optional[dict]) -> str: + """Safely extract data['content']['md'] as a string. + + Handles all corruption cases: + - data is None / missing 'content' / 'content' is not a dict (list, str, etc.) + - 'md' is missing, None, or a non-string type (dict, list, etc.) + + Returns the md string if valid, otherwise a safe coercion via ensure_md_string. + """ + if not data or not isinstance(data, dict): + return '' + content = data.get('content') + if not isinstance(content, dict): + return ensure_md_string(content) + md = content.get('md') + return ensure_md_string(md) diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 7e44d28820..89c8ca5716 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -848,18 +848,43 @@ def parse_docstring(docstring): return {} # Regex to match `:param name: description` format - param_pattern = re.compile(r':param (\w+):\s*(.+)') + param_pattern = re.compile(r':param (\w+):\s*(.*)') param_descriptions = {} - for line in docstring.splitlines(): - match = param_pattern.match(line.strip()) - if not match: - continue - param_name, param_description = match.groups() - if param_name.startswith('__'): - continue - param_descriptions[param_name] = param_description + lines = docstring.splitlines() + current_param = None + current_desc: list[str] = [] + def flush_param(): + nonlocal current_param, current_desc + if current_param and not current_param.startswith('__'): + param_descriptions[current_param] = '\n'.join(current_desc).strip() + current_param = None + current_desc = [] + + for line in lines: + # Check for new :param + match = param_pattern.match(line.strip()) + if match: + flush_param() + current_param = match.group(1) + desc_part = match.group(2).strip() + if desc_part: + current_desc.append(desc_part) + continue + + # Check for :return (ends current param) + if re.match(r'\s*:return', line): + flush_param() + continue + + # Continuation line for current param + if current_param is not None: + stripped = line.strip() + if stripped: + current_desc.append(stripped) + + flush_param() return param_descriptions