scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-11-05 07:57:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: handle non-string note content and multi-line :param docstrings

Browse source
This commit is contained in:
Scorpoon 2026-08-10 03:10:51 +02:00
parent 2dadc5435a
commit 1f72e1aa47
No known key found for this signature in database
6 changed files with 235 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
backend/open_webui/migrations/versions/0f47c530da11_fix_corrupt_note_md.py Normal file
View file

@ -0,0 +1,82 @@
"""fix corrupt note md content
Revision ID: 0f47c530da11
Revises: 1ce6ade7d93b
Create Date: 2026-08-05 21:06:03.932000
"""
from typing import Sequence, Union
import json
import sqlalchemy as sa
from alembic import op
from sqlalchemy.sql import column, select, table, update
# revision identifiers, used by Alembic.
revision: str = '0f47c530da11'
down_revision: Union[str, None] = '1ce6ade7d93b'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _ensure_md_string(value):
"""Inline copy of ensure_md_string for migration self-containment."""
if isinstance(value, str):
return value
if value is None:
return ''
return '```text\n' + json.dumps(value, indent=2, ensure_ascii=False) + '\n```'
def upgrade() -> None:
conn = op.get_bind()
note_table = table(
'note',
column('id', sa.Text),
column('data', sa.JSON),
)
result = conn.execute(select(note_table.c.id, note_table.c.data)).yield_per(100)
fixed_count = 0
for note_id, data in result:
if data is None or not isinstance(data, dict):
continue
content = data.get('content')
# Handle non-dict content (e.g. list, str, int)
if not isinstance(content, dict):
fixed_md = _ensure_md_string(content)
fixed_data = {**data, 'content': {'md': fixed_md}}
conn.execute(
update(note_table)
.where(note_table.c.id == note_id)
.values(data=fixed_data)
)
fixed_count += 1
continue
md = content.get('md')
if md is None or isinstance(md, str):
continue
fixed_md = _ensure_md_string(md)
fixed_data = {
**data,
'content': {**content, 'md': fixed_md},
}
conn.execute(
update(note_table)
.where(note_table.c.id == note_id)
.values(data=fixed_data)
)
fixed_count += 1
print(f'fix_corrupt_note_md: fixed {fixed_count} note(s)')
def downgrade() -> None:
# Irreversible: original non-string md content cannot be recovered
pass

46
backend/open_webui/models/notes.py
View file

@ -7,7 +7,8 @@ from open_webui.internal.db import Base, get_async_db_context
from open_webui.models.access_grants import AccessGrantModel, AccessGrants
from open_webui.models.groups import Groups
from open_webui.models.users import User, UserModel, UserResponse, Users
from pydantic import BaseModel, ConfigDict, Field
from open_webui.utils.notes import ensure_md_string
from pydantic import BaseModel, ConfigDict, Field, field_validator
from sqlalchemy import JSON, BigInteger, Boolean, Column, ForeignKey, Text, delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession
@ -61,12 +62,50 @@ class PinnedNote(Base):
####################
def _coerce_note_data_md(data: Optional[dict]) -> Optional[dict]:
"""Normalize data['content']['md'] through ensure_md_string.
Closes the REST API boundary gap: even if a client sends a dict/list
as md, or a non-dict value as content, it gets coerced to a safe
markdown string before hitting the DB.
"""
if not data or not isinstance(data, dict):
return data
content = data.get('content')
# Handle non-dict content (e.g. list, str, int) — coerce the whole value
if not isinstance(content, dict):
coerced = ensure_md_string(content)
data = dict(data)
data['content'] = {'md': coerced}
return data
md = content.get('md')
if md is None:
return data
coerced = ensure_md_string(md)
if coerced != md:
# Only mutate if something actually changed
data = dict(data)
content = dict(content)
content['md'] = coerced
data['content'] = content
return data
class NoteForm(BaseModel):
title: str
data: Optional[dict] = None
meta: Optional[dict] = None
access_grants: Optional[list[dict]] = None
@field_validator('data', mode='before')
@classmethod
def validate_data(cls, v: Optional[dict]) -> Optional[dict]:
return _coerce_note_data_md(v)
class NoteUpdateForm(BaseModel):
title: Optional[str] = None
@ -74,6 +113,11 @@ class NoteUpdateForm(BaseModel):
meta: Optional[dict] = None
access_grants: Optional[list[dict]] = None
@field_validator('data', mode='before')
@classmethod
def validate_data(cls, v: Optional[dict]) -> Optional[dict]:
return _coerce_note_data_md(v)
class NoteUserResponse(NoteModel):
user: Optional[UserResponse] = None

3
backend/open_webui/routers/notes.py
View file

@ -31,6 +31,7 @@ from open_webui.utils.access_control import (
has_public_write_access_grant,
)
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.notes import get_note_md
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
@ -42,7 +43,7 @@ router = APIRouter()
def _truncate_note_data(data: Optional[dict], max_length: int = 1000) -> Optional[dict]:
if not data:
return data
md = (data.get('content') or {}).get('md') or ''
md = get_note_md(data)
return {'content': {'md': md[:max_length]}}

54
backend/open_webui/tools/builtin.py
View file

@ -66,6 +66,7 @@ from open_webui.tasks import stop_item_tasks
from open_webui.tools.knowledge_fs import kb_exec # noqa: F401 — re-exported
from open_webui.utils.chat_id import is_saved_chat_id
from open_webui.utils.json_codec import JSONCodec
from open_webui.utils.notes import get_note_md
from open_webui.utils.notifications import notify_target
from open_webui.utils.sanitize import sanitize_code
@ -1050,8 +1051,8 @@ async def search_notes(
# Extract a snippet from the markdown content
content_snippet = ''
if note.data and note.data.get('content', {}).get('md'):
md_content = note.data['content']['md']
md_content = get_note_md(note.data)
if md_content:
content_lower = md_content.lower()
# Find the first matching word to center the snippet around.
@ -1137,9 +1138,7 @@ async def view_note(
return JSONCodec.dumps({'error': 'Access denied'})
# Extract markdown content
content = ''
if note.data and note.data.get('content', {}).get('md'):
content = note.data['content']['md']
content = get_note_md(note.data)
return JSONCodec.dumps(
{
@ -1165,8 +1164,18 @@ async def write_note(
"""
Create a new note with the given title and content.
IMPORTANT: The 'content' parameter must ALWAYS be a plain text string (markdown).
If the note contains JSON, code, or structured data, wrap it in a markdown code block.
Never pass raw JSON objects or dicts as content -- they will be rejected.
Examples:
- Plain text: "Hello world"
- JSON content: "```json\n{\"name\": \"test\", \"version\": \"1.0\"}\n```"
- Code content: "```python\nprint('hello')\n```"
:param title: The title of the new note
:param content: The markdown content for the note
:param content: The markdown content for the note. MUST be a string.
If the content is JSON or code, wrap it in a markdown code block (e.g., ```json ... ```).
:return: JSON with success status and new note id
"""
if __request__ is None:
@ -1180,6 +1189,12 @@ async def write_note(
user_id = __user__.get('id')
if not isinstance(content, str):
return JSONCodec.dumps({
'error': f'content must be a string (markdown text), got {type(content).__name__}',
'code': 'invalid_content_type',
})
form = NoteForm(
title=title,
data={'content': {'md': content}},
@ -1216,8 +1231,18 @@ async def replace_note_content(
"""
Update an existing note by replacing the whole markdown content or applying range operations.
IMPORTANT: The 'content' parameter must ALWAYS be a plain text string (markdown).
If the note contains JSON, code, or structured data, wrap it in a markdown code block.
Never pass raw JSON objects or dicts as content -- they will be rejected.
Examples:
- Plain text: "Hello world"
- JSON content: "```json\n{\"name\": \"test\", \"version\": \"1.0\"}\n```"
- Code content: "```python\nprint('hello')\n```"
:param note_id: The ID of the note to update
:param content: The new markdown content for a whole-note update
:param content: The new markdown content for a whole-note update. MUST be a string.
If the content is JSON or code, wrap it in a markdown code block (e.g., ```json ... ```).
:param operations: Optional note operations:
- {"action": "replace", "content": "..."}
- {"action": "replace_range", "start": 0, "end": 10, "content": "...", "expected": "..."}
@ -1242,7 +1267,7 @@ async def replace_note_content(
if __user__.get('role') != 'admin' and not await _has_write_access_to_note(note, user_id):
return JSONCodec.dumps({'error': 'Write access denied', 'code': 'write_access_denied'})
current_content = ((note.data or {}).get('content') or {}).get('md') or ''
current_content = get_note_md(note.data)
applied_operation_count = 0
if operations is not None:
if not isinstance(operations, list) or len(operations) == 0:
@ -1327,16 +1352,25 @@ async def replace_note_content(
elif content is None:
return JSONCodec.dumps({'error': 'content or operations is required', 'code': 'content_required'})
if content is not None and not isinstance(content, str):
return JSONCodec.dumps({
'error': f'content must be a string (markdown text), got {type(content).__name__}',
'code': 'invalid_content_type',
})
try:
await stop_item_tasks(__request__.app.state.redis, f'note:{note_id}')
except Exception:
pass
# Safely extract existing content dict — guard against non-dict content
existing_content = note.data.get('content') if isinstance(note.data, dict) else None
content_base = existing_content if isinstance(existing_content, dict) else {}
update_data = {
'data': {
**(note.data or {}),
'content': {
**((note.data or {}).get('content') or {}),
**content_base,
'json': None,
'html': '',
'md': content,
@ -3124,7 +3158,7 @@ async def query_knowledge_files(
permission='read',
)
):
content = note.data.get('content', {}).get('md', '')
content = get_note_md(note.data)
note_results.append(
{
'content': content,

28
backend/open_webui/utils/notes.py Normal file
View file

@ -0,0 +1,28 @@
import json
from typing import Any, Optional
def ensure_md_string(value: Any) -> str:
if isinstance(value, str):
return value
if value is None:
return ''
return '```text\n' + json.dumps(value, indent=2, ensure_ascii=False) + '\n```'
def get_note_md(data: Optional[dict]) -> str:
"""Safely extract data['content']['md'] as a string.
Handles all corruption cases:
- data is None / missing 'content' / 'content' is not a dict (list, str, etc.)
- 'md' is missing, None, or a non-string type (dict, list, etc.)
Returns the md string if valid, otherwise a safe coercion via ensure_md_string.
"""
if not data or not isinstance(data, dict):
return ''
content = data.get('content')
if not isinstance(content, dict):
return ensure_md_string(content)
md = content.get('md')
return ensure_md_string(md)

43
backend/open_webui/utils/tools.py
View file

@ -848,18 +848,43 @@ def parse_docstring(docstring):
return {}
# Regex to match `:param name: description` format
param_pattern = re.compile(r':param (\w+):\s*(.+)')
param_pattern = re.compile(r':param (\w+):\s*(.*)')
param_descriptions = {}
for line in docstring.splitlines():
match = param_pattern.match(line.strip())
if not match:
continue
param_name, param_description = match.groups()
if param_name.startswith('__'):
continue
param_descriptions[param_name] = param_description
lines = docstring.splitlines()
current_param = None
current_desc: list[str] = []
def flush_param():
nonlocal current_param, current_desc
if current_param and not current_param.startswith('__'):
param_descriptions[current_param] = '\n'.join(current_desc).strip()
current_param = None
current_desc = []
for line in lines:
# Check for new :param
match = param_pattern.match(line.strip())
if match:
flush_param()
current_param = match.group(1)
desc_part = match.group(2).strip()
if desc_part:
current_desc.append(desc_part)
continue
# Check for :return (ends current param)
if re.match(r'\s*:return', line):
flush_param()
continue
# Continuation line for current param
if current_param is not None:
stripped = line.strip()
if stripped:
current_desc.append(stripped)
flush_param()
return param_descriptions