From 189c14fc4d91d89b27f5333283f551c3b175c967 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:24:05 +0200 Subject: [PATCH] fix: match both JSON text spellings when searching serialised JSON columns (#28399) Three searches LIKE against cast(json_col AS text), which means they have to match bytes a JSON encoder wrote. Encoders disagree on non-ASCII: stdlib escapes it to \uXXXX, orjson writes it raw. Which one produced a row depends on the codec in force when it was written, so any single pattern finds only half the table. models.py hard-codes the stdlib spelling, with a comment asserting SQLite stores JSON via json.dumps(ensure_ascii=True). Model.meta is a JSONField, which has serialised through JSONCodec since ENABLE_ORJSON was introduced, so on that setting it stores raw UTF-8 and the escaped pattern matches nothing: non-ASCII workspace model tag search is broken today. prompts.py and automations.py hard-code the opposite spelling and miss rows written the other way. json_text_variants returns both spellings a string can take inside serialised JSON, collapsing to one for ASCII, and the three call sites OR over them. Rows written under either setting are now found under either setting, which also covers a database holding a mix of the two. Case handling is unchanged. models.py keeps matching non-ASCII tags case-sensitively on SQLite, whose LOWER() is ASCII-only and would not fold the stored text the way str.lower() folds the tag. ASCII tags collapse to a single variant and take exactly the query they took before. Verified on SQLite across every combination of codec-that-wrote-the-row and codec-the-app-is-running, for an ASCII and a CJK tag, over all three call sites: 24 of 24 match, against 12 of 24 before. Quoting still bounds whole-tag matches, so searching "weather" does not match a row tagged "weathervane". Co-authored-by: Claude --- backend/open_webui/models/automations.py | 9 +++++---- backend/open_webui/models/models.py | 20 +++++++------------- backend/open_webui/models/prompts.py | 9 +++++---- backend/open_webui/utils/misc.py | 11 +++++++++++ 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/backend/open_webui/models/automations.py b/backend/open_webui/models/automations.py index 1eaaca7175..11a906670b 100644 --- a/backend/open_webui/models/automations.py +++ b/backend/open_webui/models/automations.py @@ -4,6 +4,7 @@ from typing import Literal, Optional from uuid import uuid4 from open_webui.internal.db import Base, get_async_db_context +from open_webui.utils.misc import json_text_variants from pydantic import BaseModel, ConfigDict from sqlalchemy import JSON, BigInteger, Boolean, Column, Index, String, Text, cast, delete, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession @@ -189,12 +190,12 @@ class AutomationTable: stmt = stmt.filter(Automation.folder_id == folder_id) if query: - search = f'%{query}%' - # Search in name and prompt inside JSON data + # Search the name column and the prompt inside the JSON data. + data_text = cast(Automation.data, String) stmt = stmt.filter( or_( - Automation.name.ilike(search), - cast(Automation.data, String).ilike(search), + Automation.name.ilike(f'%{query}%'), + *(data_text.ilike(f'%{variant}%') for variant in json_text_variants(query)), ) ) diff --git a/backend/open_webui/models/models.py b/backend/open_webui/models/models.py index 5146724a93..eac1812c98 100755 --- a/backend/open_webui/models/models.py +++ b/backend/open_webui/models/models.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import logging import time from copy import deepcopy @@ -10,6 +9,7 @@ from open_webui.internal.db import Base, JSONField, get_async_db_context from open_webui.models.access_grants import AccessGrantModel, AccessGrants from open_webui.models.groups import Groups from open_webui.models.users import User, UserModel, UserResponse, Users +from open_webui.utils.misc import json_text_variants from open_webui.utils.validate import validate_profile_image_url from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from sqlalchemy import BigInteger, Boolean, Column, String, Text, cast, delete, func, or_, select, update @@ -374,20 +374,14 @@ class ModelsTable: tag = filter.get('tag') if tag: - # SQLite stores JSON text via json.dumps(ensure_ascii=True), - # so non-ASCII chars are \uXXXX-escaped. PostgreSQL native JSONB - # stores literal Unicode. Use the right pattern for each. - if db.bind.dialect.name == 'sqlite': - if tag.isascii(): - meta_text = func.lower(cast(Model.meta, String)) - pattern = f'%{json.dumps(tag.lower())}%' - else: - meta_text = cast(Model.meta, String) - pattern = f'%{json.dumps(tag)}%' + if db.bind.dialect.name == 'sqlite' and not tag.isascii(): + # SQLite's LOWER() is ASCII-only, so match non-ASCII tags exact-case. + meta_text = cast(Model.meta, String) + variants = json_text_variants(tag) else: meta_text = func.lower(cast(Model.meta, String)) - pattern = f'%{json.dumps(tag.lower(), ensure_ascii=False)}%' - stmt = stmt.filter(meta_text.like(pattern)) + variants = json_text_variants(tag.lower()) + stmt = stmt.filter(or_(*(meta_text.like(f'%"{variant}"%') for variant in variants))) order_by = filter.get('order_by') direction = filter.get('direction') diff --git a/backend/open_webui/models/prompts.py b/backend/open_webui/models/prompts.py index e985bcc70d..54e67ac2f4 100644 --- a/backend/open_webui/models/prompts.py +++ b/backend/open_webui/models/prompts.py @@ -14,7 +14,7 @@ from open_webui.models.access_grants import AccessGrantModel, AccessGrants from open_webui.models.groups import Groups from open_webui.models.prompt_history import PromptHistories from open_webui.models.users import User, UserModel, UserResponse, Users -from open_webui.utils.json_codec import JSONCodec +from open_webui.utils.misc import json_text_variants from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import JSON, BigInteger, Boolean, Column, String, Text, cast, delete, func, or_, select, text, update from sqlalchemy.ext.asyncio import AsyncSession @@ -342,9 +342,10 @@ class PromptsTable: 'EXISTS (SELECT 1 FROM json_array_elements_text(prompt.tags) t WHERE LOWER(t) = :tag_val)' ) else: - # Fallback: LIKE on serialised JSON text (ASCII-safe only) - tag_clause = func.lower(cast(Prompt.tags, String)).like( - f'%{JSONCodec.dumps(tag_lower, ensure_ascii=False)}%' + # Fallback for dialects with no JSON array function: LIKE on the text. + tags_text = func.lower(cast(Prompt.tags, String)) + tag_clause = or_( + *(tags_text.like(f'%"{variant}"%') for variant in json_text_variants(tag_lower)) ) tag_lower = None diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index ac3eec866d..c24e6568d8 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -833,6 +833,17 @@ def sanitize_filename(file_name): return final_file_name +def json_text_variants(value: str) -> list[str]: + """Both spellings ``value`` can take inside a serialized JSON column, unquoted. + + Encoders disagree on non-ASCII — stdlib escapes it to ``\\uXXXX``, orjson writes it + raw — so a LIKE against the stored text has to accept either. ASCII collapses to one. + """ + raw = JSONCodec.dumps(value, ensure_ascii=False)[1:-1] + escaped = JSONCodec.dumps(value, ensure_ascii=True)[1:-1] + return [raw] if raw == escaped else [raw, escaped] + + def sanitize_text_for_db(text: str) -> str: """Remove null bytes and invalid UTF-8 surrogates from text for PostgreSQL storage.""" if not isinstance(text, str):