diff --git a/backend/open_webui/internal/db.py b/backend/open_webui/internal/db.py index b0545255a6..fd7a990e8f 100644 --- a/backend/open_webui/internal/db.py +++ b/backend/open_webui/internal/db.py @@ -55,8 +55,8 @@ class JSONField(types.TypeDecorator): def handle_peewee_migration(DATABASE_URL): # db = None try: - # Replace the postgresql:// with postgres:// to handle the peewee migration - db = register_connection(DATABASE_URL.replace('postgresql://', 'postgres://')) + # register_connection() normalizes SQLAlchemy URL variants for Peewee. + db = register_connection(DATABASE_URL) migrate_dir = OPEN_WEBUI_DIR / 'internal' / 'migrations' router = Router(db, logger=log, migrate_dir=migrate_dir) router.run() diff --git a/backend/open_webui/internal/migrations/007_add_user_last_active_at.py b/backend/open_webui/internal/migrations/007_add_user_last_active_at.py index 19a26c3515..d956bf4266 100644 --- a/backend/open_webui/internal/migrations/007_add_user_last_active_at.py +++ b/backend/open_webui/internal/migrations/007_add_user_last_active_at.py @@ -33,6 +33,14 @@ with suppress(ImportError): import playhouse.postgres_ext as pw_pext +def _user_ident(database: pw.Database) -> str: + # Postgres: "user" (identifier quoting) + # MariaDB/MySQL: `user` (identifier quoting) + mod = database.__class__.__module__.lower() + name = database.__class__.__name__.lower() + return '`user`' if ('mysql' in mod or 'pymysql' in mod or 'mysql' in name or 'mariadb' in name) else '"user"' + + def migrate(migrator: Migrator, database: pw.Database, *, fake=False): """Write your migrations here.""" @@ -45,8 +53,9 @@ def migrate(migrator: Migrator, database: pw.Database, *, fake=False): ) # Populate the new fields from an existing 'timestamp' field + user_tbl = _user_ident(database) migrator.sql( - 'UPDATE "user" SET created_at = timestamp, updated_at = timestamp, last_active_at = timestamp WHERE timestamp IS NOT NULL' + f'UPDATE {user_tbl} SET created_at = timestamp, updated_at = timestamp, last_active_at = timestamp WHERE timestamp IS NOT NULL' ) # Now that the data has been copied, remove the original 'timestamp' field @@ -69,7 +78,8 @@ def rollback(migrator: Migrator, database: pw.Database, *, fake=False): # Copy the earliest created_at date back into the new timestamp field # This assumes created_at was originally a copy of timestamp - migrator.sql('UPDATE "user" SET timestamp = created_at') + user_tbl = _user_ident(database) + migrator.sql(f'UPDATE {user_tbl} SET timestamp = created_at') # Remove the created_at and updated_at fields migrator.remove_fields('user', 'created_at', 'updated_at', 'last_active_at') diff --git a/backend/open_webui/internal/migrations/009_add_models.py b/backend/open_webui/internal/migrations/009_add_models.py index 45f4a3d163..735a20edbf 100644 --- a/backend/open_webui/internal/migrations/009_add_models.py +++ b/backend/open_webui/internal/migrations/009_add_models.py @@ -29,6 +29,9 @@ from contextlib import suppress import peewee as pw from peewee_migrate import Migrator +from open_webui.internal.utils import key_text + + with suppress(ImportError): import playhouse.postgres_ext as pw_pext @@ -38,9 +41,9 @@ def migrate(migrator: Migrator, database: pw.Database, *, fake=False): @migrator.create_model class Model(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() - base_model_id = pw.TextField(null=True) + id = key_text(database, unique=True) + user_id = key_text(database) + base_model_id = key_text(database, null=True) name = pw.TextField() diff --git a/backend/open_webui/internal/migrations/012_add_tools.py b/backend/open_webui/internal/migrations/012_add_tools.py index a488678c3c..8821fea7ea 100644 --- a/backend/open_webui/internal/migrations/012_add_tools.py +++ b/backend/open_webui/internal/migrations/012_add_tools.py @@ -29,6 +29,9 @@ from contextlib import suppress import peewee as pw from peewee_migrate import Migrator +from open_webui.internal.utils import key_text + + with suppress(ImportError): import playhouse.postgres_ext as pw_pext @@ -38,8 +41,8 @@ def migrate(migrator: Migrator, database: pw.Database, *, fake=False): @migrator.create_model class Tool(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() + id = key_text(database, unique=True) + user_id = key_text(database) name = pw.TextField() content = pw.TextField() diff --git a/backend/open_webui/internal/migrations/014_add_files.py b/backend/open_webui/internal/migrations/014_add_files.py index 9c01ac08c3..caedd85643 100644 --- a/backend/open_webui/internal/migrations/014_add_files.py +++ b/backend/open_webui/internal/migrations/014_add_files.py @@ -29,6 +29,9 @@ from contextlib import suppress import peewee as pw from peewee_migrate import Migrator +from open_webui.internal.utils import key_text + + with suppress(ImportError): import playhouse.postgres_ext as pw_pext @@ -38,8 +41,8 @@ def migrate(migrator: Migrator, database: pw.Database, *, fake=False): @migrator.create_model class File(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() + id = key_text(database, unique=True) + user_id = key_text(database) filename = pw.TextField() meta = pw.TextField() created_at = pw.BigIntegerField(null=False) diff --git a/backend/open_webui/internal/migrations/015_add_functions.py b/backend/open_webui/internal/migrations/015_add_functions.py index 488e546ab1..69f765e94e 100644 --- a/backend/open_webui/internal/migrations/015_add_functions.py +++ b/backend/open_webui/internal/migrations/015_add_functions.py @@ -29,6 +29,9 @@ from contextlib import suppress import peewee as pw from peewee_migrate import Migrator +from open_webui.internal.utils import key_text + + with suppress(ImportError): import playhouse.postgres_ext as pw_pext @@ -38,8 +41,8 @@ def migrate(migrator: Migrator, database: pw.Database, *, fake=False): @migrator.create_model class Function(pw.Model): - id = pw.TextField(unique=True) - user_id = pw.TextField() + id = key_text(database, unique=True) + user_id = key_text(database) name = pw.TextField() type = pw.TextField() diff --git a/backend/open_webui/internal/utils.py b/backend/open_webui/internal/utils.py new file mode 100644 index 0000000000..b88453bd41 --- /dev/null +++ b/backend/open_webui/internal/utils.py @@ -0,0 +1,12 @@ +import peewee as pw + + +def key_text(database: pw.Database, *, max_length: int = 255, **kwargs) -> pw.Field: + """ + Dialect-aware "text-like" field for Peewee migrations: + - MySQL/MariaDB: use VARCHAR(max_length) so UNIQUE/PK/INDEX works without prefix lengths. + - Postgres/SQLite: keep TEXT to preserve existing schema/history. + """ + if isinstance(database, pw.MySQLDatabase): + return pw.CharField(max_length=max_length, **kwargs) + return pw.TextField(**kwargs) diff --git a/backend/open_webui/internal/wrappers.py b/backend/open_webui/internal/wrappers.py index 3d54d02e3a..b006d9b9f9 100644 --- a/backend/open_webui/internal/wrappers.py +++ b/backend/open_webui/internal/wrappers.py @@ -32,6 +32,9 @@ class CustomReconnectMixin(ReconnectMixin): # psycopg2 (OperationalError, 'termin'), (InterfaceError, 'closed'), + # PyMySQL / MySQLdb (MySQL/MariaDB) + (OperationalError, 'server has gone away'), + (OperationalError, 'lost connection'), # peewee (PeeWeeInterfaceError, 'closed'), ) @@ -41,7 +44,34 @@ class ReconnectingPostgresqlDatabase(CustomReconnectMixin, PostgresqlDatabase): pass -def register_connection(db_url): +class ReconnectingMySQLDatabase(CustomReconnectMixin, MySQLDatabase): + pass + + +def normalize_db_url_for_peewee(db_url: str) -> str: + """ + Peewee's db_url helper uses slightly different schemes than SQLAlchemy. + Normalize the base SQLAlchemy dialect scheme before initializing the + Peewee DB, while ignoring any SQLAlchemy driver suffix. + """ + scheme, sep, rest = db_url.partition('://') + if not sep: + return db_url + + base_dialect, _, _ = scheme.partition('+') + + scheme_map = { + 'postgresql': 'postgres', + 'mysql': 'mysql', + 'mariadb': 'mysql', + } + normalized_scheme = scheme_map.get(base_dialect, scheme) + return f'{normalized_scheme}://{rest}' + + +def register_connection(db_url: str): + db_url = normalize_db_url_for_peewee(db_url) + # Check if using SQLCipher protocol if db_url.startswith('sqlite+sqlcipher://'): database_password = os.environ.get('DATABASE_PASSWORD') @@ -60,7 +90,7 @@ def register_connection(db_url): log.info('Connected to encrypted SQLite database using SQLCipher') else: - # Standard database connection (existing logic) + # Standard database connection db = connect(db_url, unquote_user=True, unquote_password=True) if isinstance(db, PostgresqlDatabase): # Enable autoconnect for SQLite databases, managed by Peewee @@ -74,6 +104,13 @@ def register_connection(db_url): # Use our custom database class that supports reconnection db = ReconnectingPostgresqlDatabase(**connection) db.connect(reuse_if_open=True) + elif isinstance(db, MySQLDatabase): + log.info('Connected to MySQL/MariaDB database') + connection = parse(db_url, unquote_user=True, unquote_password=True) + db = ReconnectingMySQLDatabase(**connection) + db.autoconnect = True + db.reuse_if_open = True + db.connect(reuse_if_open=True) elif isinstance(db, SqliteDatabase): # Enable autoconnect for SQLite databases, managed by Peewee db.autoconnect = True diff --git a/backend/open_webui/migrations/README b/backend/open_webui/migrations/README index f1d93dff9d..fa7282127e 100644 --- a/backend/open_webui/migrations/README +++ b/backend/open_webui/migrations/README @@ -2,3 +2,14 @@ Generic single-database configuration. Create new migrations with DATABASE_URL= alembic revision --autogenerate -m "a description" + +Important notes for new migrations: + +- Keep migrations portable across SQLite, PostgreSQL, and MariaDB/MySQL-compatible backends. +- Do not use `sa.Text()` for identifier-like columns that participate in `PRIMARY KEY`, `UNIQUE`, `INDEX`, or `FOREIGN KEY` paths. + Use `open_webui.migrations.util.key_text()` instead for those key-like columns. +- Prefer explicit lengths for `sa.String(...)` in new migrations, especially for key-like columns. +- Be careful with table-level `sa.UniqueConstraint(...)` and `op.create_index(...)`: if any participating column is identifier-like, make sure its type is MariaDB-safe. +- If raw SQL is required, avoid hardcoding identifier quoting that only works on one dialect (for example `"user"`). Use dialect-aware quoting when needed. +- Treat the compatibility shims in `env.py` as a safety net for historical revisions, not as the primary design pattern for new migrations. +- Before merging a new migration, test `alembic upgrade head` on at least one fresh MariaDB database in addition to the usual SQLite/PostgreSQL path. diff --git a/backend/open_webui/migrations/env.py b/backend/open_webui/migrations/env.py index 9ee6c2dceb..1a0338ce0b 100644 --- a/backend/open_webui/migrations/env.py +++ b/backend/open_webui/migrations/env.py @@ -4,8 +4,136 @@ from logging.config import fileConfig from alembic import context from open_webui.models.auths import Auth from open_webui.env import DATABASE_URL, DATABASE_PASSWORD, LOG_FORMAT +import sqlalchemy as sa +from sqlalchemy.ext.compiler import compiles from sqlalchemy import engine_from_config, pool, create_engine +log = logging.getLogger(__name__) + + +# --- MySQL/MariaDB compatibility shims --------------------------------------- +# Goal: +# Keep existing historical Alembic revisions unchanged while making them +# executable on MySQL/MariaDB. +# +# Why this is needed: +# Open-WebUI has older migrations that were originally written with +# SQLite/PostgreSQL-friendly assumptions. In particular: +# - some revisions use sa.String() without an explicit length +# - some revisions use sa.Text() for identifier-like columns such as ids, +# unique keys, indexed columns, or foreign-key columns +# +# On MySQL/MariaDB, these patterns can fail during DDL generation because: +# - VARCHAR requires a length +# - TEXT/BLOB columns cannot be used safely for PRIMARY KEY / UNIQUE / indexed +# columns without a key length +# - foreign-key columns must be type-compatible with the referenced key +# +# Scope: +# These shims affect Alembic migration DDL compilation only. They do not change +# SQLAlchemy model definitions or runtime query behavior outside migrations. +# +# Design note: +# sa.String() can be fixed globally because the issue is type-only (missing length). +# sa.Text() cannot: TEXT is valid in general, but invalid/problematic only when used +# for PK/UNIQUE/index/FK columns, so we must inspect column context during DDL compilation. +# +# This shim is a safety net for historical revisions during CREATE TABLE DDL generation. +# New migrations should still use MariaDB-safe types explicitly (for example key_text()), +# especially for columns that may later participate in op.create_index(...) or other +# for PK/UNIQUE/index/FK columns, so we must inspect column context during DDL compilation. +# +# 1) sa.String() without a length: +# Triggered when a historical migration emits sa.String() and Alembic compiles +# it for the mysql/mariadb dialect. In that case, rewrite it to VARCHAR(255) +# so the migration remains valid without editing the historical revision. +@compiles(sa.String, 'mysql') +@compiles(sa.String, 'mariadb') +def _compile_string_mysql(type_, compiler, **kw): + if type_.length is None: + type_ = sa.String(length=255) + return compiler.visit_VARCHAR(type_, **kw) + + +# +# +# 2) sa.Text() used for key-like columns: +# Triggered when a historical migration defines a TEXT column that is also: +# - a PRIMARY KEY +# - UNIQUE +# - indexed +# - or used as a FOREIGN KEY +# +# During MySQL/MariaDB migration DDL compilation only, rewrite those columns +# from TEXT to VARCHAR(255). This preserves the intent of the old migration +# while satisfying MySQL/MariaDB key and FK requirements. +# +# How it works: +# - patch MySQLDDLCompiler.get_column_specification() +# - inspect each column as Alembic renders CREATE/ALTER TABLE DDL +# - if the column type is sa.Text and it participates in a PK/UNIQUE/index/FK, +# make a shallow column copy and replace its type with sa.String(255) +# - delegate back to SQLAlchemy's original compiler method +# +# This keeps the compatibility logic centralized in env.py instead of +# modifying many already-released migration files. +try: + from sqlalchemy.dialects.mysql.base import MySQLDDLCompiler + + def _is_mysql_key_text_column(column) -> bool: + """ + Return True when a TEXT column participates in a key-like path that is + unsafe on MySQL/MariaDB and should be rewritten to VARCHAR(255) during + CREATE TABLE DDL compilation. + + This covers both: + - column-level flags (primary_key / unique / index / foreign_keys) + - table-level constraints/indexes attached before CREATE TABLE compilation + """ + is_fk_col = bool(getattr(column, 'foreign_keys', None)) + if column.primary_key or column.unique or column.index or is_fk_col: + return True + + table = getattr(column, 'table', None) + if table is None: + return False + + for constraint in getattr(table, 'constraints', ()): + if isinstance(constraint, (sa.PrimaryKeyConstraint, sa.UniqueConstraint)): + try: + if column in constraint.columns: + return True + except Exception: + pass + + for index in getattr(table, 'indexes', ()): + try: + if column in index.columns: + return True + except Exception: + pass + + return False + + if not getattr(MySQLDDLCompiler, '_owui_key_text_patch', False): + _orig_get_colspec = MySQLDDLCompiler.get_column_specification + + def _patched_get_column_specification(self, column, **kw): + # NOTE: For mysql/mariadb, FK columns must be type-compatible with the referenced key. + # If historical migrations used TEXT for FK columns, MySQL/MariaDB will reject the FK. + if isinstance(column.type, sa.Text) and _is_mysql_key_text_column(column): + column = column.copy() + column.type = sa.String(length=255) + return _orig_get_colspec(self, column, **kw) + + MySQLDDLCompiler.get_column_specification = _patched_get_column_specification + MySQLDDLCompiler._owui_key_text_patch = True +except Exception: + if DATABASE_URL.startswith(('mysql', 'mariadb')): + log.exception('Failed to install MySQL/MariaDB compatibility shims') + raise + # If MySQL/MariaDB is not the active dialect, continue without installing the shim. + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config diff --git a/backend/open_webui/migrations/util.py b/backend/open_webui/migrations/util.py index 6ea2a5f4bb..de397fb00b 100644 --- a/backend/open_webui/migrations/util.py +++ b/backend/open_webui/migrations/util.py @@ -1,7 +1,44 @@ -from alembic import op +import sqlalchemy as sa +from alembic import context, op from sqlalchemy import Inspector +def _dialect_name(conn=None) -> str: + """ + Return current dialect name. + Prefer a live Alembic bind when available; fall back to Alembic context + during offline (`--sql`) runs. If no dialect can be determined, return + an empty string so callers can choose a safe default. + """ + if conn is not None: + return (conn.dialect.name or '').lower() + + try: + conn = op.get_bind() + except Exception: + conn = None + + if conn is not None: + return (conn.dialect.name or '').lower() + + try: + return (context.get_context().dialect.name or '').lower() + except Exception: + return '' + + +def key_text(conn=None, length: int = 255): + """ + Dialect-aware TEXT for identifiers/keys. + - MySQL/MariaDB: TEXT cannot be indexed/PK'd without prefix length => use VARCHAR(length). + - PostgreSQL/SQLite: keep TEXT (historical behavior). + """ + d = _dialect_name(conn) + if d in ('mysql', 'mariadb'): + return sa.String(length=length) + return sa.Text() + + def get_existing_tables(): con = op.get_bind() inspector = Inspector.from_engine(con) diff --git a/backend/open_webui/migrations/versions/38d63c18f30f_add_oauth_session_table.py b/backend/open_webui/migrations/versions/38d63c18f30f_add_oauth_session_table.py index d415f500f3..bfe7cdc876 100644 --- a/backend/open_webui/migrations/versions/38d63c18f30f_add_oauth_session_table.py +++ b/backend/open_webui/migrations/versions/38d63c18f30f_add_oauth_session_table.py @@ -10,6 +10,7 @@ from typing import Sequence, Union from alembic import op import sqlalchemy as sa +from open_webui.migrations.util import key_text # revision identifiers, used by Alembic. revision: str = '38d63c18f30f' @@ -52,7 +53,7 @@ def upgrade() -> None: sa.ForeignKey('user.id', ondelete='CASCADE'), nullable=False, ), - sa.Column('provider', sa.Text(), nullable=False), + sa.Column('provider', key_text(), nullable=False), sa.Column('token', sa.Text(), nullable=False), sa.Column('expires_at', sa.BigInteger(), nullable=False), sa.Column('created_at', sa.BigInteger(), nullable=False), diff --git a/backend/open_webui/migrations/versions/8452d01d26d7_add_chat_message_table.py b/backend/open_webui/migrations/versions/8452d01d26d7_add_chat_message_table.py index 3254b57858..014c81ecfe 100644 --- a/backend/open_webui/migrations/versions/8452d01d26d7_add_chat_message_table.py +++ b/backend/open_webui/migrations/versions/8452d01d26d7_add_chat_message_table.py @@ -14,6 +14,8 @@ from typing import Sequence, Union from alembic import op import sqlalchemy as sa +from open_webui.migrations.util import key_text + log = logging.getLogger(__name__) revision: str = '8452d01d26d7' @@ -59,14 +61,14 @@ def upgrade() -> None: # Step 1: Create table op.create_table( 'chat_message', - sa.Column('id', sa.Text(), primary_key=True), - sa.Column('chat_id', sa.Text(), nullable=False, index=True), - sa.Column('user_id', sa.Text(), index=True), + sa.Column('id', key_text(), primary_key=True), + sa.Column('chat_id', key_text(), nullable=False, index=True), + sa.Column('user_id', key_text(), index=True), sa.Column('role', sa.Text(), nullable=False), - sa.Column('parent_id', sa.Text(), nullable=True), + sa.Column('parent_id', key_text(), nullable=True), sa.Column('content', sa.JSON(), nullable=True), sa.Column('output', sa.JSON(), nullable=True), - sa.Column('model_id', sa.Text(), nullable=True, index=True), + sa.Column('model_id', key_text(), nullable=True, index=True), sa.Column('files', sa.JSON(), nullable=True), sa.Column('sources', sa.JSON(), nullable=True), sa.Column('embeds', sa.JSON(), nullable=True), diff --git a/backend/open_webui/migrations/versions/b10670c03dd5_update_user_table.py b/backend/open_webui/migrations/versions/b10670c03dd5_update_user_table.py index 623289d885..a7cb3f2916 100644 --- a/backend/open_webui/migrations/versions/b10670c03dd5_update_user_table.py +++ b/backend/open_webui/migrations/versions/b10670c03dd5_update_user_table.py @@ -40,6 +40,11 @@ def _drop_sqlite_indexes_for_column(table_name, column_name, conn): conn.execute(sa.text(f'DROP INDEX IF EXISTS {index_name}')) +def _quote_table(conn, name: str) -> str: + # Postgres uses "name"; MariaDB/MySQL uses `name` + return f'`{name}`' if conn.dialect.name in ('mysql', 'mariadb') else f'"{name}"' + + def _convert_column_to_json(table: str, column: str): conn = op.get_bind() dialect = conn.dialect.name @@ -139,7 +144,8 @@ def upgrade() -> None: ) conn = op.get_bind() - users = conn.execute(sa.text('SELECT id, oauth_sub FROM "user" WHERE oauth_sub IS NOT NULL')).fetchall() + q_user = _quote_table(conn, 'user') + users = conn.execute(sa.text(f'SELECT id, oauth_sub FROM {q_user} WHERE oauth_sub IS NOT NULL')).fetchall() for uid, oauth_sub in users: if oauth_sub: @@ -153,11 +159,11 @@ def upgrade() -> None: oauth_json = json.dumps({provider: {'sub': sub}}) conn.execute( - sa.text('UPDATE "user" SET oauth = :oauth WHERE id = :id'), + sa.text(f'UPDATE {q_user} SET oauth = :oauth WHERE id = :id'), {'oauth': oauth_json, 'id': uid}, ) - users_with_keys = conn.execute(sa.text('SELECT id, api_key FROM "user" WHERE api_key IS NOT NULL')).fetchall() + users_with_keys = conn.execute(sa.text(f'SELECT id, api_key FROM {q_user} WHERE api_key IS NOT NULL')).fetchall() now = int(time.time()) for uid, api_key in users_with_keys: @@ -190,7 +196,8 @@ def downgrade() -> None: op.add_column('user', sa.Column('oauth_sub', sa.Text(), nullable=True)) conn = op.get_bind() - users = conn.execute(sa.text('SELECT id, oauth FROM "user" WHERE oauth IS NOT NULL')).fetchall() + q_user = _quote_table(conn, 'user') + users = conn.execute(sa.text(f'SELECT id, oauth FROM {q_user} WHERE oauth IS NOT NULL')).fetchall() for uid, oauth in users: try: @@ -202,7 +209,7 @@ def downgrade() -> None: oauth_sub = None conn.execute( - sa.text('UPDATE "user" SET oauth_sub = :oauth_sub WHERE id = :id'), + sa.text(f'UPDATE {q_user} SET oauth_sub = :oauth_sub WHERE id = :id'), {'oauth_sub': oauth_sub, 'id': uid}, ) @@ -215,7 +222,7 @@ def downgrade() -> None: keys = conn.execute(sa.text('SELECT user_id, key FROM api_key')).fetchall() for uid, key in keys: conn.execute( - sa.text('UPDATE "user" SET api_key = :key WHERE id = :id'), + sa.text(f'UPDATE {q_user} SET api_key = :key WHERE id = :id'), {'key': key, 'id': uid}, ) diff --git a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py index c9572fe7a3..98fcc7f239 100644 --- a/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py +++ b/backend/open_webui/migrations/versions/c69f45358db4_add_folder_table.py @@ -8,6 +8,7 @@ Create Date: 2024-10-16 02:02:35.241684 from alembic import op import sqlalchemy as sa +from open_webui.migrations.util import key_text revision = 'c69f45358db4' down_revision = '3ab32c4b8f59' @@ -19,7 +20,7 @@ def upgrade(): op.create_table( 'folder', sa.Column('id', sa.Text(), nullable=False), - sa.Column('parent_id', sa.Text(), nullable=True), + sa.Column('parent_id', key_text(), nullable=True), sa.Column('user_id', sa.Text(), nullable=False), sa.Column('name', sa.Text(), nullable=False), sa.Column('items', sa.JSON(), nullable=True), @@ -38,7 +39,7 @@ def upgrade(): op.add_column( 'chat', - sa.Column('folder_id', sa.Text(), nullable=True), + sa.Column('folder_id', key_text(), nullable=True), ) diff --git a/backend/open_webui/migrations/versions/d4e5f6a7b8c9_add_automation_tables.py b/backend/open_webui/migrations/versions/d4e5f6a7b8c9_add_automation_tables.py index fc90dc417f..af4bf3a9e5 100644 --- a/backend/open_webui/migrations/versions/d4e5f6a7b8c9_add_automation_tables.py +++ b/backend/open_webui/migrations/versions/d4e5f6a7b8c9_add_automation_tables.py @@ -10,6 +10,8 @@ from typing import Union from alembic import op import sqlalchemy as sa +from open_webui.migrations.util import key_text + revision: str = 'd4e5f6a7b8c9' down_revision: Union[str, None] = 'a3dd5bedd151' branch_labels = None @@ -19,8 +21,8 @@ depends_on = None def upgrade(): op.create_table( 'automation', - sa.Column('id', sa.Text(), primary_key=True), - sa.Column('user_id', sa.Text(), nullable=False), + sa.Column('id', key_text(), primary_key=True), + sa.Column('user_id', key_text(), nullable=False), sa.Column('name', sa.Text(), nullable=False), sa.Column('data', sa.JSON(), nullable=False), sa.Column('meta', sa.JSON(), nullable=True), @@ -34,9 +36,9 @@ def upgrade(): op.create_table( 'automation_run', - sa.Column('id', sa.Text(), primary_key=True), - sa.Column('automation_id', sa.Text(), nullable=False), - sa.Column('chat_id', sa.Text(), nullable=True), + sa.Column('id', key_text(), primary_key=True), + sa.Column('automation_id', key_text(), nullable=False), + sa.Column('chat_id', key_text(), nullable=True), sa.Column('status', sa.Text(), nullable=False), sa.Column('error', sa.Text(), nullable=True), sa.Column('created_at', sa.BigInteger(), nullable=False), diff --git a/backend/open_webui/migrations/versions/f1e2d3c4b5a6_add_access_grant_table.py b/backend/open_webui/migrations/versions/f1e2d3c4b5a6_add_access_grant_table.py index 5ed572cf7a..a07a563950 100644 --- a/backend/open_webui/migrations/versions/f1e2d3c4b5a6_add_access_grant_table.py +++ b/backend/open_webui/migrations/versions/f1e2d3c4b5a6_add_access_grant_table.py @@ -18,27 +18,82 @@ import uuid from alembic import op import sqlalchemy as sa -from open_webui.migrations.util import get_existing_tables +from open_webui.migrations.util import get_existing_tables, key_text revision: str = 'f1e2d3c4b5a6' down_revision: Union[str, None] = '8452d01d26d7' branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None +""" +Column length limits used for `access_grant` only on MySQL/MariaDB. + +Why these values: +- PostgreSQL and SQLite keep the historical `Text` behavior through `key_text()`. +- MySQL/MariaDB cannot safely use unbounded `TEXT` columns in `PRIMARY KEY`, + `UNIQUE`, `INDEX`, or `FOREIGN KEY` paths. +- On `utf8mb4`, indexed `VARCHAR` columns must stay within InnoDB key-size + limits, so identifier-like fields use a conservative `191` characters. +- Enum-like fields (`resource_type`, `principal_type`, `permission`) are kept + short at `64` characters to reduce composite index width. + +Why this is safe for Open-WebUI: +- Open-WebUI identifiers are normally UUID strings (`str(uuid.uuid4())`), + which are only 36 characters long, well below the `191` limit. +- `resource_type`, `principal_type`, and `permission` store small categorical + values such as `knowledge`, `user`, `group`, `read`, and `write`, so `64` + characters provides ample headroom. + +These limits are therefore applied only for MySQL/MariaDB compatibility, +while PostgreSQL and SQLite continue using the original unrestricted text +behavior. +""" +RESOURCE_TYPE_LEN = 64 +RESOURCE_ID_LEN = 191 +PRINCIPAL_TYPE_LEN = 64 +PRINCIPAL_ID_LEN = 191 +PERMISSION_LEN = 64 + + +def _is_mysql_family(conn) -> bool: + return (conn.dialect.name or '').lower() in ('mysql', 'mariadb') + + +def _validate_access_grant_lengths(conn, resource_type, resource_id, principal_type, principal_id, permission) -> None: + # Only needed for MySQL/MariaDB, where these columns are narrowed to + # VARCHAR(...) for index/key compatibility. + if not _is_mysql_family(conn): + return + + values = [ + ('resource_type', resource_type, RESOURCE_TYPE_LEN), + ('resource_id', resource_id, RESOURCE_ID_LEN), + ('principal_type', principal_type, PRINCIPAL_TYPE_LEN), + ('principal_id', principal_id, PRINCIPAL_ID_LEN), + ('permission', permission, PERMISSION_LEN), + ] + for name, value, max_len in values: + if value is not None and len(str(value)) > max_len: + raise ValueError(f'access_grant.{name} exceeds max length {max_len}: {value!r}') + def upgrade() -> None: existing_tables = set(get_existing_tables()) # Create access_grant table if 'access_grant' not in existing_tables: + # Keep composite UNIQUE / secondary index columns tight on MySQL/MariaDB + # to avoid oversized utf8mb4 indexes. Use shorter lengths for enum-like + # fields and 191 for identifier-like fields as a conservative indexed + # string limit. op.create_table( 'access_grant', - sa.Column('id', sa.Text(), nullable=False, primary_key=True), - sa.Column('resource_type', sa.Text(), nullable=False), - sa.Column('resource_id', sa.Text(), nullable=False), - sa.Column('principal_type', sa.Text(), nullable=False), - sa.Column('principal_id', sa.Text(), nullable=False), - sa.Column('permission', sa.Text(), nullable=False), + sa.Column('id', key_text(), nullable=False, primary_key=True), + sa.Column('resource_type', key_text(length=RESOURCE_TYPE_LEN), nullable=False), + sa.Column('resource_id', key_text(length=RESOURCE_ID_LEN), nullable=False), + sa.Column('principal_type', key_text(length=PRINCIPAL_TYPE_LEN), nullable=False), + sa.Column('principal_id', key_text(length=PRINCIPAL_ID_LEN), nullable=False), + sa.Column('permission', key_text(length=PERMISSION_LEN), nullable=False), sa.Column('created_at', sa.BigInteger(), nullable=False), sa.UniqueConstraint( 'resource_type', @@ -108,6 +163,7 @@ def upgrade() -> None: key = (resource_type, resource_id, 'user', '*', 'read') if key not in inserted: + _validate_access_grant_lengths(conn, resource_type, resource_id, 'user', '*', 'read') try: conn.execute( sa.text(""" @@ -164,6 +220,7 @@ def upgrade() -> None: key = (resource_type, resource_id, 'group', group_id, permission) if key in inserted: continue + _validate_access_grant_lengths(conn, resource_type, resource_id, 'group', group_id, permission) try: conn.execute( sa.text(""" @@ -188,6 +245,7 @@ def upgrade() -> None: key = (resource_type, resource_id, 'user', user_id, permission) if key in inserted: continue + _validate_access_grant_lengths(conn, resource_type, resource_id, 'user', user_id, permission) try: conn.execute( sa.text(""" diff --git a/backend/requirements.txt b/backend/requirements.txt index a9275beaf3..a5bbd77e30 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -120,7 +120,7 @@ pgvector==0.4.2 PyMySQL==1.1.2 boto3==1.42.62 -# mariadb==1.1.14 should be added if you want to support MariaDB +mariadb==1.1.14 # required for mariadb+mariadbconnector URL support pymilvus==2.6.9 qdrant-client==1.17.0