diff --git a/chrome/content/zotero/xpcom/db.js b/chrome/content/zotero/xpcom/db.js index 5914a34d65..417fdbc09d 100644 --- a/chrome/content/zotero/xpcom/db.js +++ b/chrome/content/zotero/xpcom/db.js @@ -424,6 +424,9 @@ Zotero.DBConnection.prototype.getNextName = Zotero.Promise.coroutine(function* ( /** * @param {Function} func - Generator function that yields promises, * generally from queryAsync() and similar + * @param {Object} [options] + * @param {Boolean} [options.disableForeignKeys] - Disable foreign key constraints before + * transaction and re-enable after. (`PRAGMA foreign_keys=0|1` is a no-op during a transaction.) * @return {Promise} - Promise for result of generator function */ Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(function* (func, options) { @@ -464,6 +467,11 @@ Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(func this._callbacks.begin[i](id); } } + + if (options.disableForeignKeys) { + yield this.queryAsync("PRAGMA foreign_keys = 0"); + } + var conn = this._getConnection(options) || (yield this._getConnectionAsync(options)); var result = yield conn.executeTransaction(func); Zotero.debug(`Committed DB transaction ${id}`, 4); @@ -538,6 +546,10 @@ Zotero.DBConnection.prototype.executeTransaction = Zotero.Promise.coroutine(func throw e; } finally { + if (options.disableForeignKeys) { + yield this.queryAsync("PRAGMA foreign_keys = 1"); + } + // Reset options back to their previous values if (options) { for (let option in options) { diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js index 79882c9208..29f947f70c 100644 --- a/chrome/content/zotero/xpcom/schema.js +++ b/chrome/content/zotero/xpcom/schema.js @@ -139,7 +139,7 @@ Zotero.Schema = new function(){ } // Check if DB is coming from the DB Repair Tool and should be checked - var integrityCheck = await this.integrityCheckRequired(); + var integrityCheckRequired = await this.integrityCheckRequired(); // Check whether bundled global schema file is newer than DB var bundledGlobalSchema = await _readGlobalSchemaFromFile(); @@ -156,7 +156,7 @@ Zotero.Schema = new function(){ await Zotero.DB.backupDatabase(userdata, true); } // Automatic backup - else if (integrityCheck || bundledGlobalSchemaVersionCompare === 1) { + else if (integrityCheckRequired || bundledGlobalSchemaVersionCompare === 1) { await Zotero.DB.backupDatabase(false, true); } @@ -169,6 +169,42 @@ Zotero.Schema = new function(){ var updated; await Zotero.DB.queryAsync("PRAGMA foreign_keys = false"); try { + updated = await Zotero.DB.executeTransaction(async function (conn) { + var updated = await _updateSchema('system'); + + // Update custom tables if they exist so that changes are in + // place before user data migration + if (Zotero.DB.tableExists('customItemTypes')) { + await _updateCustomTables(); + } + + // Auto-repair databases flagged for repair or coming from the DB Repair Tool + // + // If we need to run migration steps, skip the check until after the update, since + // the integrity check is expecting to run on the current data model. + var integrityCheckDone = false; + var toVersion = await _getSchemaSQLVersion('userdata'); + if (integrityCheckRequired && userdata >= toVersion) { + await this.integrityCheck(true); + integrityCheckDone = true; + } + + updated = await _migrateUserDataSchema(userdata, options); + await _updateSchema('triggers'); + + // Populate combined tables for custom types and fields -- this is likely temporary + // + // We do this again in case custom fields were changed during user data migration + await _updateCustomTables(); + + // If we updated the DB, also do an integrity check for good measure + if (updated && !integrityCheckDone) { + await this.integrityCheck(true); + } + + return updated; + }.bind(this)); + // If bundled global schema file is newer than DB, apply it if (bundledGlobalSchemaVersionCompare === 1) { await Zotero.DB.executeTransaction(async function () { @@ -188,35 +224,6 @@ Zotero.Schema = new function(){ } await _loadGlobalSchema(data, bundledGlobalSchema.version); } - - updated = await Zotero.DB.executeTransaction(async function (conn) { - var updated = await _updateSchema('system'); - - // Update custom tables if they exist so that changes are in - // place before user data migration - if (Zotero.DB.tableExists('customItemTypes')) { - await _updateCustomTables(); - } - - // Auto-repair databases flagged for repair or coming from the DB Repair Tool - if (integrityCheck) { - // If we need to run migration steps, don't reconcile tables, since it might - // create tables that aren't expected to exist yet - let toVersion = await _getSchemaSQLVersion('userdata'); - await this.integrityCheck(true, { skipReconcile: userdata < toVersion }); - options.skipIntegrityCheck = true; - } - - updated = await _migrateUserDataSchema(userdata, options); - await _updateSchema('triggers'); - - // Populate combined tables for custom types and fields -- this is likely temporary - // - // We do this again in case custom fields were changed during user data migration - await _updateCustomTables(); - - return updated; - }.bind(this)); } finally { await Zotero.DB.queryAsync("PRAGMA foreign_keys = true"); @@ -410,7 +417,7 @@ Zotero.Schema = new function(){ /** * Update the item-type/field/creator mapping tables based on the passed schema */ - async function _updateGlobalSchema(data) { + async function _updateGlobalSchema(data, options) { Zotero.debug("Updating global schema to version " + data.version); Zotero.DB.requireTransaction(); @@ -571,7 +578,7 @@ Zotero.Schema = new function(){ var bundledVersion = (await _readGlobalSchemaFromFile()).version; await _loadGlobalSchema(data, bundledVersion); - await _reloadSchema(); + await _reloadSchema(options); // Mark that we need to migrate Extra values to any newly available fields in // Zotero.Schema.migrateExtraFields() await Zotero.DB.queryAsync( @@ -585,7 +592,7 @@ Zotero.Schema = new function(){ this._updateGlobalSchemaForTest = async function (schema) { await Zotero.DB.executeTransaction(async function () { await _updateGlobalSchema(schema); - }.bind(this)); + }.bind(this), { disableForeignKeys: true }); }; @@ -768,7 +775,7 @@ Zotero.Schema = new function(){ } yield _reloadSchema(); - }); + }, { disableForeignKeys: true }); var s = new Zotero.Search; s.name = "Overdue NSF Reviewers"; @@ -817,39 +824,44 @@ Zotero.Schema = new function(){ } yield _reloadSchema(); - }.bind(this)); + }.bind(this), { disableForeignKeys: true }); ps.alert(null, "Zotero Item Type Removed", "The 'NSF Reviewer' item type has been uninstalled."); } }); - var _reloadSchema = Zotero.Promise.coroutine(function* () { - yield _updateCustomTables(); - yield Zotero.ItemTypes.init(); - yield Zotero.ItemFields.init(); - yield Zotero.CreatorTypes.init(); - yield Zotero.SearchConditions.init(); + async function _reloadSchema(options) { + await _updateCustomTables(options); + await Zotero.ItemTypes.init(); + await Zotero.ItemFields.init(); + await Zotero.CreatorTypes.init(); + await Zotero.SearchConditions.init(); // Update item type menus in every open window Zotero.Schema.schemaUpdatePromise.then(function () { - var wm = Services.wm; - var enumerator = wm.getEnumerator("navigator:browser"); + var enumerator = Services.wm.getEnumerator("navigator:browser"); while (enumerator.hasMoreElements()) { let win = enumerator.getNext(); win.ZoteroPane.buildItemTypeSubMenu(); win.document.getElementById('zotero-editpane-item-box').buildItemTypeMenu(); } }); - }); + } - var _updateCustomTables = async function () { + var _updateCustomTables = async function (options) { Zotero.debug("Updating custom tables"); Zotero.DB.requireTransaction(); + if (!options?.foreignKeyChecksAllowed) { + if (await Zotero.DB.valueQueryAsync("PRAGMA foreign_keys")) { + throw new Error("Foreign key checks must be disabled before updating custom tables"); + } + } + await Zotero.DB.queryAsync("DELETE FROM itemTypesCombined"); - await Zotero.DB.queryAsync("DELETE FROM fieldsCombined WHERE fieldID NOT IN (SELECT fieldID FROM itemData)"); + await Zotero.DB.queryAsync("DELETE FROM fieldsCombined"); await Zotero.DB.queryAsync("DELETE FROM itemTypeFieldsCombined"); await Zotero.DB.queryAsync("DELETE FROM baseFieldMappingsCombined"); @@ -860,7 +872,7 @@ Zotero.Schema = new function(){ + "SELECT customItemTypeID + " + offset + " AS itemTypeID, typeName, display, 1 AS custom FROM customItemTypes" ); await Zotero.DB.queryAsync( - "INSERT OR IGNORE INTO fieldsCombined " + "INSERT INTO fieldsCombined " + "SELECT fieldID, fieldName, NULL AS label, fieldFormatID, 0 AS custom FROM fields UNION " + "SELECT customFieldID + " + offset + " AS fieldID, fieldName, label, NULL, 1 AS custom FROM customFields" ); @@ -1776,7 +1788,7 @@ Zotero.Schema = new function(){ * deleted */ this.integrityCheck = Zotero.Promise.coroutine(function* (fix, options = {}) { - Zotero.debug("Checking database integrity"); + Zotero.debug("Checking database schema integrity"); // Just as a sanity check, make sure combined field tables are populated, // so that we don't try to wipe out all data @@ -2205,7 +2217,7 @@ Zotero.Schema = new function(){ }); var schema = yield _readGlobalSchemaFromFile(); - yield _updateGlobalSchema(schema); + yield _updateGlobalSchema(schema, { foreignKeyChecksAllowed: true }); yield _getSchemaSQLVersion('system').then(function (version) { return _updateDBVersion('system', version); @@ -2567,11 +2579,6 @@ Zotero.Schema = new function(){ return false; } - if (!options.skipIntegrityCheck) { - // Check integrity, but don't create missing tables - yield Zotero.Schema.integrityCheck(true, { skipReconcile: true }); - } - Zotero.debug('Updating user data tables from version ' + fromVersion + ' to ' + toVersion); if (options.onBeforeUpdate) { @@ -3282,6 +3289,20 @@ Zotero.Schema = new function(){ yield Zotero.DB.queryAsync("UPDATE itemAnnotations SET color='#000000' WHERE color='#000'"); } + else if (i == 117) { + let versionFieldID = yield Zotero.DB.valueQueryAsync("SELECT fieldID FROM fields WHERE fieldName='version'"); + if (versionFieldID) { + let versionNumberFieldID = yield Zotero.DB.valueQueryAsync("SELECT fieldID FROM fields WHERE fieldName='versionNumber'"); + if (versionNumberFieldID) { + yield Zotero.DB.queryAsync("UPDATE itemData SET fieldID=? WHERE fieldID=?", [versionNumberFieldID, versionFieldID]); + yield Zotero.DB.queryAsync("DELETE FROM fields WHERE fieldID=?", versionFieldID); + } + else { + yield Zotero.DB.queryAsync("UPDATE fields SET fieldName=? WHERE fieldName=?", ['versionNumber', 'version']); + } + } + } + // If breaking compatibility or doing anything dangerous, clear minorUpdateFrom } diff --git a/resource/schema/global b/resource/schema/global index b5b3f51217..97e0a8efa2 160000 --- a/resource/schema/global +++ b/resource/schema/global @@ -1 +1 @@ -Subproject commit b5b3f51217a99b3c41585d468ee1dc0837233d13 +Subproject commit 97e0a8efa2cb2cf6c9853ceca334ec56180a9df0 diff --git a/resource/schema/system.sql b/resource/schema/system.sql index d31ecc2399..1b892f2401 100644 --- a/resource/schema/system.sql +++ b/resource/schema/system.sql @@ -23,25 +23,6 @@ -- This file creates system tables that can be safely wiped and reinitialized -- at any time, as long as existing ids are preserved. --- Valid item types ("book," "journalArticle," etc.) -DROP TABLE IF EXISTS itemTypes; -CREATE TABLE itemTypes ( - itemTypeID INTEGER PRIMARY KEY, - typeName TEXT, - templateItemTypeID INT, - display INT DEFAULT 1 -- 0 == hide, 1 == display, 2 == primary -); - --- Populated at startup from itemTypes and customItemTypes -DROP TABLE IF EXISTS itemTypesCombined; -CREATE TABLE itemTypesCombined ( - itemTypeID INT NOT NULL, - typeName TEXT NOT NULL, - display INT DEFAULT 1 NOT NULL, - custom INT NOT NULL, - PRIMARY KEY (itemTypeID) -); - -- Describes various types of fields and their format restrictions, -- and indicates whether data should be stored as strings or integers -- @@ -53,77 +34,6 @@ CREATE TABLE fieldFormats ( isInteger INT ); --- Field types for item metadata -DROP TABLE IF EXISTS fields; -CREATE TABLE fields ( - fieldID INTEGER PRIMARY KEY, - fieldName TEXT, - fieldFormatID INT, - FOREIGN KEY (fieldFormatID) REFERENCES fieldFormats(fieldFormatID) -); - --- Populated at startup from fields and customFields -DROP TABLE IF EXISTS fieldsCombined; -CREATE TABLE fieldsCombined ( - fieldID INT NOT NULL, - fieldName TEXT NOT NULL, - label TEXT, - fieldFormatID INT, - custom INT NOT NULL, - PRIMARY KEY (fieldID) -); - --- Defines valid fields for each itemType, their display order, and their default visibility -DROP TABLE IF EXISTS itemTypeFields; -CREATE TABLE itemTypeFields ( - itemTypeID INT, - fieldID INT, - hide INT, - orderIndex INT, - PRIMARY KEY (itemTypeID, orderIndex), - UNIQUE (itemTypeID, fieldID), - FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), - FOREIGN KEY (fieldID) REFERENCES fields(fieldID) -); -CREATE INDEX itemTypeFields_fieldID ON itemTypeFields(fieldID); - --- Populated at startup from itemTypeFields and customItemTypeFields -DROP TABLE IF EXISTS itemTypeFieldsCombined; -CREATE TABLE itemTypeFieldsCombined ( - itemTypeID INT NOT NULL, - fieldID INT NOT NULL, - hide INT, - orderIndex INT NOT NULL, - PRIMARY KEY (itemTypeID, orderIndex), - UNIQUE (itemTypeID, fieldID) -); -CREATE INDEX itemTypeFieldsCombined_fieldID ON itemTypeFieldsCombined(fieldID); - --- Maps base fields to type-specific fields (e.g. publisher to label in audioRecording) -DROP TABLE IF EXISTS baseFieldMappings; -CREATE TABLE baseFieldMappings ( - itemTypeID INT, - baseFieldID INT, - fieldID INT, - PRIMARY KEY (itemTypeID, baseFieldID, fieldID), - FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), - FOREIGN KEY (baseFieldID) REFERENCES fields(fieldID), - FOREIGN KEY (fieldID) REFERENCES fields(fieldID) -); -CREATE INDEX baseFieldMappings_baseFieldID ON baseFieldMappings(baseFieldID); -CREATE INDEX baseFieldMappings_fieldID ON baseFieldMappings(fieldID); - --- Populated at startup from baseFieldMappings and customBaseFieldMappings -DROP TABLE IF EXISTS baseFieldMappingsCombined; -CREATE TABLE baseFieldMappingsCombined ( - itemTypeID INT, - baseFieldID INT, - fieldID INT, - PRIMARY KEY (itemTypeID, baseFieldID, fieldID) -); -CREATE INDEX baseFieldMappingsCombined_baseFieldID ON baseFieldMappingsCombined(baseFieldID); -CREATE INDEX baseFieldMappingsCombined_fieldID ON baseFieldMappingsCombined(fieldID); - DROP TABLE IF EXISTS charsets; CREATE TABLE charsets ( charsetID INTEGER PRIMARY KEY, @@ -147,24 +57,6 @@ CREATE TABLE fileTypeMimeTypes ( ); CREATE INDEX fileTypeMimeTypes_mimeType ON fileTypeMimeTypes(mimeType); --- Defines the possible creator types (contributor, editor, author) -DROP TABLE IF EXISTS creatorTypes; -CREATE TABLE creatorTypes ( - creatorTypeID INTEGER PRIMARY KEY, - creatorType TEXT -); - -DROP TABLE IF EXISTS itemTypeCreatorTypes; -CREATE TABLE itemTypeCreatorTypes ( - itemTypeID INT, - creatorTypeID INT, - primaryField INT, - PRIMARY KEY (itemTypeID, creatorTypeID), - FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), - FOREIGN KEY (creatorTypeID) REFERENCES creatorTypes(creatorTypeID) -); -CREATE INDEX itemTypeCreatorTypes_creatorTypeID ON itemTypeCreatorTypes(creatorTypeID); - DROP TABLE IF EXISTS syncObjectTypes; CREATE TABLE syncObjectTypes ( syncObjectTypeID INTEGER PRIMARY KEY, diff --git a/resource/schema/userdata.sql b/resource/schema/userdata.sql index df535c1a60..794fdddb86 100644 --- a/resource/schema/userdata.sql +++ b/resource/schema/userdata.sql @@ -1,4 +1,4 @@ --- 116 +-- 117 -- Copyright (c) 2009 Center for History and New Media -- George Mason University, Fairfax, Virginia, USA @@ -23,6 +23,111 @@ -- This file creates tables containing user-specific data for new users -- -- any changes made here must be mirrored in transition steps in schema.js::_migrateSchema() +-- +-- Tables populated by global schema +-- + +-- Valid item types ("book," "journalArticle," etc.) +CREATE TABLE itemTypes ( + itemTypeID INTEGER PRIMARY KEY, + typeName TEXT, + templateItemTypeID INT, + display INT DEFAULT 1 -- 0 == hide, 1 == display, 2 == primary +); + +-- Populated at startup from itemTypes and customItemTypes +CREATE TABLE itemTypesCombined ( + itemTypeID INT NOT NULL, + typeName TEXT NOT NULL, + display INT DEFAULT 1 NOT NULL, + custom INT NOT NULL, + PRIMARY KEY (itemTypeID) +); + +-- Field types for item metadata +CREATE TABLE fields ( + fieldID INTEGER PRIMARY KEY, + fieldName TEXT, + fieldFormatID INT, + FOREIGN KEY (fieldFormatID) REFERENCES fieldFormats(fieldFormatID) +); + +-- Populated at startup from fields and customFields +CREATE TABLE fieldsCombined ( + fieldID INT NOT NULL, + fieldName TEXT NOT NULL, + label TEXT, + fieldFormatID INT, + custom INT NOT NULL, + PRIMARY KEY (fieldID) +); + +-- Defines valid fields for each itemType, their display order, and their default visibility +CREATE TABLE itemTypeFields ( + itemTypeID INT, + fieldID INT, + hide INT, + orderIndex INT, + PRIMARY KEY (itemTypeID, orderIndex), + UNIQUE (itemTypeID, fieldID), + FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), + FOREIGN KEY (fieldID) REFERENCES fields(fieldID) +); +CREATE INDEX itemTypeFields_fieldID ON itemTypeFields(fieldID); + +-- Populated at startup from itemTypeFields and customItemTypeFields +CREATE TABLE itemTypeFieldsCombined ( + itemTypeID INT NOT NULL, + fieldID INT NOT NULL, + hide INT, + orderIndex INT NOT NULL, + PRIMARY KEY (itemTypeID, orderIndex), + UNIQUE (itemTypeID, fieldID) +); +CREATE INDEX itemTypeFieldsCombined_fieldID ON itemTypeFieldsCombined(fieldID); + +-- Maps base fields to type-specific fields (e.g. publisher to label in audioRecording) +CREATE TABLE baseFieldMappings ( + itemTypeID INT, + baseFieldID INT, + fieldID INT, + PRIMARY KEY (itemTypeID, baseFieldID, fieldID), + FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), + FOREIGN KEY (baseFieldID) REFERENCES fields(fieldID), + FOREIGN KEY (fieldID) REFERENCES fields(fieldID) +); +CREATE INDEX baseFieldMappings_baseFieldID ON baseFieldMappings(baseFieldID); +CREATE INDEX baseFieldMappings_fieldID ON baseFieldMappings(fieldID); + +-- Populated at startup from baseFieldMappings and customBaseFieldMappings +CREATE TABLE baseFieldMappingsCombined ( + itemTypeID INT, + baseFieldID INT, + fieldID INT, + PRIMARY KEY (itemTypeID, baseFieldID, fieldID) +); +CREATE INDEX baseFieldMappingsCombined_baseFieldID ON baseFieldMappingsCombined(baseFieldID); +CREATE INDEX baseFieldMappingsCombined_fieldID ON baseFieldMappingsCombined(fieldID); + +-- Defines the possible creator types (contributor, editor, author) +CREATE TABLE creatorTypes ( + creatorTypeID INTEGER PRIMARY KEY, + creatorType TEXT +); + +CREATE TABLE itemTypeCreatorTypes ( + itemTypeID INT, + creatorTypeID INT, + primaryField INT, + PRIMARY KEY (itemTypeID, creatorTypeID), + FOREIGN KEY (itemTypeID) REFERENCES itemTypes(itemTypeID), + FOREIGN KEY (creatorTypeID) REFERENCES creatorTypes(creatorTypeID) +); +CREATE INDEX itemTypeCreatorTypes_creatorTypeID ON itemTypeCreatorTypes(creatorTypeID); + +-- +-- End of tables populated by global schema +-- CREATE TABLE version ( schema TEXT PRIMARY KEY,