scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-09-07 08:26:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix very slow purging of itemDataValues at startup in large databases

Browse source 
When an item is erased (removed from the trash or cleaned up from a
feed), we set a flag to purge values in `itemDataValues` on the next
startup, with this query:

DELETE FROM itemDataValues WHERE valueID NOT IN (SELECT valueID FROM itemData);

For some people, that query was incredibly slow and would result in
Zotero intermittently hanging on "Loading items…" for a long time at
startup. It's possible this is mostly limited to people who subscribe to
high-volume feeds and have a lot of item churn. One affected person had
>900K values in `itemDataValues` despite having only 20K items.

It turns out the slow query is due to the foreign-key constraint on
`itemData(valueID)` that references`itemDataValues(valueID)`. SQLite is
checking every row being deleted from `itemDataValues` against
`itemData`, even though the query is specifically removing rows that
don't exist in `itemData`! For the 900K-value DB, disabling foreign-key
checks causes the `DELETE` query to take 25 seconds instead of...some
much longer time that I didn't wait for.

We already had an `executeTransaction()` flag, `disableForeignKeys`, to
temporarily disable foreign-key checks, but it didn't do so in a way
that was safe for post-initialization usage -- a write query outside of
a transaction could've run between the transaction commit and
foreign-key checks being re-enabled. This commit changes it to properly
block all other queries unless they include an `ignoreDBLock` option,
meaning that queries within the function passed to the transaction need
to include that option. (And since that's not realistic for the couple
other uses of `disableForeignKeys` -- one for a test and one in code
that almost certainly hasn't been run by anyone in 15 years -- those now
just run `PRAGMA foreign_keys=OFF|ON` explicitly, leaving this as the
only current use.)
This commit is contained in:
Dan Stillman 2025-03-10 03:24:29 -04:00
parent 659d5e864b
commit e532e435f4
4 changed files with 151 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
chrome/content/zotero/xpcom/data/items.js
View file

@ -1653,19 +1653,19 @@ Zotero.Items = function() {
/**
* Purge unused data values
*/
this.purge = Zotero.Promise.coroutine(function* () {
Zotero.DB.requireTransaction();
this.purge = async function () {
if (!Zotero.Prefs.get('purge.items')) {
return;
}
var sql = "DELETE FROM itemDataValues WHERE valueID NOT IN "
+ "(SELECT valueID FROM itemData)";
yield Zotero.DB.queryAsync(sql);
await Zotero.DB.executeTransaction(async function () {
let sql = "DELETE FROM itemDataValues WHERE valueID NOT IN "
+ "(SELECT valueID FROM itemData)";
await Zotero.DB.queryAsync(sql, [], { ignoreDBLock: true });
}, { disableForeignKeys: true });
Zotero.Prefs.set('purge.items', false)
});
};

77
chrome/content/zotero/xpcom/db.js
View file

@ -419,12 +419,14 @@ Zotero.DBConnection.prototype.getNextName = async function (libraryID, table, fi
/**
* @param {Function} func - Async function containing `await Zotero.DB.queryAsync()` and similar
* @param {Object} [options]
* @param {Boolean} [options.disableForeignKeys] - Disable foreign key constraints before
* transaction and re-enable after. (`PRAGMA foreign_keys=0|1` is a no-op during a transaction.)
* @param {Boolean} [options.disableForeignKeys] - Disable foreign key checks before the
* transaction and re-enable after, while preventing any other queries from running.
* `queryAsync()` and similar within `func` must pass `ignoreDBLock: true` or they'll hang.
* (`PRAGMA foreign_keys=OFF|ON` is a no-op during a transaction, so it can't just be set within
* the function.)
* @return {Promise} - Promise for result of generator function
*/
Zotero.DBConnection.prototype.executeTransaction = async function (func, options) {
options = options || {};
Zotero.DBConnection.prototype.executeTransaction = async function (func, options = {}) {
var resolve;
// Set temporary options for this transaction that will be reset at the end
@ -462,34 +464,49 @@ Zotero.DBConnection.prototype.executeTransaction = async function (func, options
}
}
if (options.disableForeignKeys) {
await this.queryAsync("PRAGMA foreign_keys = 0");
let result;
let resolveDBLockPromise;
try {
let conn = this._getConnection(options) || (await this._getConnectionAsync(options));
if (func.constructor.name == 'GeneratorFunction') {
throw new Error("Zotero.DB.executeTransaction() no longer takes a generator function "
+ "-- pass an async function instead");
}
if (options.disableForeignKeys) {
this._dbLockPromise = new Promise(function () {
resolveDBLockPromise = arguments[0];
});
await this.queryAsync("PRAGMA foreign_keys=OFF", [], { ignoreDBLock: true });
}
result = await conn.executeTransaction(func);
Zotero.debug(`Committed DB transaction ${id}`, 4);
}
var conn = this._getConnection(options) || (await this._getConnectionAsync(options));
if (func.constructor.name == 'GeneratorFunction') {
throw new Error("Zotero.DB.executeTransaction() no longer takes a generator function "
+ "-- pass an async function instead");
finally {
if (options.disableForeignKeys) {
await this.queryAsync("PRAGMA foreign_keys=ON", [], { ignoreDBLock: true });
if (resolveDBLockPromise) {
resolveDBLockPromise();
this._dbLockPromise = undefined;
}
}
}
var result = await conn.executeTransaction(func);
Zotero.debug(`Committed DB transaction ${id}`, 4);
// Clear transaction time
if (this._transactionDate) {
this._transactionDate = null;
}
this._transactionID = null;
if (options.vacuumOnCommit) {
Zotero.debug('Vacuuming database');
await this.queryAsync('VACUUM');
Zotero.debug('Done vacuuming');
}
this._transactionID = null;
// Function to run once transaction has been committed but before any
// permanent callbacks
if (options.onCommit) {
@ -546,10 +563,6 @@ Zotero.DBConnection.prototype.executeTransaction = async function (func, options
throw e;
}
finally {
if (options.disableForeignKeys) {
await this.queryAsync("PRAGMA foreign_keys = 1");
}
// Reset options back to their previous values
if (options) {
for (let option in options) {
@ -597,13 +610,19 @@ Zotero.DBConnection.prototype.requireTransaction = function () {
* rows are Proxy objects that return values from the
* underlying mozIStorageRows based on column names.
*/
Zotero.DBConnection.prototype.queryAsync = async function (sql, params, options) {
Zotero.DBConnection.prototype.queryAsync = async function (sql, params, options = {}) {
try {
let onRow = null;
let conn = this._getConnection(options) || (await this._getConnectionAsync(options));
if (!options || !options.noParseParams) {
[sql, params] = this.parseQueryAndParams(sql, params);
}
if (this._dbLockPromise && !options.ignoreDBLock) {
Zotero.debug(`Waiting for DB lock to be released: ${sql}`, 2);
await this._dbLockPromise;
}
if (Zotero.Debug.enabled) {
this.logQuery(sql, params, options);
}
@ -719,6 +738,12 @@ Zotero.DBConnection.prototype.valueQueryAsync = async function (sql, params, opt
try {
let conn = this._getConnection(options) || (await this._getConnectionAsync(options));
[sql, params] = this.parseQueryAndParams(sql, params);
if (this._dbLockPromise && !options.ignoreDBLock) {
Zotero.debug(`Waiting for DB lock to be released: ${sql}`, 2);
await this._dbLockPromise;
}
if (Zotero.Debug.enabled) {
this.logQuery(sql, params, options);
}
@ -766,6 +791,12 @@ Zotero.DBConnection.prototype.columnQueryAsync = async function (sql, params, op
try {
let conn = this._getConnection(options) || (await this._getConnectionAsync(options));
[sql, params] = this.parseQueryAndParams(sql, params);
if (this._dbLockPromise && !options.ignoreDBLock) {
Zotero.debug(`Waiting for DB lock to be released: ${sql}`, 2);
await this._dbLockPromise;
}
if (Zotero.Debug.enabled) {
this.logQuery(sql, params, options);
}

160
chrome/content/zotero/xpcom/schema.js
View file

@ -600,9 +600,15 @@ Zotero.Schema = new function(){
this._updateGlobalSchemaForTest = async function (schema) {
await Zotero.DB.executeTransaction(async function () {
await _updateGlobalSchema(schema);
}.bind(this), { disableForeignKeys: true });
await Zotero.DB.queryAsync("PRAGMA foreign_keys=OFF");
try {
await Zotero.DB.executeTransaction(async function () {
await _updateGlobalSchema(schema);
});
}
finally {
await Zotero.DB.queryAsync("PRAGMA foreign_keys=ON");
}
};
@ -733,58 +739,64 @@ Zotero.Schema = new function(){
var itemTypeID = Zotero.ID.get('customItemTypes');
yield Zotero.DB.executeTransaction(async function () {
await Zotero.DB.queryAsync("INSERT INTO customItemTypes VALUES (?, 'nsfReviewer', 'NSF Reviewer', 1, 'chrome://zotero/skin/report_user.png')", itemTypeID);
var fields = [
['name', 'Name'],
['institution', 'Institution'],
['address', 'Address'],
['telephone', 'Telephone'],
['email', 'Email'],
['homepage', 'Webpage'],
['discipline', 'Discipline'],
['nsfID', 'NSF ID'],
['dateSent', 'Date Sent'],
['dateDue', 'Date Due'],
['accepted', 'Accepted'],
['programDirector', 'Program Director']
];
for (var i=0; i<fields.length; i++) {
var fieldID = Zotero.ItemFields.getID(fields[i][0]);
if (!fieldID) {
var fieldID = Zotero.ID.get('customFields');
await Zotero.DB.queryAsync("INSERT INTO customFields VALUES (?, ?, ?)", [fieldID, fields[i][0], fields[i][1]]);
await Zotero.DB.queryAsync("INSERT INTO customItemTypeFields VALUES (?, NULL, ?, 1, ?)", [itemTypeID, fieldID, i+1]);
}
else {
await Zotero.DB.queryAsync("INSERT INTO customItemTypeFields VALUES (?, ?, NULL, 1, ?)", [itemTypeID, fieldID, i+1]);
yield Zotero.DB.queryAsync("PRAGMA foreign_keys=OFF");
try {
yield Zotero.DB.executeTransaction(async function () {
await Zotero.DB.queryAsync("INSERT INTO customItemTypes VALUES (?, 'nsfReviewer', 'NSF Reviewer', 1, 'chrome://zotero/skin/report_user.png')", itemTypeID);
var fields = [
['name', 'Name'],
['institution', 'Institution'],
['address', 'Address'],
['telephone', 'Telephone'],
['email', 'Email'],
['homepage', 'Webpage'],
['discipline', 'Discipline'],
['nsfID', 'NSF ID'],
['dateSent', 'Date Sent'],
['dateDue', 'Date Due'],
['accepted', 'Accepted'],
['programDirector', 'Program Director']
];
for (var i=0; i<fields.length; i++) {
var fieldID = Zotero.ItemFields.getID(fields[i][0]);
if (!fieldID) {
var fieldID = Zotero.ID.get('customFields');
await Zotero.DB.queryAsync("INSERT INTO customFields VALUES (?, ?, ?)", [fieldID, fields[i][0], fields[i][1]]);
await Zotero.DB.queryAsync("INSERT INTO customItemTypeFields VALUES (?, NULL, ?, 1, ?)", [itemTypeID, fieldID, i+1]);
}
else {
await Zotero.DB.queryAsync("INSERT INTO customItemTypeFields VALUES (?, ?, NULL, 1, ?)", [itemTypeID, fieldID, i+1]);
}
switch (fields[i][0]) {
case 'name':
var baseFieldID = Zotero.ItemFields.getID('title');
break;
case 'dateSent':
var baseFieldID = Zotero.ItemFields.getID('date');
break;
case 'homepage':
var baseFieldID = Zotero.ItemFields.getID('url');
break;
default:
var baseFieldID = null;
}
if (baseFieldID) {
await Zotero.DB.queryAsync("INSERT INTO customBaseFieldMappings VALUES (?, ?, ?)", [itemTypeID, baseFieldID, fieldID]);
}
}
switch (fields[i][0]) {
case 'name':
var baseFieldID = Zotero.ItemFields.getID('title');
break;
case 'dateSent':
var baseFieldID = Zotero.ItemFields.getID('date');
break;
case 'homepage':
var baseFieldID = Zotero.ItemFields.getID('url');
break;
default:
var baseFieldID = null;
}
if (baseFieldID) {
await Zotero.DB.queryAsync("INSERT INTO customBaseFieldMappings VALUES (?, ?, ?)", [itemTypeID, baseFieldID, fieldID]);
}
}
await _reloadSchema();
}, { disableForeignKeys: true });
await _reloadSchema();
});
}
finally {
yield Zotero.DB.queryAsync("PRAGMA foreign_keys=ON");
}
var s = new Zotero.Search;
s.name = "Overdue NSF Reviewers";
@ -814,26 +826,32 @@ Zotero.Schema = new function(){
}
Zotero.debug("Uninstalling nsfReviewer item type");
yield Zotero.DB.executeTransaction(async function () {
await Zotero.DB.queryAsync("DELETE FROM customItemTypeFields WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
await Zotero.DB.queryAsync("DELETE FROM customBaseFieldMappings WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
var fields = Zotero.ItemFields.getItemTypeFields(itemTypeID);
for (let fieldID of fields) {
if (Zotero.ItemFields.isCustom(fieldID)) {
await Zotero.DB.queryAsync("DELETE FROM customFields WHERE customFieldID=?", fieldID - Zotero.ItemTypes.customIDOffset);
yield Zotero.DB.queryAsync("PRAGMA foreign_keys=OFF");
try {
yield Zotero.DB.executeTransaction(async function () {
await Zotero.DB.queryAsync("DELETE FROM customItemTypeFields WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
await Zotero.DB.queryAsync("DELETE FROM customBaseFieldMappings WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
var fields = Zotero.ItemFields.getItemTypeFields(itemTypeID);
for (let fieldID of fields) {
if (Zotero.ItemFields.isCustom(fieldID)) {
await Zotero.DB.queryAsync("DELETE FROM customFields WHERE customFieldID=?", fieldID - Zotero.ItemTypes.customIDOffset);
}
}
}
await Zotero.DB.queryAsync("DELETE FROM customItemTypes WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
var searches = Zotero.Searches.getByLibrary(Zotero.Libraries.userLibraryID);
for (let search of searches) {
if (search.name == 'Overdue NSF Reviewers') {
await search.erase();
await Zotero.DB.queryAsync("DELETE FROM customItemTypes WHERE customItemTypeID=?", itemTypeID - Zotero.ItemTypes.customIDOffset);
var searches = Zotero.Searches.getByLibrary(Zotero.Libraries.userLibraryID);
for (let search of searches) {
if (search.name == 'Overdue NSF Reviewers') {
await search.erase();
}
}
}
await _reloadSchema();
}.bind(this), { disableForeignKeys: true });
await _reloadSchema();
});
}
finally {
yield Zotero.DB.queryAsync("PRAGMA foreign_keys=ON");
}
ps.alert(null, "Zotero Item Type Removed", "The 'NSF Reviewer' item type has been uninstalled.");
}

4
chrome/content/zotero/xpcom/zotero.js
View file

@ -1754,9 +1754,7 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js");
return Zotero.Tags.purge();
});
yield Zotero.Fulltext.purgeUnusedWords();
yield Zotero.DB.executeTransaction(async function () {
return Zotero.Items.purge();
});
yield Zotero.Items.purge();
// DEBUG: this might not need to be permanent
//yield Zotero.DB.executeTransaction(async function () {
// return Zotero.Relations.purge();