From 4b60c6ca275ec69249b3ea81a669ae4823678d03 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 16 May 2019 04:56:46 -0400 Subject: [PATCH] Type/field handling overhaul This changes the way item types, item fields, creator types, and CSL mappings are defined and handled, in preparation for updated types and fields. Instead of being predefined in SQL files or code, type/field info is read from a bundled JSON file shared with other parts of the Zotero ecosystem [1], referred to as the "global schema". Updates to the bundled schema file are automatically applied to the database at first run, allowing changes to be made consistently across apps. When syncing, invalid JSON properties are now rejected instead of being ignored and processed later, which will allow for schema changes to be made without causing problems in existing clients. We considered many alternative approaches, but this approach is by far the simplest, safest, and most transparent to the user. For now, there are no actual changes to types and fields, since we'll first need to do a sync cut-off for earlier versions that don't reject invalid properties. For third-party code, the main change is that type and field IDs should no longer be hard-coded, since they may not be consistent in new installs. For example, code should use `Zotero.ItemTypes.getID('note')` instead of hard-coding `1`. [1] https://github.com/zotero/zotero-schema --- .gitmodules | 3 + chrome/content/zotero/bindings/itembox.xml | 12 +- .../content/zotero/bindings/zoterosearch.xml | 2 +- .../content/zotero/xpcom/data/cachedTypes.js | 6 +- .../content/zotero/xpcom/data/collection.js | 20 +- chrome/content/zotero/xpcom/data/creators.js | 11 +- .../zotero/xpcom/data/dataObjectUtilities.js | 7 +- chrome/content/zotero/xpcom/data/item.js | 191 ++-- .../content/zotero/xpcom/data/itemFields.js | 96 +- chrome/content/zotero/xpcom/data/items.js | 9 +- chrome/content/zotero/xpcom/data/search.js | 27 +- .../zotero/xpcom/data/searchConditions.js | 4 +- chrome/content/zotero/xpcom/db.js | 2 +- chrome/content/zotero/xpcom/duplicates.js | 7 +- chrome/content/zotero/xpcom/id.js | 5 +- chrome/content/zotero/xpcom/itemTreeView.js | 7 +- chrome/content/zotero/xpcom/report.js | 2 +- chrome/content/zotero/xpcom/retractions.js | 6 +- chrome/content/zotero/xpcom/schema.js | 526 ++++++++-- .../content/zotero/xpcom/sync/syncEngine.js | 2 +- chrome/content/zotero/xpcom/sync/syncLocal.js | 14 +- .../content/zotero/xpcom/sync/syncRunner.js | 17 + chrome/content/zotero/xpcom/utilities.js | 197 +--- .../zotero/xpcom/utilities_internal.js | 229 ++++- chrome/content/zotero/xpcom/zotero.js | 9 +- chrome/locale/en-US/zotero/zotero.properties | 2 + resource/pako.js | 1 + resource/schema/global | 1 + resource/schema/system.sql | 960 ------------------ resource/schema/triggers.sql | 101 -- resource/schema/userdata.sql | 2 +- scripts/config.js | 6 +- test/tests/collectionTest.js | 23 + test/tests/data/itemJSON.js | 2 +- test/tests/data/translatorExport.js | 2 +- test/tests/data/translatorExportLegacy.js | 4 +- test/tests/dataObjectUtilitiesTest.js | 9 + test/tests/itemTest.js | 210 +++- test/tests/schemaTest.js | 53 + test/tests/searchTest.js | 35 + test/tests/syncEngineTest.js | 15 +- test/tests/utilities_internalTest.js | 138 ++- 42 files changed, 1470 insertions(+), 1505 deletions(-) create mode 120000 resource/pako.js create mode 160000 resource/schema/global diff --git a/.gitmodules b/.gitmodules index ad421b521c..2101837ba8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ path = test/resource/chai-as-promised url = https://github.com/domenic/chai-as-promised.git branch = master +[submodule "resource/schema/global"] + path = resource/schema/global + url = git://github.com/zotero/zotero-schema.git diff --git a/chrome/content/zotero/bindings/itembox.xml b/chrome/content/zotero/bindings/itembox.xml index 62eddf215a..9c08a8b8a6 100644 --- a/chrome/content/zotero/bindings/itembox.xml +++ b/chrome/content/zotero/bindings/itembox.xml @@ -387,9 +387,7 @@ if (fieldIsClickable && !Zotero.Items.isPrimaryField(fieldName) - && (Zotero.ItemFields.isFieldOfBase(Zotero.ItemFields.getID(fieldName), 'date') - // TEMP - filingDate - || fieldName == 'filingDate') + && Zotero.ItemFields.isDate(fieldName) // TEMP - NSF && fieldName != 'dateSent') { this.addDateRow(fieldNames[i], this.item.getField(fieldName, true), tabindex); @@ -414,7 +412,7 @@ if (fieldName) { label.setAttribute("value", prefix + - Zotero.ItemFields.getLocalizedString(this.item.itemTypeID, fieldName)); + Zotero.ItemFields.getLocalizedString(fieldName)); } // TEMP - NSF (homepage) @@ -897,7 +895,7 @@ 'name' or 'firstName'/'lastName', or 'firstName'/'lastName'/'fieldMode' *
  • 'creatorType' (can be name or id) or 'creatorTypeID'
    • * + * @param {Object} [options] + * @param {Boolean} [options.strict] - Throw on invalid creator type */ -Zotero.Item.prototype.setCreator = function (orderIndex, data) { +Zotero.Item.prototype.setCreator = function (orderIndex, data, options = {}) { var itemTypeID = this._itemTypeID; if (!itemTypeID) { throw new Error('Item type must be set before setting creators'); @@ -1047,7 +1033,8 @@ Zotero.Item.prototype.setCreator = function (orderIndex, data) { this._requireData('creators'); - data = Zotero.Creators.cleanData(data); + var origCreatorType = data.creatorType; + data = Zotero.Creators.cleanData(data, options); if (data.creatorTypeID === undefined) { throw new Error("Creator data must include a valid 'creatorType' or 'creatorTypeID' property"); @@ -1055,9 +1042,14 @@ Zotero.Item.prototype.setCreator = function (orderIndex, data) { // If creatorTypeID isn't valid for this type, use the primary type if (!data.creatorTypeID || !Zotero.CreatorTypes.isValidForItemType(data.creatorTypeID, itemTypeID)) { - var msg = "Creator type '" + Zotero.CreatorTypes.getName(data.creatorTypeID) + "' " - + "isn't valid for " + Zotero.ItemTypes.getName(itemTypeID) - + " -- changing to primary creator"; + let itemType = Zotero.ItemTypes.getName(itemTypeID); + if (options.strict) { + let e = new Error(`Invalid creator type '${origCreatorType}' for type ${itemType}`); + e.name = "ZoteroInvalidDataError"; + throw e; + } + let msg = `Creator type '${origCreatorType}' isn't valid for ${itemType} -- ` + + "changing to primary creator"; Zotero.warn(msg); data.creatorTypeID = Zotero.CreatorTypes.getPrimaryIDForType(itemTypeID); } @@ -1087,7 +1079,7 @@ Zotero.Item.prototype.setCreator = function (orderIndex, data) { /** * @param {Object[]} data - An array of creator data in internal or API JSON format */ -Zotero.Item.prototype.setCreators = function (data) { +Zotero.Item.prototype.setCreators = function (data, options = {}) { // If empty array, clear all existing creators if (!data.length) { while (this.hasCreatorAt(0)) { @@ -1097,7 +1089,7 @@ Zotero.Item.prototype.setCreators = function (data) { } for (let i = 0; i < data.length; i++) { - this.setCreator(i, data[i]); + this.setCreator(i, data[i], options); } } @@ -4191,31 +4183,48 @@ Zotero.Item.prototype.isCollection = function() { /** * Populate the object's data from an API JSON data object + * + * @param {Object} json + * @param {Object} [options] + * @param {Boolean} [options.strict = false] - Throw on unknown field or invalid field for type */ -Zotero.Item.prototype.fromJSON = function (json) { +Zotero.Item.prototype.fromJSON = function (json, options = {}) { + var strict = !!options.strict; + if (!json.itemType && !this._itemTypeID) { throw new Error("itemType property not provided"); } let itemTypeID = Zotero.ItemTypes.getID(json.itemType); if (!itemTypeID) { - let e = new Error(`Invalid item type '${json.itemType}'`); - e.name = "ZoteroUnknownTypeError"; + let e = new Error(`Unknown item type '${json.itemType}'`); + e.name = "ZoteroInvalidDataError"; throw e; } this.setType(itemTypeID); var isValidForType = {}; - var setFields = {}; + var setFields = new Set(); + var { fields: extraFields, extra } = Zotero.Utilities.Internal.extractExtraFields( + json.extra !== undefined ? json.extra : '', + this, + Object.keys(json) + ); + + // Transfer valid fields from Extra to regular fields + // Currently disabled + /*for (let [field, value] of extraFields) { + this.setField(field, value); + setFields.add(field); + extraFields.delete(field); + }*/ - // Primary data for (let field in json) { let val = json[field]; switch (field) { case 'key': case 'version': - case 'synced': case 'itemType': case 'note': // Use? @@ -4226,6 +4235,7 @@ Zotero.Item.prototype.fromJSON = function (json) { case 'parentItem': case 'deleted': case 'inPublications': + case 'extra': break; case 'accessDate': @@ -4238,7 +4248,7 @@ Zotero.Item.prototype.fromJSON = function (json) { val = Zotero.Date.dateToSQL(d, true); } this.setField(field, val); - setFields[field] = true; + setFields.add(field); break; case 'dateAdded': @@ -4255,7 +4265,7 @@ Zotero.Item.prototype.fromJSON = function (json) { break; case 'creators': - this.setCreators(json.creators); + this.setCreators(json.creators, options); break; case 'tags': @@ -4298,8 +4308,19 @@ Zotero.Item.prototype.fromJSON = function (json) { default: let fieldID = Zotero.ItemFields.getID(field); if (!fieldID) { - Zotero.logError("Discarding unknown JSON field '" + field + "' for item " - + this.libraryKey); + // In strict mode, fail on unknown field + if (strict) { + let e = new Error(`Unknown field '${field}'`); + e.name = "ZoteroInvalidDataError"; + throw e; + } + // Otherwise store in Extra + if (typeof val == 'string') { + Zotero.warn(`Storing unknown field '${field}' in Extra for item ${this.libraryKey}`); + extraFields.set(field, val); + break; + } + Zotero.warn(`Discarding unknown JSON ${typeof val} '${field}' for item ${this.libraryKey}`); continue; } // Convert to base-mapped field if necessary, so that setFields has the base-mapped field @@ -4312,15 +4333,26 @@ Zotero.Item.prototype.fromJSON = function (json) { } isValidForType[field] = Zotero.ItemFields.isValidForType(fieldID, this.itemTypeID); if (!isValidForType[field]) { - Zotero.logError("Discarding invalid field '" + origField + "' for type " + itemTypeID - + " for item " + this.libraryKey); + let type = Zotero.ItemTypes.getName(itemTypeID); + // In strict mode, fail on invalid field for type + if (strict) { + let e = new Error(`Invalid field '${origField}' for type ${type}`); + e.name = "ZoteroInvalidDataError"; + throw e; + } + // Otherwise store in Extra + Zotero.warn(`Storing invalid field '${origField}' for type ${type} in Extra for ` + + `item ${this.libraryKey}`); + extraFields.set(field, val); continue; } this.setField(field, json[origField]); - setFields[field] = true; + setFields.add(field); } } + this.setField('extra', Zotero.Utilities.Internal.combineExtraFields(extra, extraFields)); + if (json.collections || this._collections.length) { this.setCollections(json.collections); } @@ -4328,7 +4360,7 @@ Zotero.Item.prototype.fromJSON = function (json) { // Clear existing fields not specified var previousFields = this.getUsedFields(true); for (let field of previousFields) { - if (!setFields[field] && isValidForType[field] !== false) { + if (!setFields.has(field) && isValidForType[field] !== false && field != 'extra') { this.setField(field, false); } } @@ -4509,6 +4541,45 @@ Zotero.Item.prototype.toResponseJSON = function (options = {}) { }; +/** + * Migrate valid fields in Extra to real fields + * + * A separate save is required + */ +Zotero.Item.prototype.migrateExtraFields = function () { + var { itemType, fields, creators, extra } = Zotero.Utilities.Internal.extractExtraFields( + this.getField('extra'), this + ); + if (itemType) { + this.setType(Zotero.ItemTypes.getID(itemType)); + } + for (let [field, value] of fields) { + this.setField(field, value); + } + if (creators.length) { + this.setCreators([...item.getCreators(), ...creators]); + } + this.setField('extra', extra); + if (!this.hasChanged()) { + return false; + } + + Zotero.debug("Migrating Extra fields for item " + this.libraryKey); + if (itemType) { + Zotero.debug("Item Type: " + itemType); + } + if (fields.size) { + Zotero.debug(Array.from(fields.entries())); + } + if (creators.length) { + Zotero.debug(creators); + } + Zotero.debug(extra); + + return true; +} + + ////////////////////////////////////////////////////////////////////////////// // // Asynchronous load methods diff --git a/chrome/content/zotero/xpcom/data/itemFields.js b/chrome/content/zotero/xpcom/data/itemFields.js index 36f97c47c4..d344485b6e 100644 --- a/chrome/content/zotero/xpcom/data/itemFields.js +++ b/chrome/content/zotero/xpcom/data/itemFields.js @@ -42,7 +42,6 @@ Zotero.ItemFields = new function() { // Privileged methods this.getName = getName; this.getID = getID; - this.getLocalizedString = getLocalizedString; this.isValidForType = isValidForType; this.isInteger = isInteger; this.getItemTypeFields = getItemTypeFields; @@ -77,17 +76,29 @@ Zotero.ItemFields = new function() { var baseFields = yield Zotero.DB.columnQueryAsync(sql); for (let field of fields) { - _fields[field['fieldID']] = { - id: field['fieldID'], + let isBaseField = baseFields.includes(field.fieldID); + let label; + try { + label = field.label || Zotero.Schema.globalSchemaLocale.fields[field.fieldName]; + } + // Some base fields aren't localized + catch (e) { + if (!isBaseField) { + throw e; + } + } + + _fields[field.fieldID] = { + id: field.fieldID, name: field.fieldName, - label: field.label, + label, custom: !!field.custom, - isBaseField: (baseFields.indexOf(field['fieldID']) != -1), - formatID: field['fieldFormatID'], - itemTypes: fieldItemTypes[field['fieldID']] + isBaseField: baseFields.includes(field.fieldID), + formatID: field.fieldFormatID, + itemTypes: fieldItemTypes[field.fieldID] }; // Store by name as well as id - _fields[field['fieldName']] = _fields[field['fieldID']]; + _fields[field.fieldName] = _fields[field.fieldID]; _allFields.push({ id: field.fieldID, name: field.fieldName @@ -133,40 +144,29 @@ Zotero.ItemFields = new function() { }; - function getLocalizedString(itemType, field) { - // unused currently - //var typeName = Zotero.ItemTypes.getName(itemType); - var fieldName = Zotero.ItemFields.getName(field); + this.getLocalizedString = function (field) { + if (arguments.length == 2) { + Zotero.warn("Zotero.ItemFields.getLocalizedString() no longer takes two arguments " + + "-- update your code"); + field = arguments[1]; + } + + var fieldName = this.getName(field); // Fields in the items table are special cases switch (field) { case 'dateAdded': case 'dateModified': case 'itemType': - return Zotero.getString("itemFields." + field); + return Zotero.Schema.globalSchemaLocale.fields[field]; } // TODO: different labels for different item types - _fieldCheck(field, 'getLocalizedString'); + _fieldCheck(field); - if (_fields[field].label) { - return _fields[field].label; - } - else { - try { - var loc = Zotero.getString("itemFields." + fieldName); - } - // If localized string not found, try base field - catch (e) { - Zotero.debug("Localized string not found for field '" + fieldName + "' -- trying base field"); - var baseFieldID = this.getBaseIDFromTypeAndField(itemType, field); - fieldName = this.getName(baseFieldID); - var loc = Zotero.getString("itemFields." + fieldName); - } - return loc; - } - } + return _fields[field].label; + }; function isValidForType(fieldID, itemTypeID) { @@ -182,15 +182,28 @@ Zotero.ItemFields = new function() { function isInteger(fieldID) { - _fieldCheck(fieldID, 'isInteger'); + _fieldCheck(fieldID); var ffid = _fields[fieldID]['formatID']; return _fieldFormats[ffid] ? _fieldFormats[ffid]['isInteger'] : false; } + this.isDate = function (field) { + var fieldID = this.getID(field); + var fieldName = this.getName(field); + if (Zotero.ItemFields.isFieldOfBase(fieldID, 'date')) { + return true; + } + if (Zotero.Schema.globalSchemaMeta.fields[fieldName]) { + return Zotero.Schema.globalSchemaMeta.fields[fieldName].type == 'date' + } + return false; + }; + + this.isCustom = function (fieldID) { - _fieldCheck(fieldID, 'isCustom'); + _fieldCheck(fieldID); return _fields[fieldID].custom; } @@ -202,7 +215,7 @@ Zotero.ItemFields = new function() { function getItemTypeFields(itemTypeID) { if (!itemTypeID) { let e = new Error("Invalid item type id '" + itemTypeID + "'"); - e.name = "ZoteroUnknownTypeError"; + e.name = "ZoteroInvalidDataError"; throw e; } @@ -219,14 +232,14 @@ Zotero.ItemFields = new function() { function isBaseField(field) { - _fieldCheck(field, arguments.callee.name); + _fieldCheck(field); return _fields[field]['isBaseField']; } function isFieldOfBase(field, baseField) { - var fieldID = _fieldCheck(field, 'isFieldOfBase'); + var fieldID = _fieldCheck(field); var baseFieldID = this.getID(baseField); if (!baseFieldID) { @@ -299,7 +312,7 @@ Zotero.ItemFields = new function() { throw new Error("Invalid item type '" + itemType + "'"); } - _fieldCheck(typeField, 'getBaseIDFromTypeAndField'); + _fieldCheck(typeField); if (!this.isValidForType(typeFieldID, itemTypeID)) { throw new Error("'" + typeField + "' is not a valid field for '" + itemType + "'"); @@ -406,11 +419,11 @@ Zotero.ItemFields = new function() { * Check whether a field is valid, throwing an exception if not * (since it should never actually happen) **/ - function _fieldCheck(field, func) { + function _fieldCheck(field) { var fieldID = Zotero.ItemFields.getID(field); if (!fieldID) { Zotero.debug((new Error).stack, 1); - throw new Error("Invalid field '" + field + (func ? "' in ItemFields." + func + "()" : "'")); + throw new Error(`Invalid field '${field}`); } return fieldID; } @@ -508,8 +521,9 @@ Zotero.ItemFields = new function() { var sql = 'SELECT itemTypeID, fieldID FROM itemTypeFieldsCombined ORDER BY orderIndex'; var rows = yield Zotero.DB.queryAsync(sql); - _itemTypeFields = {}; - _itemTypeFields[1] = []; // notes have no fields + _itemTypeFields = { + [Zotero.ItemTypes.getID('note')]: [] // Notes have no fields + }; for (let i=0; i _maxCompatibility) { - let dbClientVersion = yield Zotero.DB.valueQueryAsync( + let dbClientVersion = await Zotero.DB.valueQueryAsync( "SELECT value FROM settings " + "WHERE setting='client' AND key='lastCompatibleVersion'" ); @@ -135,53 +135,81 @@ Zotero.Schema = new function(){ } // Check if DB is coming from the DB Repair Tool and should be checked - var integrityCheck = yield Zotero.DB.valueQueryAsync( + var integrityCheck = await Zotero.DB.valueQueryAsync( "SELECT value FROM settings WHERE setting='db' AND key='integrityCheck'" ); - var schemaVersion = yield _getSchemaSQLVersion('userdata'); + // Check whether bundled global schema file is newer than DB + var bundledGlobalSchema = await _readGlobalSchemaFromFile(); + var bundledGlobalSchemaVersionCompare = await _globalSchemaVersionCompare( + bundledGlobalSchema.version + ); + + // Check whether bundled userdata schema has been updated + var userdataVersion = await _getSchemaSQLVersion('userdata'); options.minor = minorUpdateFrom && userdata >= minorUpdateFrom; // If non-minor userdata upgrade, make backup of database first - if (userdata < schemaVersion && !options.minor) { - yield Zotero.DB.backupDatabase(userdata, true); + if (userdata < userdataVersion && !options.minor) { + await Zotero.DB.backupDatabase(userdata, true); } - else if (integrityCheck) { - yield Zotero.DB.backupDatabase(false, true); + // Automatic backup + else if (integrityCheck || bundledGlobalSchemaVersionCompare === 1) { + await Zotero.DB.backupDatabase(false, true); } - yield Zotero.DB.queryAsync("PRAGMA foreign_keys = false"); + await Zotero.DB.queryAsync("PRAGMA foreign_keys = false"); try { - var updated = yield Zotero.DB.executeTransaction(function* (conn) { - var updated = yield _updateSchema('system'); + // If bundled global schema file is newer than DB, apply it + if (bundledGlobalSchemaVersionCompare === 1) { + await Zotero.DB.executeTransaction(async function () { + await _updateGlobalSchema(bundledGlobalSchema); + }); + } + else { + let data; + // If bundled global schema is up to date, use it + if (bundledGlobalSchemaVersionCompare === 0) { + data = bundledGlobalSchema; + } + // If bundled global schema is older than the DB (because of a downgrade), use the + // DB version, which will match the mapping tables + else if (bundledGlobalSchemaVersionCompare === -1) { + data = await _readGlobalSchemaFromDB(); + } + await _loadGlobalSchema(data, bundledGlobalSchema.version); + } + + var updated = await Zotero.DB.executeTransaction(async function (conn) { + var updated = await _updateSchema('system'); // Update custom tables if they exist so that changes are in // place before user data migration if (Zotero.DB.tableExists('customItemTypes')) { - yield _updateCustomTables(updated); + await _updateCustomTables(); } // Auto-repair databases coming from the DB Repair Tool if (integrityCheck) { - yield this.integrityCheck(true); - yield Zotero.DB.queryAsync( + await this.integrityCheck(true); + await Zotero.DB.queryAsync( "DELETE FROM settings WHERE setting='db' AND key='integrityCheck'" ); } - updated = yield _migrateUserDataSchema(userdata, options); - yield _updateSchema('triggers'); + updated = await _migrateUserDataSchema(userdata, options); + await _updateSchema('triggers'); // Populate combined tables for custom types and fields -- this is likely temporary // // We do this again in case custom fields were changed during user data migration - yield _updateCustomTables() + await _updateCustomTables(); return updated; }.bind(this)); } finally { - yield Zotero.DB.queryAsync("PRAGMA foreign_keys = true"); + await Zotero.DB.queryAsync("PRAGMA foreign_keys = true"); } if (updated) { @@ -216,7 +244,7 @@ Zotero.Schema = new function(){ } // Reset sync queue tries if new version - yield _checkClientVersion(); + await _checkClientVersion(); // In Standalone, don't load bundled files until after UI is ready. In Firefox, load them as // soon initialization is done so that translation works before the Zotero pane is opened. @@ -269,7 +297,348 @@ Zotero.Schema = new function(){ }); return updated; - }); + }; + + + /** + * Get bundled schema from disk + * + * @return {Object} + */ + async function _readGlobalSchemaFromFile() { + return JSON.parse( + await Zotero.File.getResourceAsync('resource://zotero/schema/global/schema.json') + ); + } + + + /** + * Get schema from database + * + * Doesn't include the .itemTypes property, which was already applied to the mapping tables + */ + async function _readGlobalSchemaFromDB() { + var pako = {}; + Services.scriptloader.loadSubScript("resource://zotero/pako.js", pako); + var data = await Zotero.DB.valueQueryAsync( + "SELECT value FROM settings WHERE schema='globalSchema' AND key='data'" + ); + if (data) { + try { + data = JSON.parse(pako.inflate(data, { to: 'string' })); + } + catch (e) { + Zotero.warn("Unable to extract global schema -- falling back to file: " + e); + } + } + else { + Zotero.warn("Global schema not found in DB -- falling back to file"); + } + + // If the data is missing or unreadable in the DB for some reason (e.g., DB corruption), + // fall back to the file, though it might be out of date + if (!data) { + data = await _readGlobalSchemaFromFile(); + } + + return data; + } + + + /** + * Compares a given version number to the version of the schema in the database + * + * @return {Number} - 1 if provided version is greater than DB (i.e., DB needs update), 0 if + * the same, -1 if DB version is newer + */ + async function _globalSchemaVersionCompare(version) { + if (!version) { + throw new Error("version not specified"); + } + + var dbVersion = await Zotero.Schema.getDBVersion('globalSchema') || null; + if (dbVersion > version) { + Zotero.debug(`Database has newer global schema (${dbVersion} > ${version}) -- skipping update`); + return -1; + } + else if (dbVersion == version) { + Zotero.debug(`Database is up to date with global schema version ${version} -- skipping update`); + return 0; + } + + Zotero.debug(`Global schema needs update from ${dbVersion} to ${version}`); + return 1; + } + + + /** + * Update the item-type/field/creator mapping tables based on the passed schema + */ + async function _updateGlobalSchema(data) { + Zotero.debug("Updating global schema to version " + data.version); + + Zotero.DB.requireTransaction(); + + await Zotero.ID.init(); + + var preItemTypeRows = await Zotero.DB.queryAsync( + "SELECT itemTypeID AS id, typeName AS name FROM itemTypes" + ); + var preFieldRows = await Zotero.DB.queryAsync( + "SELECT fieldID AS id, fieldName AS name FROM fields" + ); + var preCreatorTypeRows = await Zotero.DB.queryAsync( + "SELECT creatorTypeID AS id, creatorType AS name FROM creatorTypes" + ); + var preFields = new Set(preFieldRows.map(x => x.name)); + var preCreatorTypes = new Set(preCreatorTypeRows.map(x => x.name)); + var preItemTypeIDsByName = new Map(preItemTypeRows.map(x => [x.name, x.id])); + var preFieldIDsByName = new Map(preFieldRows.map(x => [x.name, x.id])); + var preCreatorTypeIDsByName = new Map(preCreatorTypeRows.map(x => [x.name, x.id])); + var postFields = new Set(); + var postCreatorTypes = new Set(); + var postFieldIDsByName = new Map(); + var postCreatorTypeIDsByName = new Map(); + + // Add new fields and creator types + for (let { fields, creatorTypes } of data.itemTypes) { + for (let { field, baseField } of fields) { + postFields.add(field); + if (baseField) { + postFields.add(baseField); + } + } + + for (let { creatorType } of creatorTypes) { + postCreatorTypes.add(creatorType); + } + } + var fieldsValueSets = []; + var fieldsParams = []; + for (let field of postFields) { + if (preFields.has(field)) { + postFieldIDsByName.set(field, preFieldIDsByName.get(field)); + } + else { + let id = Zotero.ID.get('fields'); + fieldsValueSets.push("(?, ?, NULL)"); + fieldsParams.push(id, field); + postFieldIDsByName.set(field, id); + } + } + if (fieldsValueSets.length) { + await Zotero.DB.queryAsync( + "INSERT INTO fields VALUES " + fieldsValueSets.join(", "), + fieldsParams + ); + } + var creatorTypesValueSets = []; + var creatorTypesParams = []; + for (let type of postCreatorTypes) { + if (preCreatorTypes.has(type)) { + postCreatorTypeIDsByName.set(type, preCreatorTypeIDsByName.get(type)); + } + else { + let id = Zotero.ID.get('creatorTypes'); + creatorTypesValueSets.push("(?, ?)"); + creatorTypesParams.push(id, type); + postCreatorTypeIDsByName.set(type, id); + } + } + if (creatorTypesValueSets.length) { + await Zotero.DB.queryAsync( + "INSERT INTO creatorTypes VALUES " + creatorTypesValueSets.join(", "), + creatorTypesParams + ); + } + + // Apply changes to DB + let itemTypeFieldsValueSets = []; + let baseFieldMappingsValueSets = []; + let itemTypeCreatorTypesValueSets = []; + for (let { itemType, fields, creatorTypes } of data.itemTypes) { + let itemTypeID = preItemTypeIDsByName.get(itemType); + // let preItemTypeCreatorTypeIDs = []; + if (itemTypeID) { + // Unused + /*preItemTypeCreatorTypeIDs = await Zotero.DB.columnQueryAsync( + "SELECT creatorTypeID FROM itemTypeCreatorTypes WHERE itemTypeID=?", + itemTypeID + );*/ + } + // New item type + else { + itemTypeID = Zotero.ID.get('itemTypes'); + await Zotero.DB.queryAsync( + "INSERT INTO itemTypes VALUES (?, ?, NULL, 1)", + [itemTypeID, itemType] + ); + } + + // Fields + let index = 0; + let postItemTypeFieldIDs = new Set(); + for (let { field, baseField } of fields) { + let fieldID = postFieldIDsByName.get(field); + postItemTypeFieldIDs.add(fieldID); + itemTypeFieldsValueSets.push(`(${itemTypeID}, ${fieldID}, 0, ${index++})`); + if (baseField) { + let baseFieldID = postFieldIDsByName.get(baseField); + baseFieldMappingsValueSets.push(`(${itemTypeID}, ${baseFieldID}, ${fieldID})`); + } + } + + + // TODO: Check for fields removed from this item type + // throw new Error(`Field ${id} was removed from ${itemType}`); + + // Creator types + for (let { creatorType, primary } of creatorTypes) { + let typeID = postCreatorTypeIDsByName.get(creatorType); + itemTypeCreatorTypesValueSets.push(`(${itemTypeID}, ${typeID}, ${primary ? 1 : 0})`); + } + + // TODO: Check for creator types removed from this item type + // throw new Error(`Creator type ${id} was removed from ${itemType}`); + + // TODO: Deal with existing types not in the schema, and their items + } + + await Zotero.DB.queryAsync("DELETE FROM itemTypeFields"); + await Zotero.DB.queryAsync("DELETE FROM baseFieldMappings"); + await Zotero.DB.queryAsync("DELETE FROM itemTypeCreatorTypes"); + + await Zotero.DB.queryAsync("INSERT INTO itemTypeFields VALUES " + + itemTypeFieldsValueSets.join(", ")); + await Zotero.DB.queryAsync("INSERT INTO baseFieldMappings VALUES " + + baseFieldMappingsValueSets.join(", ")); + await Zotero.DB.queryAsync("INSERT INTO itemTypeCreatorTypes VALUES " + + itemTypeCreatorTypesValueSets.join(", ")); + + // Store data in DB as compressed binary string. This lets us use a schema that matches the + // DB tables even if the user downgrades to a version with an earlier bundled schema file. + var pako = require('pako'); + var dbData = { ...data }; + // Don't include types and fields, which are already in the mapping tables + delete dbData.itemTypes; + await Zotero.DB.queryAsync( + "REPLACE INTO settings VALUES ('globalSchema', 'data', ?)", + pako.deflate(JSON.stringify(dbData), { to: 'string' }), + { + debugParams: false + } + ); + await _updateDBVersion('globalSchema', data.version); + + var bundledVersion = (await _readGlobalSchemaFromFile()).version; + await _loadGlobalSchema(data, bundledVersion); + await _reloadSchema(); + // Mark that we need to migrate Extra values to any newly available fields in + // Zotero.Schema.migrateExtraFields() + await Zotero.DB.queryAsync( + "REPLACE INTO settings VALUES ('globalSchema', 'migrateExtra', 1)" + ); + + return true; + } + + + this._updateGlobalSchemaForTest = async function (schema) { + await Zotero.DB.executeTransaction(async function () { + await _updateGlobalSchema(schema); + }.bind(this)); + }; + + + + /** + * Set properties on Zotero.Schema based on the passed data + * + * @param {Object} data - Global schema data ('version', 'itemTypes', 'locales', etc.) + * @param {Number} bundledVersion - Version of the bundled schema.json file + */ + async function _loadGlobalSchema(data, bundledVersion) { + if (!data) { + throw new Error("Data not provided"); + } + var locale = data.locales[Zotero.locale]; + if (!locale) { + Zotero.warn(`Locale ${Zotero.locale} not found in global schema locales`); + locale = data.locales['en-US']; + if (!locale) { + throw new Error("en-US locale not found in global schema locales"); + } + } + Zotero.Schema.globalSchemaVersion = data.version; + Zotero.Schema.globalSchemaLocale = locale; + Zotero.Schema.globalSchemaMeta = data.meta; + Zotero.Schema.CSL_TYPE_MAPPINGS = {}; + Zotero.Schema.CSL_TYPE_MAPPINGS_REVERSE = {}; + for (let cslType in data.csl.types) { + for (let zoteroType of data.csl.types[cslType]) { + Zotero.Schema.CSL_TYPE_MAPPINGS[zoteroType] = cslType; + } + Zotero.Schema.CSL_TYPE_MAPPINGS_REVERSE[cslType] = data.csl.types[cslType][0]; + } + Zotero.Schema.CSL_TEXT_MAPPINGS = data.csl.fields.text; + Zotero.Schema.CSL_DATE_MAPPINGS = data.csl.fields.date; + Zotero.Schema.CSL_NAME_MAPPINGS = data.csl.names; + } + + + /** + * Migrate values from item Extra fields that can be moved to regular item fields after a global + * schema update + * + * This needs the data object architecture to be initialized, so it's called from zotero.js + * rather than in _updateGlobalSchema(). + */ + this.migrateExtraFields = async function () { + // Check for a flag set by _updateGlobalSchema() + var needsUpdate = await Zotero.DB.valueQueryAsync( + "SELECT COUNT(*) FROM settings WHERE setting='globalSchema' AND key='migrateExtra'" + ); + if (!needsUpdate) { + return; + } + + var fieldID = Zotero.ItemFields.getID('extra'); + var sql = "SELECT itemID, value FROM itemData " + + "JOIN itemDataValues USING (valueID) " + + "WHERE fieldID=?"; + var rows = await Zotero.DB.queryAsync(sql, fieldID); + var itemIDs = []; + for (let row of rows) { + let { itemType, fields, creators } = Zotero.Utilities.Internal.extractExtraFields( + row.value + ); + if (itemType || fields.size || creators.length) { + itemIDs.push(row.itemID); + } + } + + var items = await Zotero.Items.getAsync(itemIDs); + await Zotero.Items.loadDataTypes(items, ['itemData', 'creator']); + for (let item of items) { + let changed = item.migrateExtraFields(); + if (!changed) continue; + // Don't mark the item as changed if it's not already so that every client doesn't try + // to upload new data. While in theory the changes should be the same and automatically + // merged, it's risky and unnecessary, since server data will also be updated on schema + // changes. (It would also create a huge amount of write traffic.) + if (item.synced) { + item.synced = true; + } + await item.saveTx({ + skipDateModifiedUpdate: true, + skipSelect: true + }); + } + + await Zotero.DB.queryAsync( + "DELETE FROM settings WHERE setting='globalSchema' AND key='migrateExtra'" + ); + }; // https://www.zotero.org/support/nsf @@ -331,15 +700,15 @@ Zotero.Schema = new function(){ switch (fields[i][0]) { case 'name': - var baseFieldID = 110; // title + var baseFieldID = Zotero.ItemFields.getID('title'); break; case 'dateSent': - var baseFieldID = 14; // date + var baseFieldID = Zotero.ItemFields.getID('date'); break; case 'homepage': - var baseFieldID = 1; // URL + var baseFieldID = Zotero.ItemFields.getID('url'); break; default: @@ -411,6 +780,7 @@ Zotero.Schema = new function(){ yield _updateCustomTables(); yield Zotero.ItemTypes.init(); yield Zotero.ItemFields.init(); + yield Zotero.CreatorTypes.init(); yield Zotero.SearchConditions.init(); // Update item type menus in every open window @@ -425,58 +795,40 @@ Zotero.Schema = new function(){ }); - var _updateCustomTables = Zotero.Promise.coroutine(function* (skipDelete, skipSystem) { + var _updateCustomTables = async function () { Zotero.debug("Updating custom tables"); Zotero.DB.requireTransaction(); - if (!skipDelete) { - yield Zotero.DB.queryAsync("DELETE FROM itemTypesCombined"); - yield Zotero.DB.queryAsync("DELETE FROM fieldsCombined WHERE fieldID NOT IN (SELECT fieldID FROM itemData)"); - yield Zotero.DB.queryAsync("DELETE FROM itemTypeFieldsCombined"); - yield Zotero.DB.queryAsync("DELETE FROM baseFieldMappingsCombined"); - } + await Zotero.DB.queryAsync("DELETE FROM itemTypesCombined"); + await Zotero.DB.queryAsync("DELETE FROM fieldsCombined WHERE fieldID NOT IN (SELECT fieldID FROM itemData)"); + await Zotero.DB.queryAsync("DELETE FROM itemTypeFieldsCombined"); + await Zotero.DB.queryAsync("DELETE FROM baseFieldMappingsCombined"); var offset = Zotero.ItemTypes.customIDOffset; - yield Zotero.DB.queryAsync( + await Zotero.DB.queryAsync( "INSERT INTO itemTypesCombined " - + ( - skipSystem - ? "" - : "SELECT itemTypeID, typeName, display, 0 AS custom FROM itemTypes UNION " - ) + + "SELECT itemTypeID, typeName, display, 0 AS custom FROM itemTypes UNION " + "SELECT customItemTypeID + " + offset + " AS itemTypeID, typeName, display, 1 AS custom FROM customItemTypes" ); - yield Zotero.DB.queryAsync( + await Zotero.DB.queryAsync( "INSERT OR IGNORE INTO fieldsCombined " - + ( - skipSystem - ? "" - : "SELECT fieldID, fieldName, NULL AS label, fieldFormatID, 0 AS custom FROM fields UNION " - ) + + "SELECT fieldID, fieldName, NULL AS label, fieldFormatID, 0 AS custom FROM fields UNION " + "SELECT customFieldID + " + offset + " AS fieldID, fieldName, label, NULL, 1 AS custom FROM customFields" ); - yield Zotero.DB.queryAsync( + await Zotero.DB.queryAsync( "INSERT INTO itemTypeFieldsCombined " - + ( - skipSystem - ? "" - : "SELECT itemTypeID, fieldID, hide, orderIndex FROM itemTypeFields UNION " - ) + + "SELECT itemTypeID, fieldID, hide, orderIndex FROM itemTypeFields UNION " + "SELECT customItemTypeID + " + offset + " AS itemTypeID, " + "COALESCE(fieldID, customFieldID + " + offset + ") AS fieldID, hide, orderIndex FROM customItemTypeFields" ); - yield Zotero.DB.queryAsync( + await Zotero.DB.queryAsync( "INSERT INTO baseFieldMappingsCombined " - + ( - skipSystem - ? "" - : "SELECT itemTypeID, baseFieldID, fieldID FROM baseFieldMappings UNION " - ) + + "SELECT itemTypeID, baseFieldID, fieldID FROM baseFieldMappings UNION " + "SELECT customItemTypeID + " + offset + " AS itemTypeID, baseFieldID, " + "customFieldID + " + offset + " AS fieldID FROM customBaseFieldMappings" ); - }); + }; /** @@ -1331,6 +1683,13 @@ Zotero.Schema = new function(){ } } + var attachmentID = parseInt(yield Zotero.DB.valueQueryAsync( + "SELECT itemTypeID FROM itemTypes WHERE typeName='attachment'" + )); + var noteID = parseInt(yield Zotero.DB.valueQueryAsync( + "SELECT itemTypeID FROM itemTypes WHERE typeName='note'" + )); + // Non-foreign key checks // @@ -1344,29 +1703,29 @@ Zotero.Schema = new function(){ "SELECT COUNT(*) > 0 FROM items WHERE itemTypeID IS NULL", "DELETE FROM items WHERE itemTypeID IS NULL", ], - // Attachments row with itemTypeID != 14 + // Non-attachment items in attachments table [ - "SELECT COUNT(*) > 0 FROM itemAttachments JOIN items USING (itemID) WHERE itemTypeID != 14", - "UPDATE items SET itemTypeID=14, clientDateModified=CURRENT_TIMESTAMP WHERE itemTypeID != 14 AND itemID IN (SELECT itemID FROM itemAttachments)", + `SELECT COUNT(*) > 0 FROM itemAttachments JOIN items USING (itemID) WHERE itemTypeID != ${attachmentID}`, + `UPDATE items SET itemTypeID=${attachmentID}, clientDateModified=CURRENT_TIMESTAMP WHERE itemTypeID != ${attachmentID} AND itemID IN (SELECT itemID FROM itemAttachments)`, ], // Fields not in type [ "SELECT COUNT(*) > 0 FROM itemData WHERE fieldID NOT IN (SELECT fieldID FROM itemTypeFieldsCombined WHERE itemTypeID=(SELECT itemTypeID FROM items WHERE itemID=itemData.itemID))", "DELETE FROM itemData WHERE fieldID NOT IN (SELECT fieldID FROM itemTypeFieldsCombined WHERE itemTypeID=(SELECT itemTypeID FROM items WHERE itemID=itemData.itemID))", ], - // Missing itemAttachments row + // Missing itemAttachments rows [ - "SELECT COUNT(*) > 0 FROM items WHERE itemTypeID=14 AND itemID NOT IN (SELECT itemID FROM itemAttachments)", - "INSERT INTO itemAttachments (itemID, linkMode) SELECT itemID, 0 FROM items WHERE itemTypeID=14 AND itemID NOT IN (SELECT itemID FROM itemAttachments)", + `SELECT COUNT(*) > 0 FROM items WHERE itemTypeID=${attachmentID} AND itemID NOT IN (SELECT itemID FROM itemAttachments)`, + `INSERT INTO itemAttachments (itemID, linkMode) SELECT itemID, 0 FROM items WHERE itemTypeID=${attachmentID} AND itemID NOT IN (SELECT itemID FROM itemAttachments)`, ], // Note/child parents [ - "SELECT COUNT(*) > 0 FROM itemAttachments WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (1,14))", - "UPDATE itemAttachments SET parentItemID=NULL WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (1,14))", + `SELECT COUNT(*) > 0 FROM itemAttachments WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (${noteID}, ${attachmentID}))`, + `UPDATE itemAttachments SET parentItemID=NULL WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (${noteID}, ${attachmentID}))`, ], [ - "SELECT COUNT(*) > 0 FROM itemNotes WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (1,14))", - "UPDATE itemNotes SET parentItemID=NULL WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (1,14))", + `SELECT COUNT(*) > 0 FROM itemNotes WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (${noteID}, ${attachmentID}))`, + `UPDATE itemNotes SET parentItemID=NULL WHERE parentItemID IN (SELECT itemID FROM items WHERE itemTypeID IN (${noteID}, ${attachmentID}))`, ], // Delete empty creators @@ -1378,13 +1737,13 @@ Zotero.Schema = new function(){ // Non-attachment items in the full-text index [ - "SELECT COUNT(*) > 0 FROM fulltextItemWords WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=14)", - "DELETE FROM fulltextItemWords WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=14)" + `SELECT COUNT(*) > 0 FROM fulltextItemWords WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=${attachmentID})`, + `DELETE FROM fulltextItemWords WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=${attachmentID})` ], // Full-text items must be attachments [ - "SELECT COUNT(*) > 0 FROM fulltextItems WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=14)", - "DELETE FROM fulltextItems WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=14)" + `SELECT COUNT(*) > 0 FROM fulltextItems WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=${attachmentID})`, + `DELETE FROM fulltextItems WHERE itemID NOT IN (SELECT itemID FROM items WHERE itemTypeID=${attachmentID})` ], // Invalid link mode -- set to imported url [ @@ -1528,8 +1887,8 @@ Zotero.Schema = new function(){ /* * Create new DB schema */ - function _initializeSchema(){ - return Zotero.DB.executeTransaction(function* (conn) { + async function _initializeSchema() { + await Zotero.DB.executeTransaction(function* (conn) { var userLibraryID = 1; // Enable auto-vacuuming @@ -1546,7 +1905,9 @@ Zotero.Schema = new function(){ yield _getSchemaSQL('triggers').then(function (sql) { return Zotero.DB.executeSQLFile(sql); }); - yield _updateCustomTables(true); + + var schema = yield _readGlobalSchemaFromFile(); + yield _updateGlobalSchema(schema); yield _getSchemaSQLVersion('system').then(function (version) { return _updateDBVersion('system', version); @@ -2523,6 +2884,21 @@ Zotero.Schema = new function(){ yield Zotero.DB.queryAsync("ALTER TABLE retractedItems ADD COLUMN flag INT DEFAULT 0"); } + else if (i == 106) { + yield _updateCompatibility(6); + + yield Zotero.DB.queryAsync("DROP TRIGGER insert_date_field"); + yield Zotero.DB.queryAsync("DROP TRIGGER update_date_field"); + yield Zotero.DB.queryAsync("DROP TRIGGER fki_itemAttachments"); + yield Zotero.DB.queryAsync("DROP TRIGGER fku_itemAttachments"); + yield Zotero.DB.queryAsync("DROP TRIGGER fki_itemNotes"); + yield Zotero.DB.queryAsync("DROP TRIGGER fku_itemNotes"); + + yield Zotero.DB.queryAsync("DROP TABLE transactionSets"); + yield Zotero.DB.queryAsync("DROP TABLE transactions"); + yield Zotero.DB.queryAsync("DROP TABLE transactionLog"); + } + // If breaking compatibility or doing anything dangerous, clear minorUpdateFrom } diff --git a/chrome/content/zotero/xpcom/sync/syncEngine.js b/chrome/content/zotero/xpcom/sync/syncEngine.js index a0d4801774..2c8ab88ef5 100644 --- a/chrome/content/zotero/xpcom/sync/syncEngine.js +++ b/chrome/content/zotero/xpcom/sync/syncEngine.js @@ -1244,7 +1244,7 @@ Zotero.Sync.Data.Engine.prototype._uploadObjects = Zotero.Promise.coroutine(func // Update local object with saved data if necessary, as long as it hasn't // changed locally since the upload if (!changed) { - obj.fromJSON(current.data); + obj.fromJSON(current.data, { strict: true }); toSave.push(obj); } else { diff --git a/chrome/content/zotero/xpcom/sync/syncLocal.js b/chrome/content/zotero/xpcom/sync/syncLocal.js index ab08ce7c6e..d63e733fc9 100644 --- a/chrome/content/zotero/xpcom/sync/syncLocal.js +++ b/chrome/content/zotero/xpcom/sync/syncLocal.js @@ -721,11 +721,10 @@ Zotero.Sync.Data.Local = { var ObjectType = Zotero.Utilities.capitalize(objectType); var libraryName = Zotero.Libraries.get(libraryID).name; - var knownErrors = [ - 'ZoteroUnknownTypeError', - 'ZoteroUnknownFieldError', + var knownErrors = new Set([ + 'ZoteroInvalidDataError', 'ZoteroMissingObjectError' - ]; + ]); Zotero.debug("Processing " + json.length + " downloaded " + (json.length == 1 ? objectType : objectTypePlural) @@ -1027,8 +1026,11 @@ Zotero.Sync.Data.Local = { } } catch (e) { + // This allows errors handled by syncRunner to know the library in question + e.libraryID = libraryID; + // Display nicer debug line for known errors - if (knownErrors.indexOf(e.name) != -1) { + if (knownErrors.has(e.name)) { let desc = e.name .replace(/^Zotero/, "") // Convert "MissingObjectError" to "missing object error" @@ -1407,7 +1409,7 @@ Zotero.Sync.Data.Local = { json = this._checkCacheJSON(json); if (!options.skipData) { - obj.fromJSON(json.data); + obj.fromJSON(json.data, { strict: true }); } if (obj.objectType == 'item' && obj.isImportedAttachment()) { yield this._checkAttachmentForDownload(obj, json.data.mtime, options.isNewObject); diff --git a/chrome/content/zotero/xpcom/sync/syncRunner.js b/chrome/content/zotero/xpcom/sync/syncRunner.js index 4dd71064bc..b27a380431 100644 --- a/chrome/content/zotero/xpcom/sync/syncRunner.js +++ b/chrome/content/zotero/xpcom/sync/syncRunner.js @@ -1252,6 +1252,23 @@ Zotero.Sync.Runner_Module = function (options = {}) { } } } + // Show warning for unknown data that couldn't be saved + else if (e.name && e.name == 'ZoteroInvalidDataError') { + e.message = Zotero.getString( + 'sync.error.invalidDataError', + [ + Zotero.Libraries.get(e.libraryID).name, + Zotero.clientName + ] + ) + + "\n\n" + + Zotero.getString('sync.error.invalidDataError.otherData'); + e.errorType = 'warning'; + e.dialogButtonText = Zotero.getString('general.checkForUpdates'); + e.dialogButtonCallback = () => { + Zotero.openCheckForUpdatesWindow(); + }; + } }); diff --git a/chrome/content/zotero/xpcom/utilities.js b/chrome/content/zotero/xpcom/utilities.js index 29a58fd813..39d6dbafca 100644 --- a/chrome/content/zotero/xpcom/utilities.js +++ b/chrome/content/zotero/xpcom/utilities.js @@ -26,121 +26,6 @@ ***** END LICENSE BLOCK ***** */ -/* - * Mappings for names - * Note that this is the reverse of the text variable map, since all mappings should be one to one - * and it makes the code cleaner - */ -var CSL_NAMES_MAPPINGS = { - "author":"author", - "editor":"editor", - "bookAuthor":"container-author", - "composer":"composer", - "director":"director", - "interviewer":"interviewer", - "recipient":"recipient", - "reviewedAuthor":"reviewed-author", - "seriesEditor":"collection-editor", - "translator":"translator" -} - -/* - * Mappings for text variables - */ -var CSL_TEXT_MAPPINGS = { - "title":["title"], - "container-title":["publicationTitle", "reporter", "code"], /* reporter and code should move to SQL mapping tables */ - "collection-title":["seriesTitle", "series"], - "collection-number":["seriesNumber"], - "publisher":["publisher", "distributor"], /* distributor should move to SQL mapping tables */ - "publisher-place":["place"], - "authority":["court","legislativeBody", "issuingAuthority"], - "page":["pages"], - "volume":["volume", "codeNumber"], - "issue":["issue", "priorityNumbers"], - "number-of-volumes":["numberOfVolumes"], - "number-of-pages":["numPages"], - "edition":["edition"], - "version":["versionNumber"], - "section":["section", "committee"], - "genre":["type", "programmingLanguage"], - "source":["libraryCatalog"], - "dimensions": ["artworkSize", "runningTime"], - "medium":["medium", "system"], - "scale":["scale"], - "archive":["archive"], - "archive_location":["archiveLocation"], - "event":["meetingName", "conferenceName"], /* these should be mapped to the same base field in SQL mapping tables */ - "event-place":["place"], - "abstract":["abstractNote"], - "URL":["url"], - "DOI":["DOI"], - "ISBN":["ISBN"], - "ISSN":["ISSN"], - "call-number":["callNumber", "applicationNumber"], - "note":["extra"], - "number":["number"], - "chapter-number":["session"], - "references":["history", "references"], - "shortTitle":["shortTitle"], /* preserved to read legacy data */ - "title-short":["shortTitle"], - "journalAbbreviation":["journalAbbreviation"], - "status":["legalStatus"], - "language":["language"] -} - -/* - * Mappings for dates - */ -var CSL_DATE_MAPPINGS = { - "issued":"date", - "accessed":"accessDate", - "submitted":"filingDate" -} - -/* - * Mappings for types - * Also see itemFromCSLJSON - */ -var CSL_TYPE_MAPPINGS = { - 'book':"book", - 'bookSection':'chapter', - 'journalArticle':"article-journal", - 'magazineArticle':"article-magazine", - 'newspaperArticle':"article-newspaper", - 'thesis':"thesis", - 'encyclopediaArticle':"entry-encyclopedia", - 'dictionaryEntry':"entry-dictionary", - 'conferencePaper':"paper-conference", - 'letter':"personal_communication", - 'manuscript':"manuscript", - 'interview':"interview", - 'film':"motion_picture", - 'artwork':"graphic", - 'webpage':"webpage", - 'report':"report", - 'bill':"bill", - 'case':"legal_case", - 'hearing':"bill", // ?? - 'patent':"patent", - 'statute':"legislation", // ?? - 'email':"personal_communication", - 'map':"map", - 'blogPost':"post-weblog", - 'instantMessage':"personal_communication", - 'forumPost':"post", - 'audioRecording':"song", // ?? - 'presentation':"speech", - 'videoRecording':"motion_picture", - 'tvBroadcast':"broadcast", - 'radioBroadcast':"broadcast", - 'podcast':"broadcast", - 'computerProgram':"book", // ?? - 'document':"article", - 'note':"article", - 'attachment':"article" -}; - /** * @class Functions for text manipulation and other miscellaneous purposes */ @@ -1576,7 +1461,7 @@ Zotero.Utilities = { ); } - var cslType = CSL_TYPE_MAPPINGS[zoteroItem.itemType]; + var cslType = Zotero.Schema.CSL_TYPE_MAPPINGS[zoteroItem.itemType]; if (!cslType) { throw new Error('Unexpected Zotero Item type "' + zoteroItem.itemType + '"'); } @@ -1589,9 +1474,9 @@ Zotero.Utilities = { }; // get all text variables (there must be a better way) - for(var variable in CSL_TEXT_MAPPINGS) { + for(var variable in Zotero.Schema.CSL_TEXT_MAPPINGS) { if (variable === "shortTitle") continue; // read both title-short and shortTitle, but write only title-short - var fields = CSL_TEXT_MAPPINGS[variable]; + var fields = Zotero.Schema.CSL_TEXT_MAPPINGS[variable]; for(var i=0, n=fields.length; i [this._normalizeExtraKey(x.name), x.name])); + // CSL types + for (let i in Zotero.Schema.CSL_TYPE_MAPPINGS) { + let cslType = Zotero.Schema.CSL_TYPE_MAPPINGS[i]; + itemTypes.set(cslType.toLowerCase(), i); + } + + // For fields we use arrays, because there can be multiple possibilities // // Built-in fields - var fieldNames = new Map(Zotero.ItemFields.getAll().map(x => [x.name.toLowerCase(), [x.name]])); + var fieldNames = new Map(Zotero.ItemFields.getAll().map(x => [this._normalizeExtraKey(x.name), [x.name]])); // CSL fields - for (let map of [CSL_TEXT_MAPPINGS, CSL_DATE_MAPPINGS]) { + for (let map of [Zotero.Schema.CSL_TEXT_MAPPINGS, Zotero.Schema.CSL_DATE_MAPPINGS]) { for (let cslVar in map) { - let normalized = cslVar.toLowerCase(); + let normalized = this._normalizeExtraKey(cslVar); let existing = fieldNames.get(normalized) || []; fieldNames.set(normalized, new Set([...existing, ...map[cslVar]])); } } - var lines = str.split(/\n+/g); - var fields = new Map(); + // Built-in creator types + var creatorTypes = new Map(Zotero.CreatorTypes.getAll().map(x => [this._normalizeExtraKey(x.name), x.name])); + // CSL types + for (let i in Zotero.Schema.CSL_NAME_MAPPINGS) { + let cslType = Zotero.Schema.CSL_NAME_MAPPINGS[i]; + creatorTypes.set(cslType.toLowerCase(), i); + } + + // Process Extra lines + var keepLines = []; + var skipKeys = new Set(); + var lines = extra.split(/\n/g); for (let line of lines) { - let parts = line.match(/^([a-z \-]+):(.+)/i); + let parts = line.match(/^([a-z -_]+):(.+)/i); + // Old citeproc.js cheater syntax; if (!parts) { + parts = line.match(/^{:([a-z -_]+):(.+)}/i); + } + if (!parts) { + keepLines.push(line); continue; } let [_, originalField, value] = parts; - let field = originalField.trim().toLowerCase() - // Strip spaces - .replace(/\s+/g, '') - // Old citeproc.js cheater syntax - .replace(/{:([^:]+):([^}]+)}/); - value = value.trim(); - let possibleFields = fieldNames.get(field); - // No valid fields - if (!possibleFields) { + let key = this._normalizeExtraKey(originalField); + if (skipKeys.has(key)) { + keepLines.push(line); continue; } - // Create an entry for each possible field, since we don't know what type this is for - for (let possibleField of possibleFields) { - fields.set( - possibleField, - { - originalField, - field: possibleField, - value + value = value.trim(); + + if (key == 'type') { + let possibleType = itemTypes.get(value); + if (possibleType) { + // Ignore item type that's the same as the item + if (!item || possibleType != Zotero.ItemTypes.getName(itemTypeID)) { + itemType = possibleType; + skipKeys.add(key); + continue; } - ); + } + } + + // Fields + let possibleFields = fieldNames.get(key); + // No valid fields + if (possibleFields) { + let added = false; + for (let possibleField of possibleFields) { + // If we have an item, skip fields that aren't valid for the type or that already + // have values + if (item) { + let fieldID = Zotero.ItemFields.getID(possibleField); + if (!Zotero.ItemFields.isValidForType(fieldID, itemTypeID) + || item.getField(fieldID) + || additionalFields.has(possibleField)) { + continue; + } + } + fields.set(possibleField, value); + added = true; + // If we found a valid field, don't try the other possibilities for that + // normalized key + if (item) { + break; + } + } + if (added) { + skipKeys.add(key); + continue; + } + } + + let possibleCreatorType = creatorTypes.get(key); + if (possibleCreatorType) { + let c = { + creatorType: possibleCreatorType + }; + if (value.includes('||')) { + let [first, last] = value.split(/\s*\|\|\s*/); + c.firstName = first; + c.lastName = last; + } + else { + c.name = value; + } + if (item) { + let creatorTypeID = Zotero.CreatorTypes.getID(possibleCreatorType); + if (Zotero.CreatorTypes.isValidForItemType(creatorTypeID, itemTypeID) + // Ignore if there are any creators of this type on the item already, + // to follow citeproc-js behavior + && !item.getCreators().some(x => x.creatorType == possibleCreatorType)) { + creators.push(c); + continue; + } + } + else { + creators.push(c); + continue; + } + } + + // We didn't find anything, so keep the line in Extra + keepLines.push(line); + } + + return { + itemType, + fields, + creators, + extra: keepLines.join('\n') + }; + }, + + + /** + * @param {String} extra + * @param {Map} fieldMap + * @return {String} + */ + combineExtraFields: function (extra, fields) { + var normalizedKeyMap = new Map(); + var normalizedFields = new Map(); + for (let [key, value] of fields) { + let normalizedKey = this._normalizeExtraKey(key); + normalizedFields.set(normalizedKey, value); + normalizedKeyMap.set(normalizedKey, key); + } + var keepLines = []; + var lines = extra !== '' ? extra.split(/\n/g) : []; + for (let line of lines) { + let parts = line.match(/^([a-z -_]+):(.+)/i); + // Old citeproc.js cheater syntax; + if (!parts) { + parts = line.match(/^{:([a-z -_]+):(.+)}/i); + } + if (!parts) { + keepLines.push(line); + continue; + } + let [_, originalField, value] = parts; + + let key = this._normalizeExtraKey(originalField); + + // If we have a new value for the field, update it + if (normalizedFields.has(key)) { + keepLines.push(originalField + ": " + normalizedFields.get(key)); + // Don't include with the other fields + fields.delete(normalizedKeyMap.get(key)); + } + else { + keepLines.push(line); } } - return fields; + var fieldPairs = Array.from(fields.entries()) + .map(x => x[0] + ': ' + x[1]); + fieldPairs.sort(); + return fieldPairs.join('\n') + + ((fieldPairs.length && keepLines.length) ? "\n" : "") + + keepLines.join("\n"); + }, + + + _normalizeExtraKey: function (key) { + return key + .trim() + // Convert fooBar to foo-bar + .replace(/([a-z])([A-Z])/g, '$1-$2') + .toLowerCase() + // Normalize to hyphens for spaces + .replace(/[\s-_]/g, '-'); }, diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index 440620db28..92b5a9444a 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -711,6 +711,7 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); yield Zotero.Users.init(); yield Zotero.Libraries.init(); + yield Zotero.ID.init(); yield Zotero.ItemTypes.init(); yield Zotero.ItemFields.init(); yield Zotero.CreatorTypes.init(); @@ -747,7 +748,6 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); Zotero.Date.init(); Zotero.LocateManager.init(); - yield Zotero.ID.init(); yield Zotero.Collections.init(); yield Zotero.Items.init(); yield Zotero.Searches.init(); @@ -757,6 +757,12 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); yield Zotero.Relations.init(); yield Zotero.Retractions.init(); + // Migrate fields from Extra that can be moved to item fields after a schema update + // + // Disabled for now + // + //yield Zotero.Schema.migrateExtraFields(); + // Load all library data except for items, which are loaded when libraries are first // clicked on or if otherwise necessary yield Zotero.Promise.each( @@ -769,7 +775,6 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); } })() ); - Zotero.Items.startEmptyTrashTimer(); diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 6a74bb11a5..3722fe154d 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -965,6 +965,8 @@ sync.error.creatorTooLong = The creator name “%S” in one of your i sync.error.noteEmbeddedImage = Notes with embedded images cannot currently be synced. Syncing of embedded images may be supported in a future version. sync.error.noteTooLong = The note “%S” is too long to sync. Shorten the note and sync again. sync.error.reportSiteIssuesToForums = If you receive this message repeatedly for items saved from a particular site, you can report this issue in the %S Forums. +sync.error.invalidDataError = Some data in %S could not be downloaded. It may have been saved with a newer version of %S. +sync.error.invalidDataError.otherData = Other data will continue to sync. account.unlinkWarning = Unlinking your account will prevent %S from syncing your data. account.unlinkWarning.removeData = Remove my %S data from this computer diff --git a/resource/pako.js b/resource/pako.js new file mode 120000 index 0000000000..2de85beb19 --- /dev/null +++ b/resource/pako.js @@ -0,0 +1 @@ +../node_modules/pako/dist/pako.min.js \ No newline at end of file diff --git a/resource/schema/global b/resource/schema/global new file mode 160000 index 0000000000..d6d3718e19 --- /dev/null +++ b/resource/schema/global @@ -0,0 +1 @@ +Subproject commit d6d3718e19d14dd17d9d0e14f7a09b416bd4b1fb diff --git a/resource/schema/system.sql b/resource/schema/system.sql index eaad2b52f7..d31ecc2399 100644 --- a/resource/schema/system.sql +++ b/resource/schema/system.sql @@ -172,971 +172,11 @@ CREATE TABLE syncObjectTypes ( ); CREATE INDEX syncObjectTypes_name ON syncObjectTypes(name); -DROP TABLE IF EXISTS transactionSets; -CREATE TABLE transactionSets ( - transactionSetID INTEGER PRIMARY KEY, - event TEXT, - id INT -); - -DROP TABLE IF EXISTS transactions; -CREATE TABLE transactions ( - transactionID INTEGER PRIMARY KEY, - transactionSetID INT, - context TEXT, - action TEXT -); -CREATE INDEX transactions_transactionSetID ON transactions(transactionSetID); - -DROP TABLE IF EXISTS transactionLog; -CREATE TABLE transactionLog ( - transactionID INT, - field TEXT, - value NONE, - PRIMARY KEY (transactionID, field, value), - FOREIGN KEY (transactionID) REFERENCES transactions(transactionID) -); - -- unused INSERT INTO "fieldFormats" VALUES(1, '.*', 0); INSERT INTO "fieldFormats" VALUES(2, '[0-9]*', 1); INSERT INTO "fieldFormats" VALUES(3, '[0-9]{4}', 1); -INSERT INTO itemTypes VALUES (1,'note',NULL,0); -INSERT INTO itemTypes VALUES (2,'book',NULL,2); -INSERT INTO itemTypes VALUES (3,'bookSection',2,2); -INSERT INTO itemTypes VALUES (4,'journalArticle',NULL,2); -INSERT INTO itemTypes VALUES (5,'magazineArticle',NULL,2); -INSERT INTO itemTypes VALUES (6,'newspaperArticle',NULL,2); -INSERT INTO itemTypes VALUES (7,'thesis',NULL,1); -INSERT INTO itemTypes VALUES (8,'letter',NULL,1); -INSERT INTO itemTypes VALUES (9,'manuscript',NULL,1); -INSERT INTO itemTypes VALUES (10,'interview',NULL,1); -INSERT INTO itemTypes VALUES (11,'film',NULL,1); -INSERT INTO itemTypes VALUES (12,'artwork',NULL,1); -INSERT INTO itemTypes VALUES (13,'webpage',NULL,0); -INSERT INTO itemTypes VALUES (14,'attachment',NULL,0); -INSERT INTO itemTypes VALUES (15,'report',NULL,1); -INSERT INTO itemTypes VALUES (16,'bill',NULL,1); -INSERT INTO itemTypes VALUES (17,'case',NULL,1); -INSERT INTO itemTypes VALUES (18,'hearing',NULL,1); -INSERT INTO itemTypes VALUES (19,'patent',NULL,1); -INSERT INTO itemTypes VALUES (20,'statute',NULL,1); -INSERT INTO itemTypes VALUES (21,'email',NULL,1); -INSERT INTO itemTypes VALUES (22,'map',NULL,1); -INSERT INTO itemTypes VALUES (23,'blogPost',NULL,1); -INSERT INTO itemTypes VALUES (24,'instantMessage',NULL,1); -INSERT INTO itemTypes VALUES (25,'forumPost',NULL,1); -INSERT INTO itemTypes VALUES (26,'audioRecording',NULL,1); -INSERT INTO itemTypes VALUES (27,'presentation',NULL,1); -INSERT INTO itemTypes VALUES (28,'videoRecording',NULL,1); -INSERT INTO itemTypes VALUES (29,'tvBroadcast',NULL,1); -INSERT INTO itemTypes VALUES (30,'radioBroadcast',NULL,1); -INSERT INTO itemTypes VALUES (31,'podcast',NULL,1); -INSERT INTO itemTypes VALUES (32,'computerProgram',NULL,1); -INSERT INTO itemTypes VALUES (33,'conferencePaper',NULL,1); -INSERT INTO itemTypes VALUES (34,'document',NULL,2); -INSERT INTO itemTypes VALUES (35,'encyclopediaArticle',NULL,1); -INSERT INTO itemTypes VALUES (36,'dictionaryEntry',NULL,1); - -INSERT INTO fields VALUES (1,'url',NULL); -INSERT INTO fields VALUES (2,'rights',NULL); -INSERT INTO fields VALUES (3,'series',NULL); -INSERT INTO fields VALUES (4,'volume',NULL); -INSERT INTO fields VALUES (5,'issue',NULL); -INSERT INTO fields VALUES (6,'edition',NULL); -INSERT INTO fields VALUES (7,'place',NULL); -INSERT INTO fields VALUES (8,'publisher',NULL); -INSERT INTO fields VALUES (10,'pages',NULL); -INSERT INTO fields VALUES (11,'ISBN',NULL); -INSERT INTO fields VALUES (12,'publicationTitle',NULL); -INSERT INTO fields VALUES (13,'ISSN',NULL); -INSERT INTO fields VALUES (14,'date',NULL); -INSERT INTO fields VALUES (15,'section',NULL); -INSERT INTO fields VALUES (18,'callNumber',NULL); -INSERT INTO fields VALUES (19,'archiveLocation',NULL); -INSERT INTO fields VALUES (21,'distributor',NULL); -INSERT INTO fields VALUES (22,'extra',NULL); -INSERT INTO fields VALUES (25,'journalAbbreviation',NULL); -INSERT INTO fields VALUES (26,'DOI',NULL); -INSERT INTO fields VALUES (27,'accessDate',NULL); -INSERT INTO fields VALUES (28,'seriesTitle',NULL); -INSERT INTO fields VALUES (29,'seriesText',NULL); -INSERT INTO fields VALUES (30,'seriesNumber',NULL); -INSERT INTO fields VALUES (31,'institution',NULL); -INSERT INTO fields VALUES (32,'reportType',NULL); -INSERT INTO fields VALUES (36,'code',NULL); -INSERT INTO fields VALUES (40,'session',NULL); -INSERT INTO fields VALUES (41,'legislativeBody',NULL); -INSERT INTO fields VALUES (42,'history',NULL); -INSERT INTO fields VALUES (43,'reporter',NULL); -INSERT INTO fields VALUES (44,'court',NULL); -INSERT INTO fields VALUES (45,'numberOfVolumes',NULL); -INSERT INTO fields VALUES (46,'committee',NULL); -INSERT INTO fields VALUES (48,'assignee',NULL); -INSERT INTO fields VALUES (50,'patentNumber',NULL); -INSERT INTO fields VALUES (51,'priorityNumbers',NULL); -INSERT INTO fields VALUES (52,'issueDate',NULL); -INSERT INTO fields VALUES (53,'references',NULL); -INSERT INTO fields VALUES (54,'legalStatus',NULL); -INSERT INTO fields VALUES (55,'codeNumber',NULL); -INSERT INTO fields VALUES (59,'artworkMedium',NULL); -INSERT INTO fields VALUES (60,'number',NULL); -INSERT INTO fields VALUES (61,'artworkSize',NULL); -INSERT INTO fields VALUES (62,'libraryCatalog',NULL); -INSERT INTO fields VALUES (63,'videoRecordingFormat',NULL); -INSERT INTO fields VALUES (64,'interviewMedium',NULL); -INSERT INTO fields VALUES (65,'letterType',NULL); -INSERT INTO fields VALUES (66,'manuscriptType',NULL); -INSERT INTO fields VALUES (67,'mapType',NULL); -INSERT INTO fields VALUES (68,'scale',NULL); -INSERT INTO fields VALUES (69,'thesisType',NULL); -INSERT INTO fields VALUES (70,'websiteType',NULL); -INSERT INTO fields VALUES (71,'audioRecordingFormat',NULL); -INSERT INTO fields VALUES (72,'label',NULL); -INSERT INTO fields VALUES (74,'presentationType',NULL); -INSERT INTO fields VALUES (75,'meetingName',NULL); -INSERT INTO fields VALUES (76,'studio',NULL); -INSERT INTO fields VALUES (77,'runningTime',NULL); -INSERT INTO fields VALUES (78,'network',NULL); -INSERT INTO fields VALUES (79,'postType',NULL); -INSERT INTO fields VALUES (80,'audioFileType',NULL); -INSERT INTO fields VALUES (81,'versionNumber',NULL); -INSERT INTO fields VALUES (82,'system',NULL); -INSERT INTO fields VALUES (83,'company',NULL); -INSERT INTO fields VALUES (84,'conferenceName',NULL); -INSERT INTO fields VALUES (85,'encyclopediaTitle',NULL); -INSERT INTO fields VALUES (86,'dictionaryTitle',NULL); -INSERT INTO fields VALUES (87,'language',NULL); -INSERT INTO fields VALUES (88,'programmingLanguage',NULL); -INSERT INTO fields VALUES (89,'university',NULL); -INSERT INTO fields VALUES (90,'abstractNote',NULL); -INSERT INTO fields VALUES (91,'websiteTitle',NULL); -INSERT INTO fields VALUES (92,'reportNumber',NULL); -INSERT INTO fields VALUES (93,'billNumber',NULL); -INSERT INTO fields VALUES (94,'codeVolume',NULL); -INSERT INTO fields VALUES (95,'codePages',NULL); -INSERT INTO fields VALUES (96,'dateDecided',NULL); -INSERT INTO fields VALUES (97,'reporterVolume',NULL); -INSERT INTO fields VALUES (98,'firstPage',NULL); -INSERT INTO fields VALUES (99,'documentNumber',NULL); -INSERT INTO fields VALUES (100,'dateEnacted',NULL); -INSERT INTO fields VALUES (101,'publicLawNumber',NULL); -INSERT INTO fields VALUES (102,'country',NULL); -INSERT INTO fields VALUES (103,'applicationNumber',NULL); -INSERT INTO fields VALUES (104,'forumTitle',NULL); -INSERT INTO fields VALUES (105,'episodeNumber',NULL); -INSERT INTO fields VALUES (107,'blogTitle',NULL); -INSERT INTO fields VALUES (108,'type',NULL); -INSERT INTO fields VALUES (109,'medium',NULL); -INSERT INTO fields VALUES (110,'title',NULL); -INSERT INTO fields VALUES (111,'caseName',NULL); -INSERT INTO fields VALUES (112,'nameOfAct',NULL); -INSERT INTO fields VALUES (113,'subject',NULL); -INSERT INTO fields VALUES (114,'proceedingsTitle',NULL); -INSERT INTO fields VALUES (115,'bookTitle',NULL); -INSERT INTO fields VALUES (116,'shortTitle',NULL); -INSERT INTO fields VALUES (117,'docketNumber',NULL); -INSERT INTO fields VALUES (118,'numPages',NULL); -INSERT INTO fields VALUES (119,'programTitle',NULL); -INSERT INTO fields VALUES (120,'issuingAuthority',NULL); -INSERT INTO fields VALUES (121,'filingDate',NULL); -INSERT INTO fields VALUES (122,'genre',NULL); -INSERT INTO fields VALUES (123,'archive',NULL); - -INSERT INTO itemTypeFields VALUES (2, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (2, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (2, 3, NULL, 3); -INSERT INTO itemTypeFields VALUES (2, 30, NULL, 4); -INSERT INTO itemTypeFields VALUES (2, 4, NULL, 5); -INSERT INTO itemTypeFields VALUES (2, 45, NULL, 6); -INSERT INTO itemTypeFields VALUES (2, 6, NULL, 7); -INSERT INTO itemTypeFields VALUES (2, 7, NULL, 8); -INSERT INTO itemTypeFields VALUES (2, 8, NULL, 9); -INSERT INTO itemTypeFields VALUES (2, 14, NULL, 10); -INSERT INTO itemTypeFields VALUES (2, 118, NULL, 11); -INSERT INTO itemTypeFields VALUES (2, 87, NULL, 12); -INSERT INTO itemTypeFields VALUES (2, 11, NULL, 13); -INSERT INTO itemTypeFields VALUES (2, 116, NULL, 14); -INSERT INTO itemTypeFields VALUES (2, 1, NULL, 15); -INSERT INTO itemTypeFields VALUES (2, 27, NULL, 16); -INSERT INTO itemTypeFields VALUES (2, 123, NULL, 17); -INSERT INTO itemTypeFields VALUES (2, 19, NULL, 18); -INSERT INTO itemTypeFields VALUES (2, 62, NULL, 19); -INSERT INTO itemTypeFields VALUES (2, 18, NULL, 20); -INSERT INTO itemTypeFields VALUES (2, 2, NULL, 21); -INSERT INTO itemTypeFields VALUES (2, 22, NULL, 22); -INSERT INTO itemTypeFields VALUES (3, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (3, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (3, 115, NULL, 3); -INSERT INTO itemTypeFields VALUES (3, 3, NULL, 4); -INSERT INTO itemTypeFields VALUES (3, 30, NULL, 5); -INSERT INTO itemTypeFields VALUES (3, 4, NULL, 6); -INSERT INTO itemTypeFields VALUES (3, 45, NULL, 7); -INSERT INTO itemTypeFields VALUES (3, 6, NULL, 8); -INSERT INTO itemTypeFields VALUES (3, 7, NULL, 9); -INSERT INTO itemTypeFields VALUES (3, 8, NULL, 10); -INSERT INTO itemTypeFields VALUES (3, 14, NULL, 11); -INSERT INTO itemTypeFields VALUES (3, 10, NULL, 12); -INSERT INTO itemTypeFields VALUES (3, 87, NULL, 13); -INSERT INTO itemTypeFields VALUES (3, 11, NULL, 14); -INSERT INTO itemTypeFields VALUES (3, 116, NULL, 15); -INSERT INTO itemTypeFields VALUES (3, 1, NULL, 16); -INSERT INTO itemTypeFields VALUES (3, 27, NULL, 17); -INSERT INTO itemTypeFields VALUES (3, 123, NULL, 18); -INSERT INTO itemTypeFields VALUES (3, 19, NULL, 19); -INSERT INTO itemTypeFields VALUES (3, 62, NULL, 20); -INSERT INTO itemTypeFields VALUES (3, 18, NULL, 21); -INSERT INTO itemTypeFields VALUES (3, 2, NULL, 22); -INSERT INTO itemTypeFields VALUES (3, 22, NULL, 23); -INSERT INTO itemTypeFields VALUES (4, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (4, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (4, 12, NULL, 3); -INSERT INTO itemTypeFields VALUES (4, 4, NULL, 4); -INSERT INTO itemTypeFields VALUES (4, 5, NULL, 5); -INSERT INTO itemTypeFields VALUES (4, 10, NULL, 6); -INSERT INTO itemTypeFields VALUES (4, 14, NULL, 7); -INSERT INTO itemTypeFields VALUES (4, 3, NULL, 8); -INSERT INTO itemTypeFields VALUES (4, 28, NULL, 9); -INSERT INTO itemTypeFields VALUES (4, 29, NULL, 10); -INSERT INTO itemTypeFields VALUES (4, 25, NULL, 11); -INSERT INTO itemTypeFields VALUES (4, 87, NULL, 12); -INSERT INTO itemTypeFields VALUES (4, 26, NULL, 13); -INSERT INTO itemTypeFields VALUES (4, 13, NULL, 14); -INSERT INTO itemTypeFields VALUES (4, 116, NULL, 15); -INSERT INTO itemTypeFields VALUES (4, 1, NULL, 16); -INSERT INTO itemTypeFields VALUES (4, 27, NULL, 17); -INSERT INTO itemTypeFields VALUES (4, 123, NULL, 18); -INSERT INTO itemTypeFields VALUES (4, 19, NULL, 19); -INSERT INTO itemTypeFields VALUES (4, 62, NULL, 20); -INSERT INTO itemTypeFields VALUES (4, 18, NULL, 21); -INSERT INTO itemTypeFields VALUES (4, 2, NULL, 22); -INSERT INTO itemTypeFields VALUES (4, 22, NULL, 23); -INSERT INTO itemTypeFields VALUES (5, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (5, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (5, 12, NULL, 3); -INSERT INTO itemTypeFields VALUES (5, 4, NULL, 4); -INSERT INTO itemTypeFields VALUES (5, 5, NULL, 5); -INSERT INTO itemTypeFields VALUES (5, 14, NULL, 6); -INSERT INTO itemTypeFields VALUES (5, 10, NULL, 7); -INSERT INTO itemTypeFields VALUES (5, 87, NULL, 8); -INSERT INTO itemTypeFields VALUES (5, 13, NULL, 9); -INSERT INTO itemTypeFields VALUES (5, 116, NULL, 10); -INSERT INTO itemTypeFields VALUES (5, 1, NULL, 11); -INSERT INTO itemTypeFields VALUES (5, 27, NULL, 12); -INSERT INTO itemTypeFields VALUES (5, 123, NULL, 13); -INSERT INTO itemTypeFields VALUES (5, 19, NULL, 14); -INSERT INTO itemTypeFields VALUES (5, 62, NULL, 15); -INSERT INTO itemTypeFields VALUES (5, 18, NULL, 16); -INSERT INTO itemTypeFields VALUES (5, 2, NULL, 17); -INSERT INTO itemTypeFields VALUES (5, 22, NULL, 18); -INSERT INTO itemTypeFields VALUES (6, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (6, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (6, 12, NULL, 3); -INSERT INTO itemTypeFields VALUES (6, 7, NULL, 4); -INSERT INTO itemTypeFields VALUES (6, 6, NULL, 5); -INSERT INTO itemTypeFields VALUES (6, 14, NULL, 6); -INSERT INTO itemTypeFields VALUES (6, 15, NULL, 7); -INSERT INTO itemTypeFields VALUES (6, 10, NULL, 8); -INSERT INTO itemTypeFields VALUES (6, 87, NULL, 9); -INSERT INTO itemTypeFields VALUES (6, 116, NULL, 10); -INSERT INTO itemTypeFields VALUES (6, 13, NULL, 11); -INSERT INTO itemTypeFields VALUES (6, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (6, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (6, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (6, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (6, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (6, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (6, 2, NULL, 18); -INSERT INTO itemTypeFields VALUES (6, 22, NULL, 19); -INSERT INTO itemTypeFields VALUES (7, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (7, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (7, 69, NULL, 3); -INSERT INTO itemTypeFields VALUES (7, 89, NULL, 4); -INSERT INTO itemTypeFields VALUES (7, 7, NULL, 5); -INSERT INTO itemTypeFields VALUES (7, 14, NULL, 6); -INSERT INTO itemTypeFields VALUES (7, 118, NULL, 7); -INSERT INTO itemTypeFields VALUES (7, 87, NULL, 8); -INSERT INTO itemTypeFields VALUES (7, 116, NULL, 9); -INSERT INTO itemTypeFields VALUES (7, 1, NULL, 10); -INSERT INTO itemTypeFields VALUES (7, 27, NULL, 11); -INSERT INTO itemTypeFields VALUES (7, 123, NULL, 12); -INSERT INTO itemTypeFields VALUES (7, 19, NULL, 13); -INSERT INTO itemTypeFields VALUES (7, 62, NULL, 14); -INSERT INTO itemTypeFields VALUES (7, 18, NULL, 15); -INSERT INTO itemTypeFields VALUES (7, 2, NULL, 16); -INSERT INTO itemTypeFields VALUES (7, 22, NULL, 17); -INSERT INTO itemTypeFields VALUES (8, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (8, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (8, 65, NULL, 3); -INSERT INTO itemTypeFields VALUES (8, 14, NULL, 4); -INSERT INTO itemTypeFields VALUES (8, 87, NULL, 5); -INSERT INTO itemTypeFields VALUES (8, 116, NULL, 6); -INSERT INTO itemTypeFields VALUES (8, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (8, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (8, 123, NULL, 9); -INSERT INTO itemTypeFields VALUES (8, 19, NULL, 10); -INSERT INTO itemTypeFields VALUES (8, 62, NULL, 11); -INSERT INTO itemTypeFields VALUES (8, 18, NULL, 12); -INSERT INTO itemTypeFields VALUES (8, 2, NULL, 13); -INSERT INTO itemTypeFields VALUES (8, 22, NULL, 14); -INSERT INTO itemTypeFields VALUES (9, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (9, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (9, 66, NULL, 3); -INSERT INTO itemTypeFields VALUES (9, 7, NULL, 4); -INSERT INTO itemTypeFields VALUES (9, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (9, 118, NULL, 6); -INSERT INTO itemTypeFields VALUES (9, 87, NULL, 7); -INSERT INTO itemTypeFields VALUES (9, 116, NULL, 8); -INSERT INTO itemTypeFields VALUES (9, 1, NULL, 9); -INSERT INTO itemTypeFields VALUES (9, 27, NULL, 10); -INSERT INTO itemTypeFields VALUES (9, 123, NULL, 11); -INSERT INTO itemTypeFields VALUES (9, 19, NULL, 12); -INSERT INTO itemTypeFields VALUES (9, 62, NULL, 13); -INSERT INTO itemTypeFields VALUES (9, 18, NULL, 14); -INSERT INTO itemTypeFields VALUES (9, 2, NULL, 15); -INSERT INTO itemTypeFields VALUES (9, 22, NULL, 16); -INSERT INTO itemTypeFields VALUES (10, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (10, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (10, 14, NULL, 3); -INSERT INTO itemTypeFields VALUES (10, 64, NULL, 4); -INSERT INTO itemTypeFields VALUES (10, 87, NULL, 5); -INSERT INTO itemTypeFields VALUES (10, 116, NULL, 6); -INSERT INTO itemTypeFields VALUES (10, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (10, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (10, 123, NULL, 9); -INSERT INTO itemTypeFields VALUES (10, 19, NULL, 10); -INSERT INTO itemTypeFields VALUES (10, 62, NULL, 11); -INSERT INTO itemTypeFields VALUES (10, 18, NULL, 12); -INSERT INTO itemTypeFields VALUES (10, 2, NULL, 13); -INSERT INTO itemTypeFields VALUES (10, 22, NULL, 14); -INSERT INTO itemTypeFields VALUES (11, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (11, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (11, 21, NULL, 3); -INSERT INTO itemTypeFields VALUES (11, 14, NULL, 4); -INSERT INTO itemTypeFields VALUES (11, 122, NULL, 5); -INSERT INTO itemTypeFields VALUES (11, 63, NULL, 6); -INSERT INTO itemTypeFields VALUES (11, 77, NULL, 7); -INSERT INTO itemTypeFields VALUES (11, 87, NULL, 8); -INSERT INTO itemTypeFields VALUES (11, 116, NULL, 9); -INSERT INTO itemTypeFields VALUES (11, 1, NULL, 10); -INSERT INTO itemTypeFields VALUES (11, 27, NULL, 11); -INSERT INTO itemTypeFields VALUES (11, 123, NULL, 12); -INSERT INTO itemTypeFields VALUES (11, 19, NULL, 13); -INSERT INTO itemTypeFields VALUES (11, 62, NULL, 14); -INSERT INTO itemTypeFields VALUES (11, 18, NULL, 15); -INSERT INTO itemTypeFields VALUES (11, 2, NULL, 16); -INSERT INTO itemTypeFields VALUES (11, 22, NULL, 17); -INSERT INTO itemTypeFields VALUES (12, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (12, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (12, 59, NULL, 3); -INSERT INTO itemTypeFields VALUES (12, 61, NULL, 4); -INSERT INTO itemTypeFields VALUES (12, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (12, 87, NULL, 6); -INSERT INTO itemTypeFields VALUES (12, 116, NULL, 7); -INSERT INTO itemTypeFields VALUES (12, 123, NULL, 8); -INSERT INTO itemTypeFields VALUES (12, 19, NULL, 9); -INSERT INTO itemTypeFields VALUES (12, 62, NULL, 10); -INSERT INTO itemTypeFields VALUES (12, 18, NULL, 11); -INSERT INTO itemTypeFields VALUES (12, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (12, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (12, 2, NULL, 14); -INSERT INTO itemTypeFields VALUES (12, 22, NULL, 15); -INSERT INTO itemTypeFields VALUES (13, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (13, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (13, 91, NULL, 3); -INSERT INTO itemTypeFields VALUES (13, 70, NULL, 4); -INSERT INTO itemTypeFields VALUES (13, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (13, 116, NULL, 6); -INSERT INTO itemTypeFields VALUES (13, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (13, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (13, 87, NULL, 9); -INSERT INTO itemTypeFields VALUES (13, 2, NULL, 10); -INSERT INTO itemTypeFields VALUES (13, 22, NULL, 11); -INSERT INTO itemTypeFields VALUES (14, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (14, 27, NULL, 2); -INSERT INTO itemTypeFields VALUES (14, 1, NULL, 3); -INSERT INTO itemTypeFields VALUES (15, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (15, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (15, 92, NULL, 3); -INSERT INTO itemTypeFields VALUES (15, 32, NULL, 4); -INSERT INTO itemTypeFields VALUES (15, 28, NULL, 5); -INSERT INTO itemTypeFields VALUES (15, 7, NULL, 6); -INSERT INTO itemTypeFields VALUES (15, 31, NULL, 7); -INSERT INTO itemTypeFields VALUES (15, 14, NULL, 8); -INSERT INTO itemTypeFields VALUES (15, 10, NULL, 9); -INSERT INTO itemTypeFields VALUES (15, 87, NULL, 10); -INSERT INTO itemTypeFields VALUES (15, 116, NULL, 11); -INSERT INTO itemTypeFields VALUES (15, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (15, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (15, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (15, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (15, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (15, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (15, 2, NULL, 18); -INSERT INTO itemTypeFields VALUES (15, 22, NULL, 19); -INSERT INTO itemTypeFields VALUES (16, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (16, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (16, 93, NULL, 3); -INSERT INTO itemTypeFields VALUES (16, 36, NULL, 4); -INSERT INTO itemTypeFields VALUES (16, 94, NULL, 5); -INSERT INTO itemTypeFields VALUES (16, 15, NULL, 6); -INSERT INTO itemTypeFields VALUES (16, 95, NULL, 7); -INSERT INTO itemTypeFields VALUES (16, 41, NULL, 8); -INSERT INTO itemTypeFields VALUES (16, 40, NULL, 9); -INSERT INTO itemTypeFields VALUES (16, 42, NULL, 10); -INSERT INTO itemTypeFields VALUES (16, 14, NULL, 11); -INSERT INTO itemTypeFields VALUES (16, 87, NULL, 12); -INSERT INTO itemTypeFields VALUES (16, 1, NULL, 13); -INSERT INTO itemTypeFields VALUES (16, 27, NULL, 14); -INSERT INTO itemTypeFields VALUES (16, 116, NULL, 15); -INSERT INTO itemTypeFields VALUES (16, 2, NULL, 16); -INSERT INTO itemTypeFields VALUES (16, 22, NULL, 17); -INSERT INTO itemTypeFields VALUES (17, 111, NULL, 1); -INSERT INTO itemTypeFields VALUES (17, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (17, 44, NULL, 3); -INSERT INTO itemTypeFields VALUES (17, 96, NULL, 4); -INSERT INTO itemTypeFields VALUES (17, 117, NULL, 5); -INSERT INTO itemTypeFields VALUES (17, 43, NULL, 6); -INSERT INTO itemTypeFields VALUES (17, 97, NULL, 7); -INSERT INTO itemTypeFields VALUES (17, 98, NULL, 8); -INSERT INTO itemTypeFields VALUES (17, 42, NULL, 9); -INSERT INTO itemTypeFields VALUES (17, 87, NULL, 10); -INSERT INTO itemTypeFields VALUES (17, 116, NULL, 11); -INSERT INTO itemTypeFields VALUES (17, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (17, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (17, 2, NULL, 14); -INSERT INTO itemTypeFields VALUES (17, 22, NULL, 15); -INSERT INTO itemTypeFields VALUES (18, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (18, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (18, 46, NULL, 3); -INSERT INTO itemTypeFields VALUES (18, 7, NULL, 4); -INSERT INTO itemTypeFields VALUES (18, 8, NULL, 5); -INSERT INTO itemTypeFields VALUES (18, 45, NULL, 6); -INSERT INTO itemTypeFields VALUES (18, 99, NULL, 7); -INSERT INTO itemTypeFields VALUES (18, 10, NULL, 8); -INSERT INTO itemTypeFields VALUES (18, 41, NULL, 9); -INSERT INTO itemTypeFields VALUES (18, 40, NULL, 10); -INSERT INTO itemTypeFields VALUES (18, 42, NULL, 11); -INSERT INTO itemTypeFields VALUES (18, 14, NULL, 12); -INSERT INTO itemTypeFields VALUES (18, 87, NULL, 13); -INSERT INTO itemTypeFields VALUES (18, 116, NULL, 14); -INSERT INTO itemTypeFields VALUES (18, 1, NULL, 15); -INSERT INTO itemTypeFields VALUES (18, 27, NULL, 16); -INSERT INTO itemTypeFields VALUES (18, 2, NULL, 17); -INSERT INTO itemTypeFields VALUES (18, 22, NULL, 18); -INSERT INTO itemTypeFields VALUES (19, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (19, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (19, 7, NULL, 3); -INSERT INTO itemTypeFields VALUES (19, 102, NULL, 4); -INSERT INTO itemTypeFields VALUES (19, 48, NULL, 5); -INSERT INTO itemTypeFields VALUES (19, 120, NULL, 6); -INSERT INTO itemTypeFields VALUES (19, 50, NULL, 7); -INSERT INTO itemTypeFields VALUES (19, 121, NULL, 8); -INSERT INTO itemTypeFields VALUES (19, 10, NULL, 9); -INSERT INTO itemTypeFields VALUES (19, 103, NULL, 10); -INSERT INTO itemTypeFields VALUES (19, 51, NULL, 11); -INSERT INTO itemTypeFields VALUES (19, 52, NULL, 12); -INSERT INTO itemTypeFields VALUES (19, 53, NULL, 13); -INSERT INTO itemTypeFields VALUES (19, 54, NULL, 14); -INSERT INTO itemTypeFields VALUES (19, 87, NULL, 15); -INSERT INTO itemTypeFields VALUES (19, 116, NULL, 16); -INSERT INTO itemTypeFields VALUES (19, 1, NULL, 17); -INSERT INTO itemTypeFields VALUES (19, 27, NULL, 18); -INSERT INTO itemTypeFields VALUES (19, 2, NULL, 19); -INSERT INTO itemTypeFields VALUES (19, 22, NULL, 20); -INSERT INTO itemTypeFields VALUES (20, 112, NULL, 1); -INSERT INTO itemTypeFields VALUES (20, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (20, 36, NULL, 3); -INSERT INTO itemTypeFields VALUES (20, 55, NULL, 4); -INSERT INTO itemTypeFields VALUES (20, 101, NULL, 5); -INSERT INTO itemTypeFields VALUES (20, 100, NULL, 6); -INSERT INTO itemTypeFields VALUES (20, 10, NULL, 7); -INSERT INTO itemTypeFields VALUES (20, 15, NULL, 8); -INSERT INTO itemTypeFields VALUES (20, 40, NULL, 9); -INSERT INTO itemTypeFields VALUES (20, 42, NULL, 10); -INSERT INTO itemTypeFields VALUES (20, 87, NULL, 11); -INSERT INTO itemTypeFields VALUES (20, 116, NULL, 12); -INSERT INTO itemTypeFields VALUES (20, 1, NULL, 13); -INSERT INTO itemTypeFields VALUES (20, 27, NULL, 14); -INSERT INTO itemTypeFields VALUES (20, 2, NULL, 15); -INSERT INTO itemTypeFields VALUES (20, 22, NULL, 16); -INSERT INTO itemTypeFields VALUES (21, 113, NULL, 1); -INSERT INTO itemTypeFields VALUES (21, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (21, 14, NULL, 3); -INSERT INTO itemTypeFields VALUES (21, 116, NULL, 4); -INSERT INTO itemTypeFields VALUES (21, 1, NULL, 5); -INSERT INTO itemTypeFields VALUES (21, 27, NULL, 6); -INSERT INTO itemTypeFields VALUES (21, 87, NULL, 7); -INSERT INTO itemTypeFields VALUES (21, 2, NULL, 8); -INSERT INTO itemTypeFields VALUES (21, 22, NULL, 9); -INSERT INTO itemTypeFields VALUES (22, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (22, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (22, 67, NULL, 3); -INSERT INTO itemTypeFields VALUES (22, 68, NULL, 4); -INSERT INTO itemTypeFields VALUES (22, 28, NULL, 5); -INSERT INTO itemTypeFields VALUES (22, 6, NULL, 6); -INSERT INTO itemTypeFields VALUES (22, 7, NULL, 7); -INSERT INTO itemTypeFields VALUES (22, 8, NULL, 8); -INSERT INTO itemTypeFields VALUES (22, 14, NULL, 9); -INSERT INTO itemTypeFields VALUES (22, 87, NULL, 10); -INSERT INTO itemTypeFields VALUES (22, 11, NULL, 11); -INSERT INTO itemTypeFields VALUES (22, 116, NULL, 12); -INSERT INTO itemTypeFields VALUES (22, 1, NULL, 13); -INSERT INTO itemTypeFields VALUES (22, 27, NULL, 14); -INSERT INTO itemTypeFields VALUES (22, 123, NULL, 15); -INSERT INTO itemTypeFields VALUES (22, 19, NULL, 16); -INSERT INTO itemTypeFields VALUES (22, 62, NULL, 17); -INSERT INTO itemTypeFields VALUES (22, 18, NULL, 18); -INSERT INTO itemTypeFields VALUES (22, 2, NULL, 19); -INSERT INTO itemTypeFields VALUES (22, 22, NULL, 20); -INSERT INTO itemTypeFields VALUES (23, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (23, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (23, 107, NULL, 3); -INSERT INTO itemTypeFields VALUES (23, 70, NULL, 4); -INSERT INTO itemTypeFields VALUES (23, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (23, 1, NULL, 6); -INSERT INTO itemTypeFields VALUES (23, 27, NULL, 7); -INSERT INTO itemTypeFields VALUES (23, 87, NULL, 8); -INSERT INTO itemTypeFields VALUES (23, 116, NULL, 9); -INSERT INTO itemTypeFields VALUES (23, 2, NULL, 10); -INSERT INTO itemTypeFields VALUES (23, 22, NULL, 11); -INSERT INTO itemTypeFields VALUES (24, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (24, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (24, 14, NULL, 3); -INSERT INTO itemTypeFields VALUES (24, 87, NULL, 4); -INSERT INTO itemTypeFields VALUES (24, 116, NULL, 5); -INSERT INTO itemTypeFields VALUES (24, 1, NULL, 6); -INSERT INTO itemTypeFields VALUES (24, 27, NULL, 7); -INSERT INTO itemTypeFields VALUES (24, 2, NULL, 8); -INSERT INTO itemTypeFields VALUES (24, 22, NULL, 9); -INSERT INTO itemTypeFields VALUES (25, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (25, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (25, 104, NULL, 3); -INSERT INTO itemTypeFields VALUES (25, 79, NULL, 4); -INSERT INTO itemTypeFields VALUES (25, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (25, 87, NULL, 6); -INSERT INTO itemTypeFields VALUES (25, 116, NULL, 7); -INSERT INTO itemTypeFields VALUES (25, 1, NULL, 8); -INSERT INTO itemTypeFields VALUES (25, 27, NULL, 9); -INSERT INTO itemTypeFields VALUES (25, 2, NULL, 10); -INSERT INTO itemTypeFields VALUES (25, 22, NULL, 11); -INSERT INTO itemTypeFields VALUES (26, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (26, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (26, 71, NULL, 3); -INSERT INTO itemTypeFields VALUES (26, 28, NULL, 4); -INSERT INTO itemTypeFields VALUES (26, 4, NULL, 5); -INSERT INTO itemTypeFields VALUES (26, 45, NULL, 6); -INSERT INTO itemTypeFields VALUES (26, 7, NULL, 7); -INSERT INTO itemTypeFields VALUES (26, 72, NULL, 8); -INSERT INTO itemTypeFields VALUES (26, 14, NULL, 9); -INSERT INTO itemTypeFields VALUES (26, 77, NULL, 10); -INSERT INTO itemTypeFields VALUES (26, 87, NULL, 11); -INSERT INTO itemTypeFields VALUES (26, 11, NULL, 12); -INSERT INTO itemTypeFields VALUES (26, 116, NULL, 13); -INSERT INTO itemTypeFields VALUES (26, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (26, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (26, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (26, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (26, 1, NULL, 18); -INSERT INTO itemTypeFields VALUES (26, 27, NULL, 19); -INSERT INTO itemTypeFields VALUES (26, 2, NULL, 20); -INSERT INTO itemTypeFields VALUES (26, 22, NULL, 21); -INSERT INTO itemTypeFields VALUES (27, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (27, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (27, 74, NULL, 3); -INSERT INTO itemTypeFields VALUES (27, 14, NULL, 4); -INSERT INTO itemTypeFields VALUES (27, 7, NULL, 5); -INSERT INTO itemTypeFields VALUES (27, 75, NULL, 6); -INSERT INTO itemTypeFields VALUES (27, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (27, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (27, 87, NULL, 9); -INSERT INTO itemTypeFields VALUES (27, 116, NULL, 10); -INSERT INTO itemTypeFields VALUES (27, 2, NULL, 11); -INSERT INTO itemTypeFields VALUES (27, 22, NULL, 12); -INSERT INTO itemTypeFields VALUES (28, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (28, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (28, 63, NULL, 3); -INSERT INTO itemTypeFields VALUES (28, 28, NULL, 4); -INSERT INTO itemTypeFields VALUES (28, 4, NULL, 5); -INSERT INTO itemTypeFields VALUES (28, 45, NULL, 6); -INSERT INTO itemTypeFields VALUES (28, 7, NULL, 7); -INSERT INTO itemTypeFields VALUES (28, 76, NULL, 8); -INSERT INTO itemTypeFields VALUES (28, 14, NULL, 9); -INSERT INTO itemTypeFields VALUES (28, 77, NULL, 10); -INSERT INTO itemTypeFields VALUES (28, 87, NULL, 11); -INSERT INTO itemTypeFields VALUES (28, 11, NULL, 12); -INSERT INTO itemTypeFields VALUES (28, 116, NULL, 13); -INSERT INTO itemTypeFields VALUES (28, 1, NULL, 14); -INSERT INTO itemTypeFields VALUES (28, 27, NULL, 15); -INSERT INTO itemTypeFields VALUES (28, 123, NULL, 16); -INSERT INTO itemTypeFields VALUES (28, 19, NULL, 17); -INSERT INTO itemTypeFields VALUES (28, 62, NULL, 18); -INSERT INTO itemTypeFields VALUES (28, 18, NULL, 19); -INSERT INTO itemTypeFields VALUES (28, 2, NULL, 20); -INSERT INTO itemTypeFields VALUES (28, 22, NULL, 21); -INSERT INTO itemTypeFields VALUES (29, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (29, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (29, 119, NULL, 3); -INSERT INTO itemTypeFields VALUES (29, 105, NULL, 4); -INSERT INTO itemTypeFields VALUES (29, 63, NULL, 5); -INSERT INTO itemTypeFields VALUES (29, 7, NULL, 6); -INSERT INTO itemTypeFields VALUES (29, 78, NULL, 7); -INSERT INTO itemTypeFields VALUES (29, 14, NULL, 8); -INSERT INTO itemTypeFields VALUES (29, 77, NULL, 9); -INSERT INTO itemTypeFields VALUES (29, 87, NULL, 10); -INSERT INTO itemTypeFields VALUES (29, 116, NULL, 11); -INSERT INTO itemTypeFields VALUES (29, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (29, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (29, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (29, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (29, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (29, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (29, 2, NULL, 18); -INSERT INTO itemTypeFields VALUES (29, 22, NULL, 19); -INSERT INTO itemTypeFields VALUES (30, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (30, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (30, 119, NULL, 3); -INSERT INTO itemTypeFields VALUES (30, 105, NULL, 4); -INSERT INTO itemTypeFields VALUES (30, 71, NULL, 5); -INSERT INTO itemTypeFields VALUES (30, 7, NULL, 6); -INSERT INTO itemTypeFields VALUES (30, 78, NULL, 7); -INSERT INTO itemTypeFields VALUES (30, 14, NULL, 8); -INSERT INTO itemTypeFields VALUES (30, 77, NULL, 9); -INSERT INTO itemTypeFields VALUES (30, 87, NULL, 10); -INSERT INTO itemTypeFields VALUES (30, 116, NULL, 11); -INSERT INTO itemTypeFields VALUES (30, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (30, 27, NULL, 13); -INSERT INTO itemTypeFields VALUES (30, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (30, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (30, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (30, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (30, 2, NULL, 18); -INSERT INTO itemTypeFields VALUES (30, 22, NULL, 19); -INSERT INTO itemTypeFields VALUES (31, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (31, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (31, 28, NULL, 3); -INSERT INTO itemTypeFields VALUES (31, 105, NULL, 4); -INSERT INTO itemTypeFields VALUES (31, 80, NULL, 5); -INSERT INTO itemTypeFields VALUES (31, 77, NULL, 6); -INSERT INTO itemTypeFields VALUES (31, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (31, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (31, 87, NULL, 9); -INSERT INTO itemTypeFields VALUES (31, 116, NULL, 10); -INSERT INTO itemTypeFields VALUES (31, 2, NULL, 11); -INSERT INTO itemTypeFields VALUES (31, 22, NULL, 12); -INSERT INTO itemTypeFields VALUES (32, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (32, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (32, 28, NULL, 3); -INSERT INTO itemTypeFields VALUES (32, 81, NULL, 4); -INSERT INTO itemTypeFields VALUES (32, 14, NULL, 5); -INSERT INTO itemTypeFields VALUES (32, 82, NULL, 6); -INSERT INTO itemTypeFields VALUES (32, 7, NULL, 7); -INSERT INTO itemTypeFields VALUES (32, 83, NULL, 8); -INSERT INTO itemTypeFields VALUES (32, 88, NULL, 9); -INSERT INTO itemTypeFields VALUES (32, 11, NULL, 10); -INSERT INTO itemTypeFields VALUES (32, 116, NULL, 11); -INSERT INTO itemTypeFields VALUES (32, 1, NULL, 12); -INSERT INTO itemTypeFields VALUES (32, 2, NULL, 13); -INSERT INTO itemTypeFields VALUES (32, 123, NULL, 14); -INSERT INTO itemTypeFields VALUES (32, 19, NULL, 15); -INSERT INTO itemTypeFields VALUES (32, 62, NULL, 16); -INSERT INTO itemTypeFields VALUES (32, 18, NULL, 17); -INSERT INTO itemTypeFields VALUES (32, 27, NULL, 18); -INSERT INTO itemTypeFields VALUES (32, 22, NULL, 19); -INSERT INTO itemTypeFields VALUES (33, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (33, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (33, 14, NULL, 3); -INSERT INTO itemTypeFields VALUES (33, 114, NULL, 4); -INSERT INTO itemTypeFields VALUES (33, 84, NULL, 5); -INSERT INTO itemTypeFields VALUES (33, 7, NULL, 6); -INSERT INTO itemTypeFields VALUES (33, 8, NULL, 7); -INSERT INTO itemTypeFields VALUES (33, 4, NULL, 8); -INSERT INTO itemTypeFields VALUES (33, 10, NULL, 9); -INSERT INTO itemTypeFields VALUES (33, 3, NULL, 10); -INSERT INTO itemTypeFields VALUES (33, 87, NULL, 11); -INSERT INTO itemTypeFields VALUES (33, 26, NULL, 12); -INSERT INTO itemTypeFields VALUES (33, 11, NULL, 13); -INSERT INTO itemTypeFields VALUES (33, 116, NULL, 14); -INSERT INTO itemTypeFields VALUES (33, 1, NULL, 15); -INSERT INTO itemTypeFields VALUES (33, 27, NULL, 16); -INSERT INTO itemTypeFields VALUES (33, 123, NULL, 17); -INSERT INTO itemTypeFields VALUES (33, 19, NULL, 18); -INSERT INTO itemTypeFields VALUES (33, 62, NULL, 19); -INSERT INTO itemTypeFields VALUES (33, 18, NULL, 20); -INSERT INTO itemTypeFields VALUES (33, 2, NULL, 21); -INSERT INTO itemTypeFields VALUES (33, 22, NULL, 22); -INSERT INTO itemTypeFields VALUES (34, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (34, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (34, 8, NULL, 3); -INSERT INTO itemTypeFields VALUES (34, 14, NULL, 4); -INSERT INTO itemTypeFields VALUES (34, 87, NULL, 5); -INSERT INTO itemTypeFields VALUES (34, 116, NULL, 6); -INSERT INTO itemTypeFields VALUES (34, 1, NULL, 7); -INSERT INTO itemTypeFields VALUES (34, 27, NULL, 8); -INSERT INTO itemTypeFields VALUES (34, 123, NULL, 9); -INSERT INTO itemTypeFields VALUES (34, 19, NULL, 10); -INSERT INTO itemTypeFields VALUES (34, 62, NULL, 11); -INSERT INTO itemTypeFields VALUES (34, 18, NULL, 12); -INSERT INTO itemTypeFields VALUES (34, 2, NULL, 13); -INSERT INTO itemTypeFields VALUES (34, 22, NULL, 14); -INSERT INTO itemTypeFields VALUES (35, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (35, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (35, 85, NULL, 3); -INSERT INTO itemTypeFields VALUES (35, 3, NULL, 4); -INSERT INTO itemTypeFields VALUES (35, 30, NULL, 5); -INSERT INTO itemTypeFields VALUES (35, 4, NULL, 6); -INSERT INTO itemTypeFields VALUES (35, 45, NULL, 7); -INSERT INTO itemTypeFields VALUES (35, 6, NULL, 8); -INSERT INTO itemTypeFields VALUES (35, 7, NULL, 9); -INSERT INTO itemTypeFields VALUES (35, 8, NULL, 10); -INSERT INTO itemTypeFields VALUES (35, 14, NULL, 11); -INSERT INTO itemTypeFields VALUES (35, 10, NULL, 12); -INSERT INTO itemTypeFields VALUES (35, 11, NULL, 13); -INSERT INTO itemTypeFields VALUES (35, 116, NULL, 14); -INSERT INTO itemTypeFields VALUES (35, 1, NULL, 15); -INSERT INTO itemTypeFields VALUES (35, 27, NULL, 16); -INSERT INTO itemTypeFields VALUES (35, 87, NULL, 17); -INSERT INTO itemTypeFields VALUES (35, 123, NULL, 18); -INSERT INTO itemTypeFields VALUES (35, 19, NULL, 19); -INSERT INTO itemTypeFields VALUES (35, 62, NULL, 20); -INSERT INTO itemTypeFields VALUES (35, 18, NULL, 21); -INSERT INTO itemTypeFields VALUES (35, 2, NULL, 22); -INSERT INTO itemTypeFields VALUES (35, 22, NULL, 23); -INSERT INTO itemTypeFields VALUES (36, 110, NULL, 1); -INSERT INTO itemTypeFields VALUES (36, 90, NULL, 2); -INSERT INTO itemTypeFields VALUES (36, 86, NULL, 3); -INSERT INTO itemTypeFields VALUES (36, 3, NULL, 4); -INSERT INTO itemTypeFields VALUES (36, 30, NULL, 5); -INSERT INTO itemTypeFields VALUES (36, 4, NULL, 6); -INSERT INTO itemTypeFields VALUES (36, 45, NULL, 7); -INSERT INTO itemTypeFields VALUES (36, 6, NULL, 8); -INSERT INTO itemTypeFields VALUES (36, 7, NULL, 9); -INSERT INTO itemTypeFields VALUES (36, 8, NULL, 10); -INSERT INTO itemTypeFields VALUES (36, 14, NULL, 11); -INSERT INTO itemTypeFields VALUES (36, 10, NULL, 12); -INSERT INTO itemTypeFields VALUES (36, 87, NULL, 13); -INSERT INTO itemTypeFields VALUES (36, 11, NULL, 14); -INSERT INTO itemTypeFields VALUES (36, 116, NULL, 15); -INSERT INTO itemTypeFields VALUES (36, 1, NULL, 16); -INSERT INTO itemTypeFields VALUES (36, 27, NULL, 17); -INSERT INTO itemTypeFields VALUES (36, 123, NULL, 18); -INSERT INTO itemTypeFields VALUES (36, 19, NULL, 19); -INSERT INTO itemTypeFields VALUES (36, 62, NULL, 20); -INSERT INTO itemTypeFields VALUES (36, 18, NULL, 21); -INSERT INTO itemTypeFields VALUES (36, 2, NULL, 22); -INSERT INTO itemTypeFields VALUES (36, 22, NULL, 23); - - -INSERT INTO baseFieldMappings VALUES (16, 4, 94); -- bill/volume/codeVolume -INSERT INTO baseFieldMappings VALUES (17, 4, 97); -- case/volume/reporterVolume -INSERT INTO baseFieldMappings VALUES (7, 8, 89); -- thesis/publisher/university -INSERT INTO baseFieldMappings VALUES (11, 8, 21); -- film/publisher/distributor -INSERT INTO baseFieldMappings VALUES (15, 8, 31); -- report/publisher/institution -INSERT INTO baseFieldMappings VALUES (26, 8, 72); -- audioRecording/publisher/label -INSERT INTO baseFieldMappings VALUES (28, 8, 76); -- videoRecording/publisher/studio -INSERT INTO baseFieldMappings VALUES (29, 8, 78); -- tvBroadcast/publisher/network -INSERT INTO baseFieldMappings VALUES (30, 8, 78); -- radioBroadcast/publisher/network -INSERT INTO baseFieldMappings VALUES (32, 8, 83); -- computerProgram/publisher/company -INSERT INTO baseFieldMappings VALUES (16, 10, 95); -- bill/pages/codePages -INSERT INTO baseFieldMappings VALUES (17, 10, 98); -- case/pages/firstPage -INSERT INTO baseFieldMappings VALUES (3, 12, 115); -- bookSection/publicationTitle/bookTitle -INSERT INTO baseFieldMappings VALUES (33, 12, 114); -- conferencePaper/publicationTitle/proceedingsTitle -INSERT INTO baseFieldMappings VALUES (13, 12, 91); -- webpage/publicationTitle/websiteTitle -INSERT INTO baseFieldMappings VALUES (23, 12, 107); -- blogPost/publicationTitle/blogTitle -INSERT INTO baseFieldMappings VALUES (25, 12, 104); -- forumPost/publicationTitle/forumTitle -INSERT INTO baseFieldMappings VALUES (29, 12, 119); -- tvBroadcast/publicationTitle/programTitle -INSERT INTO baseFieldMappings VALUES (30, 12, 119); -- radioBroadcast/publicationTitle/programTitle -INSERT INTO baseFieldMappings VALUES (35, 12, 85); -- encyclopediaEntry/publicationTitle/encyclopediaTitle -INSERT INTO baseFieldMappings VALUES (36, 12, 86); -- dictionaryEntry/publicationTitle/dictionaryTitle -INSERT INTO baseFieldMappings VALUES (17, 14, 96); -- case/date/dateDecided -INSERT INTO baseFieldMappings VALUES (19, 14, 52); -- patent/date/issueDate -INSERT INTO baseFieldMappings VALUES (20, 14, 100); -- statute/date/dateEnacted -INSERT INTO baseFieldMappings VALUES (15, 60, 92); -- report/number/reportNumber -INSERT INTO baseFieldMappings VALUES (16, 60, 93); -- bill/number/billNumber -INSERT INTO baseFieldMappings VALUES (17, 60, 117); -- case/number/docketNumber -INSERT INTO baseFieldMappings VALUES (18, 60, 99); -- hearing/number/documentNumber -INSERT INTO baseFieldMappings VALUES (19, 60, 50); -- patent/number/patentNumber -INSERT INTO baseFieldMappings VALUES (20, 60, 101); -- statute/number/publicLawNumber -INSERT INTO baseFieldMappings VALUES (29, 60, 105); -- tvBroadcast/number/episodeNumber -INSERT INTO baseFieldMappings VALUES (30, 60, 105); -- radioBroadcast/number/episodeNumber -INSERT INTO baseFieldMappings VALUES (31, 60, 105); -- podcast/number/episodeNumber -INSERT INTO baseFieldMappings VALUES (7, 108, 69); -- thesis/type/thesisType -INSERT INTO baseFieldMappings VALUES (8, 108, 65); -- letter/type/letterType -INSERT INTO baseFieldMappings VALUES (9, 108, 66); -- manuscript/type/manuscriptType -INSERT INTO baseFieldMappings VALUES (11, 108, 122); -- film/type/genre -INSERT INTO baseFieldMappings VALUES (13, 108, 70); -- webpage/type/websiteType -INSERT INTO baseFieldMappings VALUES (15, 108, 32); -- report/type/reportType -INSERT INTO baseFieldMappings VALUES (22, 108, 67); -- map/type/mapType -INSERT INTO baseFieldMappings VALUES (23, 108, 70); -- blogPost/type/websiteType -INSERT INTO baseFieldMappings VALUES (25, 108, 79); -- forumPost/type/postType -INSERT INTO baseFieldMappings VALUES (27, 108, 74); -- presentation/type/presentationType -INSERT INTO baseFieldMappings VALUES (10, 109, 64); -- interview/medium/interviewMedium -INSERT INTO baseFieldMappings VALUES (11, 109, 63); -- film/medium/videoRecordingFormat -INSERT INTO baseFieldMappings VALUES (12, 109, 59); -- artwork/medium/artworkMedium -INSERT INTO baseFieldMappings VALUES (26, 109, 71); -- audioRecording/medium/audioRecordingFormat -INSERT INTO baseFieldMappings VALUES (28, 109, 63); -- videoRecording/medium/videoRecordingFormat -INSERT INTO baseFieldMappings VALUES (29, 109, 63); -- tvBroadcast/medium/videoRecodingMedium -INSERT INTO baseFieldMappings VALUES (30, 109, 71); -- radioBroadcast/medium/audioRecordingFormat -INSERT INTO baseFieldMappings VALUES (31, 109, 80); -- podcast/medium/audioFileType -INSERT INTO baseFieldMappings VALUES (17, 110, 111); -- case/title/caseName -INSERT INTO baseFieldMappings VALUES (20, 110, 112); -- statute/title/nameOfAct -INSERT INTO baseFieldMappings VALUES (21, 110, 113); -- email/title/subject - -INSERT INTO creatorTypes VALUES(1, "author"); -INSERT INTO creatorTypes VALUES(2, "contributor"); -INSERT INTO creatorTypes VALUES(3, "editor"); -INSERT INTO creatorTypes VALUES(4, "translator"); -INSERT INTO creatorTypes VALUES(5, "seriesEditor"); -INSERT INTO creatorTypes VALUES(6, "interviewee"); -INSERT INTO creatorTypes VALUES(7, "interviewer"); -INSERT INTO creatorTypes VALUES(8, "director"); -INSERT INTO creatorTypes VALUES(9, "scriptwriter"); -INSERT INTO creatorTypes VALUES(10, "producer"); -INSERT INTO creatorTypes VALUES(11, "castMember"); -INSERT INTO creatorTypes VALUES(12, "sponsor"); -INSERT INTO creatorTypes VALUES(13, "counsel"); -INSERT INTO creatorTypes VALUES(14, "inventor"); -INSERT INTO creatorTypes VALUES(15, "attorneyAgent"); -INSERT INTO creatorTypes VALUES(16, "recipient"); -INSERT INTO creatorTypes VALUES(17, "performer"); -INSERT INTO creatorTypes VALUES(18, "composer"); -INSERT INTO creatorTypes VALUES(19, "wordsBy"); -INSERT INTO creatorTypes VALUES(20, "cartographer"); -INSERT INTO creatorTypes VALUES(21, "programmer"); -INSERT INTO creatorTypes VALUES(22, "artist"); -INSERT INTO creatorTypes VALUES(23, "commenter"); -INSERT INTO creatorTypes VALUES(24, "presenter"); -INSERT INTO creatorTypes VALUES(25, "guest"); -INSERT INTO creatorTypes VALUES(26, "podcaster"); -INSERT INTO creatorTypes VALUES(27, "reviewedAuthor"); -INSERT INTO creatorTypes VALUES(28, "cosponsor"); -INSERT INTO creatorTypes VALUES(29, "bookAuthor"); - -INSERT INTO itemTypeCreatorTypes VALUES(2,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(2,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(2,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(2,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(2,5,0); -INSERT INTO itemTypeCreatorTypes VALUES(3,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(3,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(3,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(3,29,0); -INSERT INTO itemTypeCreatorTypes VALUES(3,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(3,5,0); -INSERT INTO itemTypeCreatorTypes VALUES(4,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(4,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(4,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(4,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(4,27,0); -INSERT INTO itemTypeCreatorTypes VALUES(5,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(5,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(5,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(5,27,0); -INSERT INTO itemTypeCreatorTypes VALUES(6,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(6,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(6,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(6,27,0); -INSERT INTO itemTypeCreatorTypes VALUES(7,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(7,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(8,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(8,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(8,16,0); -INSERT INTO itemTypeCreatorTypes VALUES(9,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(9,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(9,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(10,6,1); -INSERT INTO itemTypeCreatorTypes VALUES(10,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(10,7,0); -INSERT INTO itemTypeCreatorTypes VALUES(10,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(11,8,1); -INSERT INTO itemTypeCreatorTypes VALUES(11,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(11,9,0); -INSERT INTO itemTypeCreatorTypes VALUES(11,10,0); -INSERT INTO itemTypeCreatorTypes VALUES(12,22,1); -INSERT INTO itemTypeCreatorTypes VALUES(12,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(13,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(13,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(13,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(15,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(15,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(15,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(15,5,0); -INSERT INTO itemTypeCreatorTypes VALUES(16,12,1); -INSERT INTO itemTypeCreatorTypes VALUES(16,28,0); -INSERT INTO itemTypeCreatorTypes VALUES(16,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(17,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(17,13,0); -INSERT INTO itemTypeCreatorTypes VALUES(17,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(18,2,1); -INSERT INTO itemTypeCreatorTypes VALUES(19,14,1); -INSERT INTO itemTypeCreatorTypes VALUES(19,15,0); -INSERT INTO itemTypeCreatorTypes VALUES(19,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(20,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(20,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(21,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(21,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(21,16,0); -INSERT INTO itemTypeCreatorTypes VALUES(22,20,1); -INSERT INTO itemTypeCreatorTypes VALUES(22,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(22,5,0); -INSERT INTO itemTypeCreatorTypes VALUES(23,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(23,23,0); -INSERT INTO itemTypeCreatorTypes VALUES(23,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(24,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(24,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(24,16,0); -INSERT INTO itemTypeCreatorTypes VALUES(25,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(25,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(26,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(26,17,1); -INSERT INTO itemTypeCreatorTypes VALUES(26,18,0); -INSERT INTO itemTypeCreatorTypes VALUES(26,19,0); -INSERT INTO itemTypeCreatorTypes VALUES(27,24,1); -INSERT INTO itemTypeCreatorTypes VALUES(27,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(28,8,1); -INSERT INTO itemTypeCreatorTypes VALUES(28,9,0); -INSERT INTO itemTypeCreatorTypes VALUES(28,10,0); -INSERT INTO itemTypeCreatorTypes VALUES(28,11,0); -INSERT INTO itemTypeCreatorTypes VALUES(28,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(29,8,1); -INSERT INTO itemTypeCreatorTypes VALUES(29,9,0); -INSERT INTO itemTypeCreatorTypes VALUES(29,10,0); -INSERT INTO itemTypeCreatorTypes VALUES(29,11,0); -INSERT INTO itemTypeCreatorTypes VALUES(29,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(29,25,0); -INSERT INTO itemTypeCreatorTypes VALUES(30,8,1); -INSERT INTO itemTypeCreatorTypes VALUES(30,9,0); -INSERT INTO itemTypeCreatorTypes VALUES(30,10,0); -INSERT INTO itemTypeCreatorTypes VALUES(30,11,0); -INSERT INTO itemTypeCreatorTypes VALUES(30,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(30,25,0); -INSERT INTO itemTypeCreatorTypes VALUES(31,26,1); -INSERT INTO itemTypeCreatorTypes VALUES(31,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(31,25,0); -INSERT INTO itemTypeCreatorTypes VALUES(32,21,1); -INSERT INTO itemTypeCreatorTypes VALUES(32,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(33, 1, 1); -INSERT INTO itemTypeCreatorTypes VALUES(33, 2, 0); -INSERT INTO itemTypeCreatorTypes VALUES(33, 3, 0); -INSERT INTO itemTypeCreatorTypes VALUES(33, 4, 0); -INSERT INTO itemTypeCreatorTypes VALUES(33, 5, 0); -INSERT INTO itemTypeCreatorTypes VALUES(34,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(34,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(34,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(34,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(34,27,0); -INSERT INTO itemTypeCreatorTypes VALUES(35,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(35,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(35,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(35,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(35,5,0); -INSERT INTO itemTypeCreatorTypes VALUES(36,1,1); -INSERT INTO itemTypeCreatorTypes VALUES(36,2,0); -INSERT INTO itemTypeCreatorTypes VALUES(36,3,0); -INSERT INTO itemTypeCreatorTypes VALUES(36,4,0); -INSERT INTO itemTypeCreatorTypes VALUES(36,5,0); - INSERT INTO "charsets" VALUES (1, "utf-8"); INSERT INTO "charsets" VALUES (2, "big5"); INSERT INTO "charsets" VALUES (3, "euc-jp"); diff --git a/resource/schema/triggers.sql b/resource/schema/triggers.sql index 041ce800a9..666877ac8d 100644 --- a/resource/schema/triggers.sql +++ b/resource/schema/triggers.sql @@ -22,34 +22,6 @@ -- ";---" is an ugly hack for Zotero.DB.executeSQLFile() --- Triggers to validate date field -DROP TRIGGER IF EXISTS insert_date_field; -CREATE TRIGGER insert_date_field BEFORE INSERT ON itemData - FOR EACH ROW WHEN NEW.fieldID IN (14, 27, 52, 96, 100) - BEGIN - SELECT CASE - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 1, 4) AS INT) BETWEEN 0 AND 9999 AND - SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 5, 1) = '-' AND - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 6, 2) AS INT) BETWEEN 0 AND 12 AND - SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 8, 1) = '-' AND - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 9, 2) AS INT) BETWEEN 0 AND 31 - WHEN 0 THEN RAISE (ABORT, 'Date field must begin with SQL date') END;--- - END; - -DROP TRIGGER IF EXISTS update_date_field; -CREATE TRIGGER update_date_field BEFORE UPDATE ON itemData - FOR EACH ROW WHEN NEW.fieldID IN (14, 27, 52, 96, 100) - BEGIN - SELECT CASE - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 1, 4) AS INT) BETWEEN 0 AND 9999 AND - SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 5, 1) = '-' AND - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 6, 2) AS INT) BETWEEN 0 AND 12 AND - SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 8, 1) = '-' AND - CAST(SUBSTR((SELECT value FROM itemDataValues WHERE valueID=NEW.valueID), 9, 2) AS INT) BETWEEN 0 AND 31 - WHEN 0 THEN RAISE (ABORT, 'Date field must begin with SQL date') END;--- - END; - - -- Don't allow empty creators DROP TRIGGER IF EXISTS insert_creatorData; CREATE TRIGGER insert_creators BEFORE INSERT ON creators @@ -137,79 +109,6 @@ CREATE TRIGGER fku_itemNotes_parentItemID_collectionItems_itemID DELETE FROM collectionItems WHERE itemID = NEW.itemID;--- END; - --- itemAttachments -DROP TRIGGER IF EXISTS fki_itemAttachments; -CREATE TRIGGER fki_itemAttachments - BEFORE INSERT ON itemAttachments - FOR EACH ROW BEGIN - SELECT RAISE(ABORT, 'insert on table "itemAttachments" violates foreign key constraint "fki_itemAttachments"') - WHERE NEW.parentItemID IS NOT NULL AND - (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.parentItemID);--- - - -- Make sure this is an attachment item - SELECT RAISE(ABORT, 'item is not an attachment') - WHERE (SELECT itemTypeID FROM items WHERE itemID = NEW.itemID) != 14;--- - - -- Make sure parent is a regular item - SELECT RAISE(ABORT, 'parent is not a regular item') - WHERE NEW.parentItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.parentItemID) IN (1,14);--- - - -- If child, make sure attachment is not in a collection - SELECT RAISE(ABORT, 'collection item must be top level') - WHERE NEW.parentItemID IS NOT NULL AND (SELECT COUNT(*) FROM collectionItems WHERE itemID=NEW.itemID)>0;--- - END; - -DROP TRIGGER IF EXISTS fku_itemAttachments; -CREATE TRIGGER fku_itemAttachments - BEFORE UPDATE ON itemAttachments - FOR EACH ROW BEGIN - SELECT RAISE(ABORT, 'update on table "itemAttachments" violates foreign key constraint "fku_itemAttachments"') - WHERE NEW.parentItemID IS NOT NULL AND - (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.parentItemID);--- - - -- Make sure parent is a regular item - SELECT RAISE(ABORT, 'parent is not a regular item') - WHERE NEW.parentItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.parentItemID) IN (1,14);--- - END; - - --- itemNotes -DROP TRIGGER IF EXISTS fki_itemNotes; -CREATE TRIGGER fki_itemNotes - BEFORE INSERT ON itemNotes - FOR EACH ROW BEGIN - SELECT RAISE(ABORT, 'insert on table "itemNotes" violates foreign key constraint "fki_itemNotes_libraryID"') - WHERE NEW.parentItemID IS NOT NULL AND - (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.parentItemID);--- - - -- Make sure this is a note or attachment item - SELECT RAISE(ABORT, 'item is not a note or attachment') WHERE - (SELECT itemTypeID FROM items WHERE itemID = NEW.itemID) NOT IN (1,14);--- - - -- Make sure parent is a regular item - SELECT RAISE(ABORT, 'parent is not a regular item') WHERE - NEW.parentItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.parentItemID) IN (1,14);--- - - -- If child, make sure note is not in a collection - SELECT RAISE(ABORT, 'collection item must be top level') WHERE - NEW.parentItemID IS NOT NULL AND (SELECT COUNT(*) FROM collectionItems WHERE itemID=NEW.itemID)>0;--- - END; - -DROP TRIGGER IF EXISTS fku_itemNotes; -CREATE TRIGGER fku_itemNotes - BEFORE UPDATE ON itemNotes - FOR EACH ROW BEGIN - SELECT RAISE(ABORT, 'update on table "itemNotes" violates foreign key constraint "fku_itemNotes"') - WHERE NEW.parentItemID IS NOT NULL AND - (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.parentItemID);--- - - -- Make sure parent is a regular item - SELECT RAISE(ABORT, 'parent is not a regular item') WHERE - NEW.parentItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.parentItemID) IN (1,14);--- - END; - - -- Make sure tags aren't empty DROP TRIGGER IF EXISTS fki_tags; CREATE TRIGGER fki_tags diff --git a/resource/schema/userdata.sql b/resource/schema/userdata.sql index 998af81053..4708048571 100644 --- a/resource/schema/userdata.sql +++ b/resource/schema/userdata.sql @@ -1,4 +1,4 @@ --- 105 +-- 106 -- Copyright (c) 2009 Center for History and New Media -- George Mason University, Fairfax, Virginia, USA diff --git a/scripts/config.js b/scripts/config.js index ec64adb9fc..a504b503bd 100644 --- a/scripts/config.js +++ b/scripts/config.js @@ -62,7 +62,11 @@ const browserifyConfigs = [ ]; // exclude mask used for js, copy, symlink and sass tasks -const ignoreMask = ['**/#*', '**/_*.scss']; +const ignoreMask = [ + '**/#*', + '**/_*.scss', + 'resource/schema/global/schema.json.gz' +]; const jsFiles = [ `{${dirs.join(',')}}/**/*.js`, diff --git a/test/tests/collectionTest.js b/test/tests/collectionTest.js index 1f5d8b027c..ec60aedc38 100644 --- a/test/tests/collectionTest.js +++ b/test/tests/collectionTest.js @@ -262,6 +262,29 @@ describe("Zotero.Collection", function() { }); }) + describe("#fromJSON()", function () { + it("should ignore unknown property in non-strict mode", function () { + var json = { + name: "Collection", + foo: "Bar" + }; + var s = new Zotero.Collection(); + s.fromJSON(json); + }); + + it("should throw on unknown property in strict mode", function () { + var json = { + name: "Collection", + foo: "Bar" + }; + var s = new Zotero.Collection(); + var f = () => { + s.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Unknown collection property/); + }); + }); + describe("#toJSON()", function () { it("should set 'parentCollection' to false when cleared", function* () { var col1 = yield createDataObject('collection'); diff --git a/test/tests/data/itemJSON.js b/test/tests/data/itemJSON.js index b63ea4df9f..efdbb96ec8 100644 --- a/test/tests/data/itemJSON.js +++ b/test/tests/data/itemJSON.js @@ -1262,7 +1262,7 @@ "dateAdded": "2015-04-12T09:00:22Z", "dateModified": "2015-04-12T09:00:22Z", "extra": "Extra", - "filingDate": "2000-01-02 2000-01-02", + "filingDate": "2000-01-02", "issueDate": "1999-12-31", "issuingAuthority": "Issuing authority", "itemType": "patent", diff --git a/test/tests/data/translatorExport.js b/test/tests/data/translatorExport.js index a8cc3ab43c..80a9556b67 100644 --- a/test/tests/data/translatorExport.js +++ b/test/tests/data/translatorExport.js @@ -1311,7 +1311,7 @@ "dateAdded": "2015-04-12T05:45:15Z", "dateModified": "2015-04-12T05:45:15Z", "extra": "Extra", - "filingDate": "2000-01-02 2000-01-02", + "filingDate": "2000-01-02", "issueDate": "1999-12-31", "issuingAuthority": "Issuing authority", "itemType": "patent", diff --git a/test/tests/data/translatorExportLegacy.js b/test/tests/data/translatorExportLegacy.js index e4840b1363..1c865a5896 100644 --- a/test/tests/data/translatorExportLegacy.js +++ b/test/tests/data/translatorExportLegacy.js @@ -1899,7 +1899,7 @@ "dateAdded": "2015-04-26 06:40:48", "dateModified": "2015-04-26 06:40:48", "extra": "Extra", - "filingDate": "2000-01-02 2000-01-02", + "filingDate": "2000-01-02", "issueDate": "1999-12-31", "issuingAuthority": "Issuing authority", "itemID": 95, @@ -1929,7 +1929,7 @@ "country": "Country", "date": "1999-12-31", "extra": "Extra", - "filingDate": "2000-01-02 2000-01-02", + "filingDate": "2000-01-02", "issuingAuthority": "Issuing authority", "language": "en-US", "legalStatus": "Legal status", diff --git a/test/tests/dataObjectUtilitiesTest.js b/test/tests/dataObjectUtilitiesTest.js index 5372ca0b1b..148fc36945 100644 --- a/test/tests/dataObjectUtilitiesTest.js +++ b/test/tests/dataObjectUtilitiesTest.js @@ -47,6 +47,15 @@ describe("Zotero.DataObjectUtilities", function() { assert.equal(obj.conditions[0].value, 'B'); assert.equal(obj.conditions[1].value, 'en'); }) + + it("should omit unknown base properties", function () { + var patchBase = { + unknownField: 'Foo' + }; + var obj = {}; + obj = Zotero.DataObjectUtilities.patch(patchBase, obj); + assert.notProperty(obj, 'unknownField'); + }); }) describe("#diff()", function () { diff --git a/test/tests/itemTest.js b/test/tests/itemTest.js index 97b06d8b2f..d5947b53ea 100644 --- a/test/tests/itemTest.js +++ b/test/tests/itemTest.js @@ -507,7 +507,7 @@ describe("Zotero.Item", function () { }); }); - describe("#setCreators", function () { + describe("#setCreators()", function () { it("should accept an array of creators in API JSON format", function* () { var creators = [ { @@ -534,13 +534,13 @@ describe("Zotero.Item", function () { firstName: "First", lastName: "Last", fieldMode: 0, - creatorTypeID: 1 + creatorTypeID: Zotero.CreatorTypes.getID('author') }, { firstName: "", lastName: "Test Name", fieldMode: 1, - creatorTypeID: 2 + creatorTypeID: Zotero.CreatorTypes.getID('editor') } ]; @@ -558,13 +558,75 @@ describe("Zotero.Item", function () { firstName: "First", lastName: "Last", fieldMode: 0, - creatorTypeID: 1 + creatorTypeID: Zotero.CreatorTypes.getID('author') } ]); assert.lengthOf(item.getCreators(), 1); item.setCreators([]); assert.lengthOf(item.getCreators(), 0); }); + + it("should switch to primary creator type if unknown type given", function () { + var item = createUnsavedDataObject('item', { itemType: 'book' }); + item.setCreators([ + { + firstName: "First", + lastName: "Last", + creatorType: "unknown" + } + ]); + assert.equal(item.getCreators()[0].creatorTypeID, Zotero.CreatorTypes.getID('author')); + }); + + it("should switch to primary creator type on invalid creator type for a given item type", function () { + var item = createUnsavedDataObject('item', { itemType: 'book' }); + item.setCreators([ + { + firstName: "First", + lastName: "Last", + creatorType: "interviewee" + } + ]); + assert.equal(item.getCreators()[0].creatorTypeID, Zotero.CreatorTypes.getID('author')); + }); + + it("should throw on unknown creator type in strict mode", function () { + var item = createUnsavedDataObject('item', { itemType: 'book' }); + var f = () => { + item.setCreators( + [ + { + firstName: "First", + lastName: "Last", + creatorType: "unknown" + } + ], + { + strict: true + } + ); + }; + assert.throws(f, /^Unknown creator type/); + }); + + it("should throw on invalid creator type for a given item type in strict mode", function () { + var item = createUnsavedDataObject('item', { itemType: 'book' }); + var f = () => { + item.setCreators( + [ + { + firstName: "First", + lastName: "Last", + creatorType: "interviewee" + } + ], + { + strict: true + } + ); + } + assert.throws(f, /^Invalid creator type/); + }); }) @@ -1692,16 +1754,150 @@ describe("Zotero.Item", function () { assert.isFalse(item.inPublications); }); - it("should ignore unknown fields", function* () { + // Not currently following this behavior + /*it("should move valid field in Extra to field if not set", function () { + var doi = '10.1234/abcd'; var json = { itemType: "journalArticle", title: "Test", - foo: "Invalid" + extra: `DOI: ${doi}` + }; + var item = new Zotero.Item; + item.fromJSON(json); + assert.equal(item.getField('DOI'), doi); + assert.equal(item.getField('extra'), ''); + }); + + it("shouldn't move valid field in Extra to field if also present in JSON", function () { + var doi1 = '10.1234/abcd'; + var doi2 = '10.2345/bcde'; + var json = { + itemType: "journalArticle", + title: "Test", + DOI: doi1, + extra: `doi: ${doi2}` + }; + var item = new Zotero.Item; + item.fromJSON(json); + assert.equal(item.getField('DOI'), doi1); + assert.equal(item.getField('extra'), `doi: ${doi2}`); + }); + + it("shouldn't move valid field in Extra to field if already set", function () { + var doi1 = '10.1234/abcd'; + var doi2 = '10.2345/bcde'; + var json = { + itemType: "journalArticle", + title: "Test", + DOI: doi1, + extra: `doi: ${doi2}` + }; + var item = new Zotero.Item('journalArticle'); + item.setField('DOI', doi1); + item.fromJSON(json); + assert.equal(item.getField('DOI'), doi1); + assert.equal(item.getField('extra'), `doi: ${doi2}`); + });*/ + + it("should store unknown field in Extra in non-strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + foo: "Bar" }; var item = new Zotero.Item; item.fromJSON(json); assert.equal(item.getField('title'), 'Test'); - }) + assert.equal(item.getField('extra'), 'foo: Bar'); + }); + + it("should replace unknown field in Extra in non-strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + foo: "BBB", + extra: "Foo: AAA\nBar: CCC" + }; + var item = new Zotero.Item; + item.fromJSON(json); + assert.equal(item.getField('title'), 'Test'); + assert.equal(item.getField('extra'), 'Foo: BBB\nBar: CCC'); + }); + + it("should handle Extra in non-strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + extra: "Here's some extra text" + }; + var item = new Zotero.Item(); + item.fromJSON(json); + assert.equal(item.getField('extra'), json.extra); + }); + + it("should throw on unknown field in strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + foo: "Bar" + }; + var item = new Zotero.Item; + var f = () => { + item.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Unknown field/); + }); + + it("should throw on invalid field for a given item type in strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + numPages: "123" + }; + var item = new Zotero.Item; + var f = () => { + item.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Invalid field/); + }); + + it("should throw on unknown creator type in strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + creators: [ + { + firstName: "First", + lastName: "Last", + creatorType: "unknown" + } + ] + }; + var item = new Zotero.Item; + var f = () => { + item.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Unknown creator type/); + }); + + it("should throw on invalid creator type for a given item type in strict mode", function () { + var json = { + itemType: "journalArticle", + title: "Test", + creators: [ + { + firstName: "First", + lastName: "Last", + creatorType: "interviewee" + } + ] + }; + var item = new Zotero.Item; + var f = () => { + item.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Invalid creator type/); + }); it("should accept ISO 8601 dates", function* () { var json = { diff --git a/test/tests/schemaTest.js b/test/tests/schemaTest.js index b9f9ef523d..21292b9dc2 100644 --- a/test/tests/schemaTest.js +++ b/test/tests/schemaTest.js @@ -25,6 +25,59 @@ describe("Zotero.Schema", function() { }); }); + describe("Global Schema", function () { + var schemaJSON, schema; + + before(async function () { + schemaJSON = await Zotero.File.getResourceAsync('resource://zotero/schema/global/schema.json'); + }); + + beforeEach(async function () { + await resetDB({ + thisArg: this, + skipBundledFiles: true + }); + schema = JSON.parse(schemaJSON); + }); + + after(async function() { + await resetDB({ + thisArg: this, + skipBundledFiles: true + }); + }); + + describe("#migrateExtraFields()", function () { + it("should add a new field and migrate values from Extra", async function () { + var item = await createDataObject('item', { itemType: 'book' }); + item.setField('numPages', "10"); + item.setField('extra', 'Foo Bar: This is a value.\nnumber-of-pages: 11\nThis is another line.'); + item.synced = true; + await item.saveTx(); + + schema.version++; + schema.itemTypes.find(x => x.itemType == 'book').fields.splice(0, 1, { field: 'fooBar' }) + var newLocales = {}; + Object.keys(schema.locales).forEach((locale) => { + var o = schema.locales[locale]; + o.fields.fooBar = 'Foo Bar'; + newLocales[locale] = o; + }); + await Zotero.Schema._updateGlobalSchemaForTest(schema); + await Zotero.Schema.migrateExtraFields(); + + assert.isNumber(Zotero.ItemFields.getID('fooBar')); + assert.equal(Zotero.ItemFields.getLocalizedString('fooBar'), 'Foo Bar'); + assert.equal(item.getField('fooBar'), 'This is a value.'); + // Existing fields shouldn't be overwritten and should be left in Extra + assert.equal(item.getField('numPages'), '10'); + assert.equal(item.getField('extra'), 'number-of-pages: 11\nThis is another line.'); + assert.isTrue(item.synced); + }); + }); + }); + + describe("#integrityCheck()", function () { before(function* () { yield resetDB({ diff --git a/test/tests/searchTest.js b/test/tests/searchTest.js index d66c687eee..d9caf0a924 100644 --- a/test/tests/searchTest.js +++ b/test/tests/searchTest.js @@ -509,5 +509,40 @@ describe("Zotero.Search", function() { assert.equal(conditions["1"].operator, 'is'); assert.equal(conditions["1"].value, '2016'); }); + + it("should ignore unknown property in non-strict mode", function () { + var json = { + name: "Search", + conditions: [ + { + condition: 'title', + operator: 'contains', + value: 'foo' + } + ], + foo: "Bar" + }; + var s = new Zotero.Search(); + s.fromJSON(json); + }); + + it("should throw on unknown property in strict mode", function () { + var json = { + name: "Search", + conditions: [ + { + condition: 'title', + operator: 'contains', + value: 'foo' + } + ], + foo: "Bar" + }; + var s = new Zotero.Search(); + var f = () => { + s.fromJSON(json, { strict: true }); + }; + assert.throws(f, /^Unknown search property/); + }); }); }); diff --git a/test/tests/syncEngineTest.js b/test/tests/syncEngineTest.js index 87d8b23a13..906998e5fb 100644 --- a/test/tests/syncEngineTest.js +++ b/test/tests/syncEngineTest.js @@ -1480,7 +1480,7 @@ describe("Zotero.Sync.Data.Engine", function () { yield engine.start(); }) - it("should ignore errors when saving downloaded objects", function* () { + it("should add objects to sync queue if they can't be saved", function* () { ({ engine, client, caller } = yield setup({ stopOnError: false })); @@ -1560,7 +1560,7 @@ describe("Zotero.Sync.Data.Engine", function () { key: "CCCCCCCC", version: 1, name: "C", - // Unknown field -- should be ignored + // Unknown field -- collection should be queued unknownField: 5 }) ] @@ -1622,7 +1622,7 @@ describe("Zotero.Sync.Data.Engine", function () { version: 3, itemType: "book", title: "G", - // Unknown item field -- should be ignored + // Unknown item field -- item should be queued unknownField: "B" }), makeItemJSON({ @@ -1668,7 +1668,6 @@ describe("Zotero.Sync.Data.Engine", function () { // Check for saved objects yield assert.eventually.ok(Zotero.Collections.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA")); yield assert.eventually.ok(Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "DDDDDDDD")); - yield assert.eventually.ok(Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "GGGGGGGG")); // Check for queued objects var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('collection', userLibraryID); @@ -1678,9 +1677,13 @@ describe("Zotero.Sync.Data.Engine", function () { assert.sameMembers(keys, ['EEEEEEEE', 'FFFFFFFF']); var keys = yield Zotero.Sync.Data.Local.getObjectsFromSyncQueue('item', userLibraryID); - assert.sameMembers(keys, ['HHHHHHHH', 'JJJJJJJJ']); + assert.sameMembers(keys, ['GGGGGGGG', 'HHHHHHHH', 'JJJJJJJJ']); - assert.equal(spy.callCount, 3); + // Unknown search condition, search operator, item field, and item type + // + // Missing parent collection and item collection don't throw errors because they don't + // indicate a guaranteed problem with the client + assert.equal(spy.callCount, 4); }); it("should delay on second upload conflict", function* () { diff --git a/test/tests/utilities_internalTest.js b/test/tests/utilities_internalTest.js index 312160ce64..212db95930 100644 --- a/test/tests/utilities_internalTest.js +++ b/test/tests/utilities_internalTest.js @@ -114,32 +114,152 @@ describe("Zotero.Utilities.Internal", function () { describe("#extractExtraFields()", function () { + it("should extract a CSL type", function () { + var str = 'type: motion_picture'; + var { itemType, fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); + assert.equal(itemType, 'videoRecording'); + assert.equal(fields.size, 0); + assert.strictEqual(extra, ''); + }); + it("should extract a field", function () { var val = '10.1234/abcdef'; var str = `DOI: ${val}`; - var fields = Zotero.Utilities.Internal.extractExtraFields(str); + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); assert.equal(fields.size, 1); - assert.equal(fields.get('DOI').value, val); + assert.equal(fields.get('DOI'), val); + assert.strictEqual(extra, ''); + }); + + it("should extract a field for a given item", function () { + var item = createUnsavedDataObject('item', { itemType: 'journalArticle' }); + var val = '10.1234/abcdef'; + var str = `DOI: ${val}`; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str, item); + assert.equal(fields.size, 1); + assert.equal(fields.get('DOI'), val); + assert.strictEqual(extra, ''); + }); + + it("should extract a CSL field", function () { + var val = '10.1234/abcdef'; + var str = `container-title: ${val}`; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); + assert.equal(fields.size, Zotero.Schema.CSL_TEXT_MAPPINGS['container-title'].length); + assert.equal(fields.get('publicationTitle'), val); + assert.strictEqual(extra, ''); + }); + + it("should extract a CSL field for a given item type", function () { + var item = createUnsavedDataObject('item', { itemType: 'journalArticle' }); + var val = '10.1234/abcdef'; + var str = `container-title: ${val}`; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str, item); + assert.equal(fields.size, 1); + assert.equal(fields.get('publicationTitle'), val); + assert.strictEqual(extra, ''); }); it("should extract a field with different case", function () { var val = '10.1234/abcdef'; var str = `doi: ${val}`; - var fields = Zotero.Utilities.Internal.extractExtraFields(str); + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); assert.equal(fields.size, 1); - assert.equal(fields.get('DOI').value, val); + assert.equal(fields.get('DOI'), val); + assert.strictEqual(extra, ''); }); it("should extract a field with other fields, text, and whitespace", function () { - var originalDateVal = '1989'; - var doiVal = '10.1234/abcdef'; - var str = `\nOriginal Date: ${originalDateVal}\nDOI: ${doiVal}\n\n`; - var fields = Zotero.Utilities.Internal.extractExtraFields(str); + var place = 'New York'; + var doi = '10.1234/abcdef'; + var str = `Line 1\nPublisher Place: ${place}\nFoo: Bar\nDOI: ${doi}\n\nLine 2`; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); + assert.equal(fields.size, 2); + assert.equal(fields.get('DOI'), doi); + assert.equal(fields.get('place'), place); + assert.equal(extra, 'Line 1\nFoo: Bar\n\nLine 2'); + }); + + it("should extract the first instance of a field", function () { + var place1 = 'New York'; + var place2 = 'London'; + var str = `Publisher Place: ${place1}\nPublisher Place: ${place2}`; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str); assert.equal(fields.size, 1); - assert.equal(fields.get('DOI').value, doiVal); + assert.equal(fields.get('place'), place1); + assert.equal(extra, "Publisher Place: " + place2); + }); + + it("shouldn't extract a field that already exists on the item", function () { + var item = createUnsavedDataObject('item', { itemType: 'book' }); + item.setField('numPages', 10); + var str = 'number-of-pages: 11'; + var { fields, extra } = Zotero.Utilities.Internal.extractExtraFields(str, item); + assert.equal(fields.size, 0); + }); + + it("should extract a CSL name", function () { + var str = 'container-author: First || Last'; + var { creators, extra } = Zotero.Utilities.Internal.extractExtraFields(str); + assert.lengthOf(creators, 1); + assert.propertyVal(creators[0], 'creatorType', 'bookAuthor'); + assert.propertyVal(creators[0], 'firstName', 'First'); + assert.propertyVal(creators[0], 'lastName', 'Last'); + assert.strictEqual(extra, ''); + }); + + it("should extract a CSL name that's valid for a given item type", function () { + var item = createUnsavedDataObject('item', { itemType: 'bookSection' }); + var str = 'container-author: First || Last'; + var { creators, extra } = Zotero.Utilities.Internal.extractExtraFields(str, item); + assert.lengthOf(creators, 1); + assert.propertyVal(creators[0], 'creatorType', 'bookAuthor'); + assert.propertyVal(creators[0], 'firstName', 'First'); + assert.propertyVal(creators[0], 'lastName', 'Last'); + assert.strictEqual(extra, ''); + }); + + it("shouldn't extract a CSL name that's not valid for a given item type", function () { + var item = createUnsavedDataObject('item', { itemType: 'journalArticle' }); + var str = 'container-author: First || Last'; + var { creators, extra } = Zotero.Utilities.Internal.extractExtraFields(str, item); + assert.lengthOf(creators, 0); + assert.strictEqual(extra, str); }); }); + describe("#combineExtraFields", function () { + var originalDate = "1887"; + var publicationPlace = "New York"; + var doi = '10.1234/123456789'; + var fieldMap = new Map(); + fieldMap.set('originalDate', originalDate); + fieldMap.set('publicationPlace', publicationPlace); + fieldMap.set('DOI', doi); + var fieldStr = `DOI: ${doi}\noriginalDate: ${originalDate}\npublicationPlace: ${publicationPlace}`; + + it("should create 'field: value' pairs from field map", function () { + var extra = ""; + var newExtra = ZUI.combineExtraFields(extra, fieldMap); + assert.equal(newExtra, fieldStr); + }); + + it("should add fields above existing Extra content", function () { + var extra = "This is a note."; + var newExtra = ZUI.combineExtraFields(extra, fieldMap); + assert.equal(newExtra, fieldStr + '\n' + extra); + }); + + it("should replace existing fields", function () { + var extra = "This is a note.\nOriginal Date: 1886\nFoo: Bar"; + var newExtra = ZUI.combineExtraFields(extra, fieldMap); + assert.equal( + newExtra, + fieldStr.split(/\n/).filter(x => !x.startsWith('originalDate')).join("\n") + + "\nThis is a note.\nOriginal Date: 1887\nFoo: Bar" + ); + }); + }); describe("#extractIdentifiers()", function () { it("should extract ISBN-10", async function () {