From 415e644211cd445cb6370f18fbdff7efef61bdb0 Mon Sep 17 00:00:00 2001 From: Martynas Bagdonas Date: Fri, 2 Apr 2021 18:33:02 +0300 Subject: [PATCH] Upgrade note schema to v2: - Automatically upgrade all editable v1 notes when note is being opened - Pull `itemData` from existing citations and annotations into metadata container attribute `data-citation-items` - Strip `abstract` from `itemData` - Fill `citationItem` (in citations and annotations) with `itemData` when passing an HTML fragment for further processing in `note-editor` - Keep only `uri`, `text`, `color`, `pageLabel`, `position`, `citationItem` annotation properties --- chrome/content/zotero/bindings/noteeditor.xml | 7 + chrome/content/zotero/xpcom/data/notes.js | 187 +++++++++++++++--- chrome/content/zotero/xpcom/editorInstance.js | 125 ++++++++++-- note-editor | 2 +- pdf-reader | 2 +- 5 files changed, 274 insertions(+), 49 deletions(-) diff --git a/chrome/content/zotero/bindings/noteeditor.xml b/chrome/content/zotero/bindings/noteeditor.xml index 370268e1e0..7e9dd36be6 100644 --- a/chrome/content/zotero/bindings/noteeditor.xml +++ b/chrome/content/zotero/bindings/noteeditor.xml @@ -95,6 +95,13 @@ if (this._editorInstance) { this._editorInstance.uninit(); } + + // Automatically upgrade editable v1 note before it's loaded + // TODO: Remove this at some point + if (this.editable) { + await Zotero.Notes.upgradeSchemaV1(this._item); + } + this._editorInstance = new Zotero.EditorInstance(); await this._editorInstance.init({ state, diff --git a/chrome/content/zotero/xpcom/data/notes.js b/chrome/content/zotero/xpcom/data/notes.js index 48678db664..890a8f4aae 100644 --- a/chrome/content/zotero/xpcom/data/notes.js +++ b/chrome/content/zotero/xpcom/data/notes.js @@ -125,41 +125,64 @@ Zotero.Notes = new function() { if (!item.isNote()) { throw new Error('Item is not a note'); } - var note = item.getNote(); + let note = item.getNote(); - var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + let parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] .createInstance(Components.interfaces.nsIDOMParser); - var doc = parser.parseFromString(note, 'text/html'); + let doc = parser.parseFromString(note, 'text/html'); - var nodes = doc.querySelectorAll('img[data-attachment-key]'); - for (var node of nodes) { - var attachmentKey = node.getAttribute('data-attachment-key'); - if (attachmentKey) { - var attachment = Zotero.Items.getByLibraryAndKey(item.libraryID, attachmentKey); - if (attachment && attachment.parentID == item.id) { - var dataURI = await attachment.attachmentDataURI; - node.setAttribute('src', dataURI); - } - } - node.removeAttribute('data-attachment-key'); - } - - var nodes = doc.querySelectorAll('.citation[data-citation]'); - for (var node of nodes) { - var citation = node.getAttribute('data-citation'); - try { - citation = JSON.parse(decodeURIComponent(citation)); - for (var citationItem of citation.citationItems) { - var item = await Zotero.EditorInstance.getItemFromURIs(citationItem.uris); - if (item) { - citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item); + // Make sure this is the new note + let metadataContainer = doc.querySelector('body > div[data-schema-version]'); + if (metadataContainer) { + // Load base64 image data into src + let nodes = doc.querySelectorAll('img[data-attachment-key]'); + for (let node of nodes) { + let attachmentKey = node.getAttribute('data-attachment-key'); + if (attachmentKey) { + let attachment = Zotero.Items.getByLibraryAndKey(item.libraryID, attachmentKey); + if (attachment && attachment.parentID == item.id) { + let dataURI = await attachment.attachmentDataURI; + node.setAttribute('src', dataURI); } } - citation = encodeURIComponent(JSON.stringify(citation)); - node.setAttribute('data-citation', citation); + node.removeAttribute('data-attachment-key'); } - catch (e) { - Zotero.logError(e); + + // Set itemData for each citation citationItem + nodes = doc.querySelectorAll('.citation[data-citation]'); + for (let node of nodes) { + let citation = node.getAttribute('data-citation'); + try { + citation = JSON.parse(decodeURIComponent(citation)); + for (let citationItem of citation.citationItems) { + // Get itemData from existing item + let item = await Zotero.EditorInstance.getItemFromURIs(citationItem.uris); + if (item) { + citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item); + } + // Get itemData from note metadata container + else { + try { + let items = JSON.parse(decodeURIComponent(metadataContainer.getAttribute('data-citation-items'))); + let item = items.find(item => item.uris.some(uri => citationItem.uris.includes(uri))); + if (item) { + citationItem.itemData = item.itemData; + } + } + catch (e) { + } + } + if (!citationItem.itemData) { + node.replaceWith('(MISSING CITATION)'); + break; + } + } + citation = encodeURIComponent(JSON.stringify(citation)); + node.setAttribute('data-citation', citation); + } + catch (e) { + Zotero.logError(e); + } } } return doc.body.innerHTML; @@ -171,6 +194,112 @@ Zotero.Notes = new function() { let doc = parser.parseFromString(note, 'text/html'); return !!doc.querySelector('body > div[data-schema-version]'); }; + + /** + * Upgrade v1 notes: + * - Pull itemData from citations, highlights, images into metadata container + * - Strip abstract field from itemData + * - For `data-annotation` keep only the following fields: + * - uri + * - text + * - color + * - pageLabel + * - position + * - citationItem + * - Increase schema version number + * + * @param item + * @returns {Promise} + */ + this.upgradeSchemaV1 = async function (item) { + let note = item.note; + + let parser = Components.classes['@mozilla.org/xmlextras/domparser;1'] + .createInstance(Components.interfaces.nsIDOMParser); + let doc = parser.parseFromString(note, 'text/html'); + + let metadataContainer = doc.querySelector('body > div[data-schema-version]'); + if (!metadataContainer) { + return false; + } + + let schemaVersion = parseInt(metadataContainer.getAttribute('data-schema-version')); + if (schemaVersion !== 1) { + return false; + } + + let storedCitationItems = []; + try { + let data = JSON.parse(decodeURIComponent(metadataContainer.getAttribute('data-citation-items'))); + if (Array.isArray(data)) { + storedCitationItems = data; + } + } catch (e) { + } + + function pullItemData(citationItem) { + let { uris, itemData } = citationItem; + if (itemData) { + delete citationItem.itemData.abstract; + delete citationItem.itemData; + let item = storedCitationItems.find(item => item.uris.some(uri => uris.includes(uri))); + if (!item) { + storedCitationItems.push({ uris, itemData }); + } + } + } + + let nodes = doc.querySelectorAll('.citation[data-citation]'); + for (let node of nodes) { + let citation = node.getAttribute('data-citation'); + try { + citation = JSON.parse(decodeURIComponent(citation)); + citation.citationItems.forEach(citationItem => pullItemData(citationItem)); + citation = encodeURIComponent(JSON.stringify(citation)); + node.setAttribute('data-citation', citation); + } + catch (e) { + Zotero.logError(e); + } + } + + // img[data-annotation] and div.highlight[data-annotation] + nodes = doc.querySelectorAll('*[data-annotation]'); + for (let node of nodes) { + let annotation = node.getAttribute('data-annotation'); + try { + annotation = JSON.parse(decodeURIComponent(annotation)); + if (annotation.citationItem) { + pullItemData(annotation.citationItem); + } + annotation = { + uri: annotation.uri, + text: annotation.text, + color: annotation.color, + pageLabel: annotation.pageLabel, + position: annotation.position, + citationItem: annotation.citationItem + }; + annotation = encodeURIComponent(JSON.stringify(annotation)); + node.setAttribute('data-annotation', annotation); + } + catch (e) { + Zotero.logError(e); + } + } + + if (storedCitationItems.length) { + storedCitationItems = encodeURIComponent(JSON.stringify(storedCitationItems)); + metadataContainer.setAttribute('data-citation-items', storedCitationItems); + } + schemaVersion++; + metadataContainer.setAttribute('data-schema-version', schemaVersion); + note = doc.body.innerHTML; + note = note.trim(); + item.setNote(note); + await item.saveTx({ skipDateModifiedUpdate: true }); + return true; + }; }; if (typeof process === 'object' && process + '' === '[object process]') { diff --git a/chrome/content/zotero/xpcom/editorInstance.js b/chrome/content/zotero/xpcom/editorInstance.js index fae2ee2138..66f9afe3d5 100644 --- a/chrome/content/zotero/xpcom/editorInstance.js +++ b/chrome/content/zotero/xpcom/editorInstance.js @@ -36,7 +36,7 @@ const DOWNLOADED_IMAGE_TYPE = [ ]; // Schema version here has to be the same as in note-editor! -const SCHEMA_VERSION = 1; +const SCHEMA_VERSION = 2; class EditorInstance { constructor() { @@ -159,7 +159,7 @@ class EditorInstance { async insertAnnotations(annotations) { await this._ensureNoteCreated(); - let html = await this._serializeAnnotations(annotations); + let [html] = await this._serializeAnnotations(annotations); if (html) { this._postMessage({ action: 'insertHTML', pos: -1, html }); } @@ -197,9 +197,11 @@ class EditorInstance { /** * @param {Zotero.Item[]} annotations + * @param {Boolean} skipEmbeddingItemData Do not add itemData to citation items * @return {String} - HTML string */ - async _serializeAnnotations(annotations) { + async _serializeAnnotations(annotations, skipEmbeddingItemData) { + let storedCitationItems = []; let html = ''; for (let annotation of annotations) { let attachmentItem = await Zotero.Items.getAsync(annotation.attachmentItemID); @@ -218,20 +220,36 @@ class EditorInstance { let highlightHTML = ''; let commentHTML = ''; - annotation.uri = Zotero.URI.getItemURI(attachmentItem); + let storedAnnotation = { + uri: Zotero.URI.getItemURI(attachmentItem), + text: annotation.text, + color: annotation.color, + pageLabel: annotation.pageLabel, + position: annotation.position + }; // Citation let parentItem = attachmentItem.parentID && await Zotero.Items.getAsync(attachmentItem.parentID); if (parentItem) { + let uris = [Zotero.URI.getItemURI(parentItem)]; + let citationItem = { + uris, + locator: annotation.pageLabel + }; + // TODO: Find a more elegant way to call this let itemData = Zotero.Cite.System.prototype.retrieveItem(parentItem); delete itemData.abstract; - let citationItem = { - uris: [Zotero.URI.getItemURI(parentItem)], - itemData, - locator: annotation.pageLabel - }; - annotation.citationItem = citationItem; + if (!skipEmbeddingItemData) { + citationItem.itemData = itemData; + } + + let item = storedCitationItems.find(item => item.uris.some(uri => uris.includes(uri))); + if (!item) { + storedCitationItems.push({ uris, itemData }); + } + + storedAnnotation.citationItem = citationItem; let citation = { citationItems: [citationItem], properties: {} @@ -255,12 +273,12 @@ class EditorInstance { const PDFJS_DEFAULT_SCALE = 1.25; let width = Math.round(rectWidth * CSS_UNITS * PDFJS_DEFAULT_SCALE); let height = Math.round(rectHeight * width / rectWidth); - imageHTML = ``; + imageHTML = ``; } // Text if (annotation.text) { - highlightHTML = `“${annotation.text}”`; + highlightHTML = `“${annotation.text}”`; } // Note @@ -274,7 +292,7 @@ class EditorInstance { } html += '

' + imageHTML + otherHTML + '

\n'; } - return html; + return [html, storedCitationItems]; } async _digestItems(ids) { @@ -299,6 +317,66 @@ class EditorInstance { } else if (item.isNote()) { let note = item.note; + + let parser = Components.classes['@mozilla.org/xmlextras/domparser;1'] + .createInstance(Components.interfaces.nsIDOMParser); + let doc = parser.parseFromString(note, 'text/html'); + + // Get citationItems with itemData from note metadata + let storedCitationItems = []; + let containerNode = doc.querySelector('body > div[data-schema-version]'); + if (containerNode) { + try { + let data = JSON.parse(decodeURIComponent(containerNode.getAttribute('data-citation-items'))); + if (Array.isArray(data)) { + storedCitationItems = data; + } + } + catch (e) { + } + } + + if (storedCitationItems.length) { + let fillWithItemData = (citationItems) => { + for (let citationItem of citationItems) { + let item = storedCitationItems.find(item => item.uris.some(uri => citationItem.uris.includes(uri))); + if (item) { + citationItem.itemData = item.itemData; + } + } + }; + + let nodes = doc.querySelectorAll('.citation[data-citation]'); + for (let node of nodes) { + let citation = node.getAttribute('data-citation'); + try { + citation = JSON.parse(decodeURIComponent(citation)); + fillWithItemData(citation.citationItems); + citation = encodeURIComponent(JSON.stringify(citation)); + node.setAttribute('data-citation', citation); + } + catch (e) { + Zotero.logError(e); + } + } + + // img[data-annotation] and div.highlight[data-annotation] + nodes = doc.querySelectorAll('*[data-annotation]'); + for (let node of nodes) { + let annotation = node.getAttribute('data-annotation'); + try { + annotation = JSON.parse(decodeURIComponent(annotation)); + fillWithItemData([annotation.citationItem]); + annotation = encodeURIComponent(JSON.stringify(annotation)); + node.setAttribute('data-annotation', annotation); + } + catch (e) { + Zotero.logError(e); + } + } + } + + // Clone all note image attachments and replace keys in the new note let attachments = await Zotero.Items.getAsync(item.getAttachments()); for (let attachment of attachments) { let path = await attachment.getFilePathAsync(); @@ -315,9 +393,14 @@ class EditorInstance { } } }); - note = note.replace(attachment.key, clonedAttachment.key); + + let node = doc.querySelector(`img[data-attachment-key=${attachment.key}]`); + if (node) { + node.setAttribute('data-attachment-key', clonedAttachment.key); + } } - html += `

${note}

`; + + html += `

${doc.body.innerHTML}

`; } } return html; @@ -343,7 +426,7 @@ class EditorInstance { } else if (type === 'zotero/annotation') { let annotations = JSON.parse(data); - html = await this._serializeAnnotations(annotations); + [html] = await this._serializeAnnotations(annotations); } if (html) { this._postMessage({ action: 'insertHTML', pos, html }); @@ -462,6 +545,7 @@ class EditorInstance { availableCitationItems.push({ ...citationItem, id: item.id }); } } + // Notice: Citation items that don't exist in the library aren't shown in the popup citation.citationItems = availableCitationItems; let libraryID = this._item.libraryID; this._openQuickFormatDialog(nodeID, citation, [libraryID]); @@ -857,6 +941,7 @@ class EditorInstance { delete citationItem.id; citationItem.uris = [Zotero.URI.getItemURI(item)]; citationItem.itemData = Zotero.Cite.System.prototype.retrieveItem(item); + delete citationItem.itemData.abstract; } let formattedCitation = (await that._getFormattedCitationParts(citation)).join(';'); @@ -1019,8 +1104,12 @@ class EditorInstance { jsonAnnotations.push(jsonAnnotation); } let html = `

${Zotero.getString('note.annotationsWithDate', new Date().toLocaleString())}

\n`; - html += await editorInstance._serializeAnnotations(jsonAnnotations); - html = `
${html}
`; + let [serializedHTML, storedCitationItems] = await editorInstance._serializeAnnotations(jsonAnnotations, true); + + html += serializedHTML; + + storedCitationItems = encodeURIComponent(JSON.stringify(storedCitationItems)); + html = `
${html}
`; note.setNote(html); await note.saveTx(); return note; diff --git a/note-editor b/note-editor index f649d1a4e6..c5dfc982c0 160000 --- a/note-editor +++ b/note-editor @@ -1 +1 @@ -Subproject commit f649d1a4e6f74715a8b57ac298c058a07d8ea4e0 +Subproject commit c5dfc982c09915e5a2fb6103edf0395dc17ce728 diff --git a/pdf-reader b/pdf-reader index 310698f3a7..047f753389 160000 --- a/pdf-reader +++ b/pdf-reader @@ -1 +1 @@ -Subproject commit 310698f3a7ebadad8557db1f11e8aa5decaae432 +Subproject commit 047f753389e30e38e967cd48828ab8ea0441575c