scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-10-10 03:28:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Overhaul testing in Scaffold (#5301)

Browse source
This commit is contained in:
Abe Jellinek 2025-07-29 01:52:49 -04:00 • committed by Dan Stillman
parent 2e135d035f
commit c1270e635e
10 changed files with 571 additions and 523 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

583
chrome/content/scaffold/scaffold.js
View file

@ -26,9 +26,12 @@
var { E10SUtils } = ChromeUtils.importESModule("resource://gre/modules/E10SUtils.sys.mjs");
var { Subprocess } = ChromeUtils.importESModule("resource://gre/modules/Subprocess.sys.mjs");
var { RemoteTranslate } = ChromeUtils.importESModule("chrome://zotero/content/RemoteTranslate.mjs");
var { ContentDOMReference } = ChromeUtils.importESModule("resource://gre/modules/ContentDOMReference.sys.mjs");
var { Zotero } = ChromeUtils.importESModule("chrome://zotero/content/zotero.mjs");
var { FilePicker } = ChromeUtils.importESModule('chrome://zotero/content/modules/filePicker.mjs');
var { TranslatorTester } = ChromeUtils.importESModule('chrome://zotero/content/xpcom/translate/testTranslators/translatorTester.mjs');
var { Test } = ChromeUtils.importESModule('chrome://zotero/content/xpcom/translate/testTranslators/test.mjs');
var { ZoteroWebTranslationEnvironment } = ChromeUtils.importESModule('chrome://scaffold/content/zoteroWebTranslationEnvironment.mjs');
var { CompletionCopilot, registerCompletion } = ChromeUtils.importESModule('resource://zotero/monacopilot.mjs');
@ -44,6 +47,7 @@ var Scaffold = new function () {
var _translatorProvider = null;
var _lastModifiedTime = 0;
var _needRebuildTranslatorSuggestions = true;
var _invalidateCodeLenses = null;
var _copilot;
var _prefsObserverID;
@ -482,51 +486,37 @@ var Scaffold = new function () {
};
this.createTestCodeLensProvider = function (monaco, editor) {
let runTestsCommand = editor.addCommand(
let runCommand = editor.addCommand(
0,
(_ctx, testIndices) => {
let tests;
try {
tests = JSON.parse(editor.getValue());
}
catch (e) {
_logOutput('Error parsing tests:\n' + e);
}
if (testIndices) {
tests = testIndices.map(index => tests[index]);
}
this.runTests(tests);
async (_ctx, testIndices) => {
testIndices = testIndices || [..._loadTestsFromPane().keys()];
await this.runTests(testIndices);
},
'');
let updateTestsCommand = editor.addCommand(
let applyUpdatesCommand = editor.addCommand(
0,
async (_ctx, testIndices) => {
testIndices = testIndices || Object.keys(allTests);
try {
var allTests = JSON.parse(editor.getValue());
}
catch (e) {
_logOutput('Error parsing tests:\n' + e);
return;
}
let tests = testIndices.map(index => allTests[index]);
await this.updateTests(tests,
(newTest) => {
allTests[testIndices.shift()] = newTest;
});
_writeTestsToPane(allTests);
testIndices = testIndices || [..._loadTestsFromPane().keys()];
await this.updateTests(testIndices);
},
'');
return {
onDidChange: (callback) => {
_invalidateCodeLenses = callback;
return { dispose() {} };
},
provideCodeLenses: (model, _token) => {
let testsWithUpdates = new Set();
let testItems = Array.from(document.getElementById('testing-listbox').itemChildren);
for (let [testIndex, itemChild] of testItems.entries()) {
if (itemChild.dataset.updatedTestString) {
testsWithUpdates.add(testIndex);
}
}
let lenses = [];
let firstChar = {
@ -538,36 +528,41 @@ var Scaffold = new function () {
lenses.push({
range: firstChar,
command: {
id: runTestsCommand,
id: runCommand,
title: 'Run All'
}
});
lenses.push({
range: firstChar,
command: {
id: updateTestsCommand,
title: 'Run and Update All'
}
});
if (testsWithUpdates.size) {
lenses.push({
range: firstChar,
command: {
id: applyUpdatesCommand,
title: 'Apply All Updates'
}
});
}
for (let [testIndex, range] of _findTestObjectTops(monaco, model).entries()) {
lenses.push({
range: range,
command: {
id: runTestsCommand,
id: runCommand,
title: 'Run',
arguments: [[testIndex]]
}
});
lenses.push({
range: range,
command: {
id: updateTestsCommand,
title: 'Run and Update',
arguments: [[testIndex]]
}
});
if (testsWithUpdates.has(testIndex)) {
lenses.push({
range: range,
command: {
id: applyUpdatesCommand,
title: 'Apply Updates',
arguments: [[testIndex]]
}
});
}
}
return { lenses, dispose() {} };
@ -935,20 +930,27 @@ var Scaffold = new function () {
}
};
this.handleTestSelect = function (event) {
let selected = event.target.selectedItems[0];
if (!selected) return;
this.handleTestingContextMenuShowing = function () {
let selectedItems = Array.from(document.getElementById('testing-listbox').selectedItems);
if (!selectedItems.length) return;
let editImport = document.getElementById('testing_editImport');
let openURL = document.getElementById('testing_openURL');
if (selected.dataset.testType == 'web') {
editImport.setAttribute('disabled', true);
openURL.removeAttribute('disabled');
let editImport = document.getElementById('testing-editImport');
let openURL = document.getElementById('testing-openURL');
let applyUpdates = document.getElementById('testing-applyUpdates');
if (selectedItems.length > 1) {
editImport.disabled = true;
openURL.disabled = true;
}
else {
editImport.removeAttribute('disabled');
openURL.setAttribute('disabled', true);
let selectedItem = selectedItems[0];
editImport.disabled = selectedItem.dataset.testType === 'web';
openURL.disabled = selectedItem.dataset.testType !== 'web';
}
applyUpdates.disabled = selectedItems.some(item => !item.dataset.updatedTestString);
};
this.handleTestingListboxDblClick = function () {
this.runSelectedTests();
};
// Add special keydown handling for the editors
@ -1214,12 +1216,11 @@ var Scaffold = new function () {
/*
* called to select items
*/
function _selectItems(obj, itemList) {
function _selectItems(obj, itemList, callback) {
var io = { dataIn: itemList, dataOut: null };
window.openDialog("chrome://scaffold/content/select.xhtml",
"_blank", "chrome,modal,centerscreen,resizable=yes", io);
return io.dataOut;
callback(io.dataOut);
}
/*
@ -1240,18 +1241,8 @@ var Scaffold = new function () {
* logs item output
*/
function _myItemDone(obj, jsonItem) {
// If the pane hasn't been resized, it won't have a fixed width,
// so it'll grow when wrapping text is added. Fix its width now.
let rightPane = document.querySelector('#right-pane');
rightPane.style.width = rightPane.getBoundingClientRect().width + 'px';
let itemPreviews = document.querySelector('#item-previews');
if (itemPreviews.hidden) {
itemPreviews.hidden = false;
itemPreviews.jsonItems = [];
}
itemPreviews.jsonItems.push(jsonItem);
itemPreviews.render();
itemPreviews.addItemPair(jsonItem, null);
}
/*
@ -1474,23 +1465,6 @@ var Scaffold = new function () {
_writeToEditor(_editors.tests, _stringifyTests(tests));
}
function _confirmCreateExpectedFailTest() {
return Services.prompt.confirm(null,
'Detection Failed',
'Add test ensuring that detection always fails on this page?');
}
/* sanitizes all items in a test
*/
function _sanitizeItemsInTest(test) {
if (test.items && typeof test.items != 'string' && test.items.length) {
for (var i = 0, n = test.items.length; i < n; i++) {
test.items[i] = Zotero_TranslatorTester._sanitizeItem(test.items[i]);
}
}
return test;
}
/* stringifies an array of tests
* Output is the same as JSON.stringify (with pretty print), except that
* Zotero.Item objects are stringified in a deterministic manner (mostly):
@ -1610,6 +1584,7 @@ var Scaffold = new function () {
_writeTestsToPane([..._loadTestsFromPane(), test]);
}
catch (e) {
Zotero.logError(e);
_logOutput('Creation failed');
return;
}
@ -1626,51 +1601,49 @@ var Scaffold = new function () {
|| (type === "import" && !document.getElementById('checkbox-import').checked)
|| (type === "search" && !document.getElementById('checkbox-search').checked)) {
_logOutput(`Translator does not support ${type} tests`);
return Promise.reject(new Error());
throw new Error();
}
if (type == 'export') {
return Promise.reject(new Error(`Test of type export cannot be created`));
throw new Error(`Test of type export cannot be created`);
}
let input = await _getInput(type);
if (type == "web") {
let translate = new RemoteTranslate({ disableErrorReporting: true });
try {
await translate.setBrowser(_browser);
await translate.setTranslatorProvider(_translatorProvider);
translate.setTranslator(_getTranslatorFromPane());
translate.setHandler("debug", _debug);
translate.setHandler("error", _error);
translate.setHandler("newTestDetectionFailed", _confirmCreateExpectedFailTest);
let newTest = await translate.newTest();
if (!newTest) {
throw new Error('Creation failed');
}
newTest = _sanitizeItemsInTest(newTest);
return newTest;
}
finally {
translate.dispose();
}
if (input === _browser) {
input = _getCurrentURI(_browser);
}
else if (type == "import" || type == "search") {
let test = { type, input: input, items: [] };
// TranslatorTester doesn't handle these correctly, so we do it manually
return new Promise(
resolve => _run(`do${type == 'import' ? 'Import' : 'Search'}`, input, null, function (obj, item) {
if (item) {
test.items.push(Zotero_TranslatorTester._sanitizeItem(item));
}
}, null, function () {
resolve(test);
})
let testDraft = new Test({ type, input, items: [] });
let [{ updatedTest: test }] = await Array.fromAsync(
this.runTestsInternal([testDraft])
);
// If we didn't get an item the first time, try again with defer: true
if (!test) {
testDraft.defer = true;
let [{ updatedTest }] = await Array.fromAsync(
this.runTestsInternal([testDraft])
);
test = updatedTest;
}
return Promise.reject(new Error('Invalid type: ' + type));
// But give up after that
if (!test) {
if (Services.prompt.confirm(
null,
'Detection Failed',
`Add test ensuring that detection always fails on this ${type === 'web' ? 'page' : 'input'}?`
)) {
testDraft.detectedItemType = false;
testDraft.defer = false;
return testDraft.toJSON();
}
else {
throw new Error('Not creating expected-fail test');
}
}
return test.toJSON();
};
/*
@ -1698,15 +1671,16 @@ var Scaffold = new function () {
let listBox = document.getElementById("testing-listbox");
let count = listBox.getRowCount();
let oldStatuses = {};
let oldStatusNodes = {};
for (let i = 0; i < count; i++) {
let item = listBox.getItemAtIndex(i);
let [, statusCell] = item.children;
oldStatuses[item.dataset.testString] = statusCell.textContent;
oldStatusNodes[item.dataset.testString] = item.querySelector('.status');
}
let testIndex = 0;
for (let test of tests) {
while (listBox.itemChildren.length > tests.length) {
listBox.itemChildren[listBox.itemChildren.length - 1].remove();
}
for (let [testIndex, test] of tests.entries()) {
let testString = _stringifyTests(test, 1);
// try to reuse old rows
@ -1720,8 +1694,25 @@ var Scaffold = new function () {
input.append(getTestLabel(test));
item.appendChild(wrapWithHBox(input, { flex: 1 }));
let status = document.createXULElement('label');
status.append(oldStatuses[testString] || 'Not run');
let status = oldStatusNodes[testString];
if (!status) {
status = document.createXULElement('label');
status.classList.add('status');
status.addEventListener('click', (event) => {
if (!status.classList.contains('needs-update')) {
return;
}
event.preventDefault();
if (!Services.prompt.confirm(
null,
'Scaffold',
`Apply updates to this test?`
)) {
return;
}
this.updateTests([testIndex]);
});
}
item.appendChild(wrapWithHBox(status, { width: 150 }));
let defer = document.createXULElement('checkbox');
@ -1738,19 +1729,16 @@ var Scaffold = new function () {
});
item.appendChild(wrapWithHBox(defer, { pack: 'center', width: 75 }));
// Remove updatedTestString if the test changed or updates have been applied
if (item.dataset.testString !== testString || item.dataset.updatedTestString === testString) {
delete item.dataset.updatedTestString;
}
item.dataset.testString = testString;
item.dataset.testType = test.type;
if (testIndex >= count) {
listBox.appendChild(item);
}
testIndex++;
}
// remove old rows that we didn't reuse
while (listBox.getItemAtIndex(testIndex)) {
listBox.getItemAtIndex(testIndex).remove();
}
};
@ -1762,7 +1750,7 @@ var Scaffold = new function () {
var indicesToRemove = [...listbox.selectedItems].map(item => listbox.getIndexOfItem(item));
let tests = _loadTestsFromPane();
indicesToRemove.forEach(i => tests.splice(i, 1));
tests = tests.filter((_, i) => !indicesToRemove.includes(i));
_writeTestsToPane(tests);
this.populateTests();
@ -1798,7 +1786,7 @@ var Scaffold = new function () {
this.copyToClipboard = function () {
var listbox = document.getElementById("testing-listbox");
var item = listbox.selectedItems[0];
var url = item.getElementsByTagName("label")[0].getAttribute("value");
var url = item.getElementsByTagName("label")[0].textContent;
var test = JSON.parse(item.dataset.testString);
var urlOrData = (test.input !== undefined) ? test.input : url;
if (typeof urlOrData !== 'string') {
@ -1826,110 +1814,137 @@ var Scaffold = new function () {
_showTab('browser');
}
};
this.runTests = async function (testIndices) {
let listbox = document.getElementById('testing-listbox');
let items = [...listbox.itemChildren];
let itemsToUpdate = testIndices.map(index => items[index]);
this.runTests = function (tests, callback) {
callback = callback || (() => {});
let itemPreviews = document.querySelector('#item-previews');
_clearOutput();
let tests = [];
for (let listItem of itemsToUpdate) {
listItem.querySelector('.status').textContent = 'Running';
let testsByType = {
import: [],
export: [],
web: [],
search: []
};
for (let test of tests) {
testsByType[test.type].push(test);
let test = new Test(JSON.parse(listItem.dataset.testString));
tests.push(test);
}
let numTests = tests.length;
let currentTest = 1; // For logging
_logOutput(`Running ${numTests} ${Zotero.Utilities.pluralize(numTests, 'test')}`);
for await (let { test, status, reason, updatedTest } of this.runTestsInternal(tests)) {
let statusText;
let needsUpdate = false;
let logPrefix = `Test ${currentTest}/${numTests}: `;
if (status === 'success') {
statusText = 'Succeeded';
_logOutput(logPrefix + statusText);
}
else {
statusText = reason;
needsUpdate = !!updatedTest;
_logOutput(logPrefix + statusText);
}
// Show the preview pane as long as we got new item data,
// even if there weren't substantive changes
let totalStats = { added: 0, removed: 0 };
if (Array.isArray(test.items) && Array.isArray(updatedTest?.items)) {
for (let i = 0; i < test.items.length || i < updatedTest.items.length; i++) {
let preItem = i < test.items.length ? test.items[i] : {};
let postItem = i < updatedTest.items.length ? updatedTest.items[i] : {};
let preview = itemPreviews.addItemPair(preItem, postItem);
let statsHere = preview.diffStats;
totalStats.added += statsHere.added;
totalStats.removed += statsHere.removed;
}
}
let listItem = itemsToUpdate[tests.indexOf(test)];
let statusLabel = listItem.querySelector('.status');
statusLabel.classList.toggle('needs-update', needsUpdate);
let textElem = document.createElement('span');
textElem.classList.add('text');
textElem.textContent = statusText;
let addedElem = document.createElement('span');
addedElem.classList.add('added');
addedElem.textContent = `+${totalStats.added}`;
let removedElem = document.createElement('span');
removedElem.classList.add('removed');
removedElem.textContent = `-${totalStats.removed}`;
statusLabel.replaceChildren(textElem, addedElem, removedElem);
if (needsUpdate) {
listItem.dataset.updatedTestString = _stringifyTests(updatedTest.toJSON(), 1);
}
else {
delete listItem.dataset.updatedTestString;
}
currentTest++;
}
_invalidateCodeLenses?.();
};
this.runSelectedTests = async function () {
let listbox = document.getElementById('testing-listbox');
let selectedItems = [...listbox.selectedItems];
await this.runTests(
[...selectedItems].map(item => listbox.getIndexOfItem(item))
);
};
this.runTestsInternal = async function* (tests) {
_clearOutput();
let rememberCookies = document.getElementById('checkbox-remember-cookies').checked;
for (let [type, testsOfType] of Object.entries(testsByType)) {
if (testsOfType.length) {
let tester = new Zotero_TranslatorTester(
_getTranslatorFromPane(),
type,
_debug,
_translatorProvider
);
if (!rememberCookies) {
tester.setCookieSandbox(new Zotero.CookieSandbox());
}
tester.setTests(testsOfType);
tester.runTests(callback);
}
}
};
/*
* Run selected test(s)
*/
this.runSelectedTests = function () {
var listbox = document.getElementById("testing-listbox");
var items = listbox.selectedItems;
if (!items || items.length == 0) return; // No action if nothing selected
var tests = [];
for (let item of items) {
item.getElementsByTagName("label")[1].textContent = "Running";
var test = JSON.parse(item.dataset.testString);
test["ui-item"] = ContentDOMReference.get(item);
tests.push(test);
}
this.runTests(tests, (obj, test, status, message) => {
ContentDOMReference.resolve(test["ui-item"]).getElementsByTagName("label")[1].textContent = message;
let tester = new TranslatorTester(_getTranslatorFromPane(), {
translatorProvider: _translatorProvider,
cookieSandbox: rememberCookies ? null : new Zotero.CookieSandbox(),
debug: _logOutput,
webTranslationEnvironment: new ZoteroWebTranslationEnvironment(),
});
for (let test of tests) {
yield { test, ...await tester.run(test) };
}
};
this.updateTests = function (tests, testUpdatedCallback) {
_clearOutput();
this.updateTests = async function (testIndices) {
let listbox = document.getElementById('testing-listbox');
let items = [...listbox.itemChildren];
let itemsToUpdate = testIndices.map(index => items[index]);
var updater = new TestUpdater(tests);
return new Promise(resolve => updater.updateTests(
testUpdatedCallback,
resolve
));
let tests = _loadTestsFromPane();
for (let item of itemsToUpdate) {
let updatedTestString = item.dataset.updatedTestString;
if (!updatedTestString) {
continue;
}
let updatedTest = JSON.parse(updatedTestString);
tests[items.indexOf(item)] = updatedTest;
item.dataset.testString = _stringifyTests(updatedTest, 1);
let status = item.querySelector('.status');
status.textContent = 'Updated';
status.classList.remove('needs-update');
}
_writeTestsToPane(tests);
_logOutput('Tests updated.');
};
/*
* Update selected test(s)
*/
this.updateSelectedTests = async function () {
var listbox = document.getElementById("testing-listbox");
var items = [...listbox.selectedItems];
if (!items || items.length == 0) return; // No action if nothing selected
var itemIndices = items.map(item => listbox.getIndexOfItem(item));
var tests = [];
for (let item of items) {
item.getElementsByTagName("label")[1].textContent = "Updating";
var test = JSON.parse(item.dataset.testString);
tests.push(test);
}
var testsDone = 0;
await this.updateTests(tests,
(newTest) => {
let message;
// Assume sequential. TODO: handle this properly via test ID of some sort
if (newTest) {
message = "Test updated";
items[testsDone].dataset.testString = _stringifyTests(newTest, 1);
tests[testsDone] = newTest;
}
else {
message = "Update failed";
}
items[testsDone].getElementsByTagName("label")[1].textContent = message;
testsDone++;
});
let allTests = _loadTestsFromPane();
for (let [i, test] of Object.entries(tests)) {
allTests[itemIndices[i]] = test;
}
_writeTestsToPane(allTests);
_logOutput("Tests updated.");
let listbox = document.getElementById('testing-listbox');
let selectedItems = [...listbox.selectedItems];
await this.updateTests(
[...selectedItems].map(item => listbox.getIndexOfItem(item))
);
};
this.populateLinterMenu = function () {
@ -1954,98 +1969,6 @@ var Scaffold = new function () {
tabBox.selectedIndex = tabNumber - 1;
};
var TestUpdater = function (tests) {
this.testsToUpdate = tests.slice();
this.numTestsTotal = this.testsToUpdate.length;
this.newTests = [];
};
TestUpdater.prototype.updateTests = function (testDoneCallback, doneCallback) {
this.testDoneCallback = testDoneCallback || function () { /* no-op */ };
this.doneCallback = doneCallback || function () { /* no-op */ };
this._updateTests();
};
TestUpdater.prototype._updateTests = async function () {
if (!this.testsToUpdate.length) {
this.doneCallback(this.newTests);
return;
}
var test = this.testsToUpdate.shift();
_logOutput("Updating test " + (this.numTestsTotal - this.testsToUpdate.length));
if (test.type == 'web') {
_logOutput("Loading web page from " + test.url);
const { HiddenBrowser } = ChromeUtils.importESModule("chrome://zotero/content/HiddenBrowser.mjs");
let browser = new HiddenBrowser({
docShell: { allowMetaRedirects: true }
});
try {
await browser.load(test.url, {
requireSuccessfulStatus: true
});
let translate = new RemoteTranslate({ disableErrorReporting: true });
try {
await translate.setBrowser(browser);
await translate.setTranslatorProvider(_translatorProvider);
translate.setTranslator(_getTranslatorFromPane());
translate.setHandler("debug", _debug);
translate.setHandler("error", _error);
translate.setHandler("newTestDetectionFailed", _confirmCreateExpectedFailTest);
let newTest = await translate.newTest({ defer: test.defer });
newTest = _sanitizeItemsInTest(newTest);
if (newTest.url != test.url) {
_logOutput("Page URL differs from test. Will be updated. " + newTest.url);
}
this.newTests.push(newTest);
this.testDoneCallback(newTest);
this._updateTests();
}
finally {
translate.dispose();
}
}
catch (e) {
Zotero.logError(e);
this.newTests.push(false);
this.testDoneCallback(false);
this._updateTests();
}
finally {
if (browser) browser.destroy();
}
}
else {
test.items = [];
const methods = {
import: 'doImport',
export: 'doExport', // not supported, will error
search: 'doSearch'
};
// Re-runs the test.
// TranslatorTester doesn't handle these correctly, so we do it manually
_run(methods[test.type], test.input, null, (obj, item) => {
if (item) {
test.items.push(Zotero_TranslatorTester._sanitizeItem(item));
}
}, null, () => {
if (!test.items.length) test = false;
this.newTests.push(test);
this.testDoneCallback(test);
this._updateTests();
});
}
};
/*
* Normalize whitespace to the Zotero norm of tabs
*/
@ -2060,8 +1983,8 @@ var Scaffold = new function () {
*/
function _clearOutput() {
document.getElementById('output').value = '';
let itemPane = document.querySelector('#item-previews');
itemPane.hidden = true;
let itemPreviews = document.querySelector('#item-previews');
itemPreviews.clearItemPairs();
}
/*

33
chrome/content/scaffold/scaffold.xhtml
View file

@ -61,7 +61,6 @@
Services.scriptloader.loadSubScript("chrome://zotero/content/titlebar.js", this);
Services.scriptloader.loadSubScript("chrome://zotero/content/platformKeys.js", this);
Services.scriptloader.loadSubScript("chrome://zotero/content/editMenuOverlay.js", this);
Services.scriptloader.loadSubScript("chrome://zotero/content/xpcom/translate/testTranslators/translatorTester.js", this);
Services.scriptloader.loadSubScript("chrome://scaffold/content/translators.js", this);
Services.scriptloader.loadSubScript("chrome://scaffold/content/scaffold.js", this);
@ -139,15 +138,19 @@
</keyset>
<popupset>
<menupopup id="testing-context-menu">
<menupopup id="testing-context-menu" onpopupshowing="Scaffold.handleTestingContextMenuShowing()">
<menuitem label="&scaffold.testing.copyToClipboard;" tooltiptext="Copy the URL or data for the current test to the clipboard" oncommand="Scaffold.copyToClipboard()"/>
<menuitem id="testing_editImport" label="&scaffold.testing.edit;" tooltiptext="Edit the input data for the current test" oncommand="Scaffold.editImportFromTest()"/>
<menu id="testing_openURL" label="&scaffold.testing.openUrl;">
<menuitem id="testing-editImport" label="&scaffold.testing.edit;" tooltiptext="Edit the input data for the current test" oncommand="Scaffold.editImportFromTest()"/>
<menu id="testing-openURL" label="&scaffold.testing.openUrl;">
<menupopup>
<menuitem label="&scaffold.testing.openUrl.internally;" tooltiptext="Open the URL for the current test in the Scaffold browser" oncommand="Scaffold.openURL(false)"/>
<menuitem label="&scaffold.testing.openUrl.externally;" tooltiptext="Open the URL for the current test in your default browser" oncommand="Scaffold.openURL(true)"/>
</menupopup>
</menu>
<menuseparator/>
<menuitem id="testing-delete" label="Delete" oncommand="Scaffold.deleteSelectedTests()"/>
<menuitem id="testing-run" label="Run" oncommand="Scaffold.runSelectedTests()"/>
<menuitem id="testing-applyUpdates" label="&scaffold.testing.applyUpdates;" oncommand="Scaffold.updateSelectedTests()"/>
</menupopup>
</popupset>
@ -488,24 +491,18 @@
observes="validate-tests"
flex="1"
seltype="multiple"
onselect="Scaffold.handleTestSelect(event)"
context="testing-context-menu"
focus-on-tab-select="true"
ondblclick="Scaffold.handleTestingListboxDblClick()"
/>
</vbox>
<hbox id="testing-buttons">
<button observes="validate-tests" label="&scaffold.testing.delete;" tooltiptext="Delete the selected tests" oncommand="Scaffold.deleteSelectedTests()"/>
<button observes="validate-tests" label="&scaffold.testing.run;" tooltiptext="Run the selected tests" oncommand="Scaffold.runSelectedTests()"/>
<button observes="validate-tests" label="&scaffold.testing.update;" tooltiptext="Run the selected tests and update the test definitions with the latest data" oncommand="Scaffold.updateSelectedTests()"/>
<hbox flex="1" pack="end">
<checkbox
id="checkbox-remember-cookies"
label="&scaffold.testing.rememberCookies;"
native="true"
checked="true"
/>
</hbox>
<hbox pack="end">
<checkbox
id="checkbox-remember-cookies"
label="&scaffold.testing.rememberCookies;"
native="true"
checked="true"
/>
</hbox>
<iframe src="monaco/monaco.html" id="editor-tests" onmousedown="this.focus()"/>
</vbox>

181
chrome/content/scaffold/scaffoldItemPreview.js
View file

@ -26,6 +26,8 @@
"use strict";
{
let { diff } = ChromeUtils.importESModule('chrome://zotero/content/xpcom/translate/testTranslators/diff.mjs');
class ScaffoldItemPreview extends XULElementBase {
content = MozXULElement.parseXULToFragment(`
<html:div class="diff" readonly="true"/>
@ -36,34 +38,85 @@
<tags-box/>
`);
_jsonItem = null;
_preItem = null;
_postItem = null;
_cachedDiff = null;
get jsonItem() {
return this._jsonItem;
get itemPair() {
return [this._preItem, this._postItem];
}
set jsonItem(jsonItem) {
// Don't pass this any objects you really care about
// Delete ID (meaningless)
delete jsonItem.id;
// _itemDone() sets 'title' in addition to the item type's mapped
// title field for some reason. Delete it so it doesn't show up in the diff.
let titleField = Zotero.ItemFields.getName(
Zotero.ItemFields.getFieldIDFromTypeAndBase(jsonItem.itemType, 'title')
);
if (titleField !== 'title' && jsonItem[titleField]) {
delete jsonItem.title;
set itemPair([preItem, postItem]) {
this._preItem = this._normalizeItem(preItem);
if (!postItem) {
// Default to cleaned version of preItem
let zoteroPreItem = new Zotero.Item();
zoteroPreItem.libraryID = Zotero.Libraries.userLibraryID;
zoteroPreItem.fromJSON(this._preItem);
postItem = Zotero.Utilities.Internal.itemToExportFormat(zoteroPreItem, {
legacy: true,
skipBaseFields: true
});
delete postItem.relations;
delete postItem.collections;
delete postItem.uniqueFields;
delete postItem.uri;
for (let [key, value] of Object.entries(postItem)) {
if (value === null) {
delete postItem[key];
}
}
// Because we're just diffing the result of cleaning, differences
// in these arrays are mostly meaningless
postItem.attachments = structuredClone(preItem.attachments);
postItem.notes = structuredClone(preItem.notes);
postItem.tags = structuredClone(preItem.tags);
}
if (jsonItem.accessDate === 'CURRENT_TIMESTAMP') {
jsonItem.accessDate = Zotero.Date.dateToISO(new Date());
}
this._jsonItem = jsonItem;
this._postItem = this._normalizeItem(postItem);
this._cachedDiff = null;
this.render();
}
get diff() {
if (!this._cachedDiff) {
if (!this._preItem || !this._postItem) {
return null;
}
let diffString = diff(this._preItem, this._postItem)
.split('\n')
.filter(line => (
// Remove lines without differences
!/^\s*"\w+": (".*"|\d+|true|false)$/.test(line)
// And accessDate diff lines - they just show a meaningless format change
&& !/^\s*[-+]\s*"accessDate": "/.test(line)
))
.join('\n');
// Remove now-empty object/array literals
let emptyLiteralRe = /(\n|^)\s*("\w+": )?(\{\s*}|\[\s*])/g;
while (emptyLiteralRe.test(diffString)) {
diffString = diffString.replace(emptyLiteralRe, '');
}
this._cachedDiff = diffString;
}
return this._cachedDiff;
}
get diffStats() {
let added = 0;
let removed = 0;
for (let diffLine of this.diff.split('\n')) {
if (diffLine.startsWith('+')) {
added++;
}
else if (diffLine.startsWith('-')) {
removed++;
}
}
return { added, removed };
}
init() {
this.render();
@ -72,22 +125,32 @@
render() {
if (!this.initialized) return;
let zoteroItem = new Zotero.Item();
zoteroItem.libraryID = Zotero.Libraries.userLibraryID;
zoteroItem.fromJSON(this._jsonItem);
let postItem = this._postItem;
let zoteroPostItem = new Zotero.Item();
zoteroPostItem.libraryID = Zotero.Libraries.userLibraryID;
if (Object.entries(postItem).length) {
zoteroPostItem.fromJSON(postItem);
}
let [diffBox, infoBox, abstractBox, attachmentsPreview, notesPreview, tagsBox] = this.children;
for (let box of [infoBox, abstractBox, tagsBox]) {
box.mode = 'view';
box.editable = false;
box.item = zoteroItem;
box.item = zoteroPostItem;
box._forceRenderAll();
}
attachmentsPreview.replaceChildren(
...this._jsonItem.attachments.map((jsonAttachment) => {
...(postItem.attachments ?? []).map((jsonAttachment) => {
let zoteroAttachment = new Zotero.Item('attachment');
zoteroAttachment.setField('title', `${jsonAttachment.title} (${jsonAttachment.url || jsonAttachment.path})`);
let displayTitle = jsonAttachment.title;
let urlOrPath = jsonAttachment.url || jsonAttachment.path;
if (urlOrPath) {
displayTitle += ` (${urlOrPath})`;
}
zoteroAttachment.setField('title', displayTitle);
zoteroAttachment.attachmentContentType = jsonAttachment.mimeType;
if (jsonAttachment.snapshot === false && jsonAttachment.mimeType === 'text/html') {
zoteroAttachment.attachmentLinkMode = Zotero.Attachments.LINK_MODE_LINKED_URL;
@ -102,7 +165,7 @@
})
);
notesPreview.replaceChildren(
...this._jsonItem.notes.map((jsonNote) => {
...(postItem.notes ?? []).map((jsonNote) => {
let row = document.createXULElement('note-row');
row.note = {
title: '',
@ -112,39 +175,7 @@
})
);
let preCleaning = JSON.parse(JSON.stringify(this._jsonItem));
let postCleaning = Zotero.Utilities.Internal.itemToExportFormat(zoteroItem, {
legacy: true,
skipBaseFields: true
});
delete postCleaning.relations;
delete postCleaning.collections;
delete postCleaning.uniqueFields;
delete postCleaning.uri;
for (let [key, value] of Object.entries(postCleaning)) {
if (value === null) {
delete postCleaning[key];
}
}
// Differences in these arrays are mostly meaningless
postCleaning.attachments = preCleaning.attachments;
postCleaning.notes = preCleaning.notes;
postCleaning.tags = preCleaning.tags;
let diff = Zotero_TranslatorTester._generateDiff(preCleaning, postCleaning)
.split('\n')
.filter(line => (
// Remove lines without differences
!/^\s*"\w+": (".*"|\d+|true|false)$/.test(line)
// And accessDate diff lines - they just show a meaningless format change
&& !/^\s*[-+]\s*"accessDate": "/.test(line)
))
.join('\n');
// Remove now-empty object/array literals
let emptyLiteralRe = /(\n|^)\s*("\w+": )?(\{\s*}|\[\s*])/g;
while (emptyLiteralRe.test(diff)) {
diff = diff.replace(emptyLiteralRe, '');
}
for (let diffLine of diff.split('\n')) {
for (let diffLine of this.diff.split('\n')) {
let lineContainer = document.createElement('div');
lineContainer.textContent = diffLine;
if (diffLine.startsWith('+')) {
@ -156,6 +187,32 @@
diffBox.append(lineContainer);
}
}
_normalizeItem(jsonItem) {
jsonItem = JSON.parse(JSON.stringify(jsonItem));
if (!Object.entries(jsonItem).length) {
return jsonItem;
}
// Delete ID (meaningless)
delete jsonItem.id;
// _itemDone() sets 'title' in addition to the item type's mapped
// title field for some reason. Delete it so it doesn't show up in the diff.
let titleField = Zotero.ItemFields.getName(
Zotero.ItemFields.getFieldIDFromTypeAndBase(jsonItem.itemType, 'title')
);
if (titleField !== 'title' && jsonItem[titleField]) {
delete jsonItem.title;
}
if (jsonItem.accessDate === 'CURRENT_TIMESTAMP') {
jsonItem.accessDate = Zotero.Date.dateToISO(new Date());
}
return jsonItem;
}
}
customElements.define('scaffold-item-preview', ScaffoldItemPreview);

75
chrome/content/scaffold/scaffoldItemPreviews.js
View file

@ -35,55 +35,66 @@
</html:div>
<deck/>
`);
_deck;
_jsonItems = [];
_switcher;
get jsonItems() {
return this._jsonItems;
}
set jsonItems(jsonItems) {
this._jsonItems = jsonItems;
this.render();
}
_itemPairs = [];
init() {
this._deck = this.querySelector('deck');
this._switcher = this.querySelector('.switcher');
this.querySelector('.previous').addEventListener('command', () => {
this.querySelector('deck').selectedIndex--;
this.render();
this._deck.selectedIndex--;
this._renderSwitcher();
});
this.querySelector('.next').addEventListener('command', () => {
this.querySelector('deck').selectedIndex++;
this.render();
this._deck.selectedIndex++;
this._renderSwitcher();
});
}
clearItemPairs() {
this._itemPairs = [];
this._deck.selectedIndex = 0;
this._deck.replaceChildren();
this._renderSwitcher();
this.render();
this.hidden = true;
}
render() {
if (!this.initialized) return;
/**
* @param {any} preItem JSON item
* @param {any} [postItem] JSON item (defaults to cleaned version of preItem)
* @returns {ScaffoldItemPreview}
*/
addItemPair(preItem, postItem) {
let preview = document.createXULElement('scaffold-item-preview');
preview.itemPair = [preItem, postItem];
this._deck.append(preview);
this._renderSwitcher();
let deck = this.querySelector('deck');
for (let [i, jsonItem] of this._jsonItems.entries()) {
if (deck.children[i]?.jsonItem === jsonItem) {
continue;
if (this.hidden) {
let splitterPane = this.closest('splitter + *');
if (splitterPane) {
// If the pane hasn't been resized, it won't have a fixed width,
// so it'll grow when wrapping text is added. Fix its width now.
splitterPane.style.width = splitterPane.getBoundingClientRect().width + 'px';
}
let preview = document.createXULElement('scaffold-item-preview');
preview.jsonItem = jsonItem;
deck.append(preview);
}
while (deck.children.length > this._jsonItems.length) {
deck.lastElementChild.remove();
}
if (deck.selectedIndex >= deck.children.length) {
deck.selectedIndex = 0;
this.hidden = false;
}
return preview;
}
_renderSwitcher() {
let switcher = this.querySelector('.switcher');
if (deck.children.length > 1) {
if (this._deck.children.length > 1) {
switcher.hidden = false;
switcher.querySelector('.current').textContent = deck.selectedIndex + 1;
switcher.querySelector('.max').textContent = deck.children.length;
switcher.querySelector('.current').textContent = this._deck.selectedIndex + 1;
switcher.querySelector('.max').textContent = this._deck.children.length;
}
else {
switcher.hidden = true;

77
chrome/content/scaffold/zoteroWebTranslationEnvironment.mjs Normal file
View file

@ -0,0 +1,77 @@
import { AbstractWebTranslationEnvironment } from 'chrome://zotero/content/xpcom/translate/testTranslators/translatorTester.mjs';
export class ZoteroWebTranslationEnvironment extends AbstractWebTranslationEnvironment {
/**
* @param {string} url
* @param {TranslatorTester} tester
* @returns {Promise<HiddenBrowser>}
*/
async fetchPage(url, { tester }) {
const { HiddenBrowser } = ChromeUtils.importESModule('chrome://zotero/content/HiddenBrowser.mjs');
let browser = new HiddenBrowser({
docShell: { allowMetaRedirects: true },
cookieSandbox: tester.cookieSandbox,
});
await browser.load(url, { requireSuccessfulStatus: true });
return browser;
}
/**
* @param {HiddenBrowser} browser
* @param {TranslatorTester} tester
* @returns {Promise<false>} False for now, because we aren't sure that
* detection would work without the defer delay.
*/
async waitForLoad(browser, { tester }) {
return false;
}
/**
* @param {HiddenBrowser} browser
* @param {TranslatorTester} tester
* @param {Record<string, Function>} handlers
* @param {AbortSignal} signal
* @returns {Promise<{
* detectedItemType?: string;
* items?: Zotero.Item[];
* reason?: string;
* }>}
*/
async runTranslation(browser, { tester, handlers, signal }) {
const { RemoteTranslate } = ChromeUtils.importESModule('chrome://zotero/content/RemoteTranslate.mjs');
let translate = new RemoteTranslate({ disableErrorReporting: true });
try {
await translate.setBrowser(browser);
await translate.setTranslatorProvider(tester.translatorProvider);
translate.setTranslator(tester.translator);
for (let [type, fn] of Object.entries(handlers)) {
translate.setHandler(type, fn);
}
let detectedTranslators = await translate.detect();
if (!detectedTranslators.length) {
return { items: null, reason: 'Detection failed' };
}
let detectedItemType = detectedTranslators[0].itemType;
let items = await translate.translate({ libraryID: false });
return { detectedItemType, items };
}
catch (e) {
return { items: null, reason: 'Translation failed: ' + e };
}
finally {
translate.dispose();
}
}
/**
* @param {HiddenBrowser} browser
*/
destroy(browser) {
browser.destroy();
}
}

3
chrome/content/zotero/RemoteTranslate.mjs
View file

@ -80,8 +80,7 @@ export class RemoteTranslate {
* Set a handler on the proxied Zotero.Translate instance.
* The handler function is passed this RemoteTranslate as its first argument.
*
* Supports all Zotero.Translate handlers in addition to newTestDetectionFailed, which can be called from
* {@link newTest} if detection fails to confirm the creation of an expected-fail test.
* Supports all Zotero.Translate handlers.
*
* @param {String} name
* @param {Function} handler

69
chrome/content/zotero/actors/TranslationChild.mjs
View file

@ -30,7 +30,6 @@ const TRANSLATE_SCRIPT_PATHS = [
'src/rdf/n3parser.js',
'src/rdf/rdfparser.js',
'src/rdf/serialize.js',
'testTranslators/translatorTester.js',
];
const OTHER_SCRIPT_URIS = [
@ -88,72 +87,6 @@ export class TranslationChild extends JSWindowActorChild {
return null;
}
}
case 'runTest': {
let { translator, test, id } = data;
let { Zotero_TranslatorTester } = this._sandbox;
try {
let tester = new Zotero_TranslatorTester(
Cu.cloneInto(translator, this._sandbox),
test.type,
(_tester, obj) => this._debug(id, obj),
this._makeTranslatorProvider(id),
);
return await new Promise((resolve) => {
tester.runTest(
Cu.cloneInto(test, this._sandbox),
this.contentWindow.document,
Cu.exportFunction(
(_, test, status, message) => resolve({ test, status, message }),
this._sandbox
)
);
});
}
catch (e) {
this._error(id, e);
return null;
}
}
case 'newTest': {
let { translator, testInit, id } = data;
let { Zotero_TranslatorTester } = this._sandbox;
try {
let tester = new Zotero_TranslatorTester(
Cu.cloneInto(translator, this._sandbox),
'web',
(_tester, obj) => this._debug(id, obj),
this._makeTranslatorProvider(id),
);
if (testInit) {
await tester.waitForDeferDelay(testInit);
}
return await new Promise((resolve) => {
tester.newTest(
this.contentWindow.document,
Cu.exportFunction(
(_, test) => {
if (testInit?.defer) {
test.defer = testInit.defer;
}
resolve(test);
},
this._sandbox
),
Cu.exportFunction(
() => this._sendQuerySafe('Translate:runHandler', {
id,
name: 'newTestDetectionFailed'
}),
this._sandbox
)
);
});
}
catch (e) {
this._error(id, e);
return null;
}
}
}
}
@ -248,7 +181,7 @@ export class TranslationChild extends JSWindowActorChild {
}
else if (name == 'select') {
handler = (_, items, callback) => {
this.sendQuery('Translate:runHandler', { id, name, arg: items }).then(items => {
this.sendQuery('Translate:runHandler', { id, name, arg: items }).then((items) => {
callback(Cu.cloneInto(items, this._sandbox));
});
};

9
chrome/content/zotero/actors/TranslationParent.mjs
View file

@ -51,7 +51,14 @@ export const TranslationManager = new (class {
if (handlers) {
for (let handler of handlers) {
try {
returnValue = await handler(remoteTranslate, ...args);
// 'select' wants a callback for some reason
if (name === 'select') {
let items = args[0];
returnValue = await new Promise(resolve => handler(remoteTranslate, items, resolve));
}
else {
returnValue = await handler(remoteTranslate, ...args);
}
}
catch (e) {
Zotero.logError(e);

2
chrome/locale/en-US/scaffold/scaffold.dtd
View file

@ -71,7 +71,7 @@
<!ENTITY scaffold.testing.save "Save">
<!ENTITY scaffold.testing.delete "Delete">
<!ENTITY scaffold.testing.run "Run">
<!ENTITY scaffold.testing.update "Run and Update">
<!ENTITY scaffold.testing.applyUpdates "Apply Updates">
<!ENTITY scaffold.testing.rememberCookies "Remember cookies">
<!ENTITY scaffold.testing.create.web "Create Web Test">
<!ENTITY scaffold.testing.create.import "Create Import Test">

62
scss/scaffold.scss
View file

@ -183,6 +183,12 @@ browser,
flex: 1;
max-width: 100%;
gap: 8px;
listheader {
// Keep aligned with item columns when scrollbars are visible
scrollbar-gutter: stable;
overflow-y: hidden;
}
richlistbox {
min-width: 200px;
@ -191,15 +197,53 @@ browser,
richlistitem {
padding: 0;
align-items: center;
&:nth-child(odd):not([selected]) {
background: var(--material-stripe);
}
.status {
display: flex;
align-items: center;
gap: 0.5em;
&:not(.needs-update) {
.added, .removed {
opacity: 0.8;
}
}
&.needs-update {
cursor: pointer;
.text {
color: LinkText;
text-decoration: underline;
}
.added {
color: rgb(40 161 40 / 0.8);
}
.removed {
color: rgb(210 50 50 / 0.8);
}
}
}
}
}
&:focus richlistitem[selected] {
.status {
&.needs-update {
.text {
color: var(--color-accent-text);
}
#testing-buttons {
gap: 8px;
button,
checkbox {
margin: 0;
.added, .removed {
filter: brightness(1.5);
}
}
}
}
}
@ -298,11 +342,11 @@ scaffold-item-preview {
overflow-wrap: anywhere;
&.added {
background-color: rgb(0 255 0 / 0.1)
background-color: rgb(0 255 0 / 0.1);
}
&.removed {
background-color: rgb(255 0 0 / 0.1)
background-color: rgb(255 0 0 / 0.1);
}
}
}