diff --git a/chrome/content/zotero/collectionViewItemTree.jsx b/chrome/content/zotero/collectionViewItemTree.jsx index ceec992aa4..d8595b0494 100644 --- a/chrome/content/zotero/collectionViewItemTree.jsx +++ b/chrome/content/zotero/collectionViewItemTree.jsx @@ -185,7 +185,23 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider { * @return {Promise} - The scoreable items */ async _applyBestMatch(items) { - let query = this.collectionTreeRows.find(rowIsBestMatchSearch).searchText; + // With multiple selected rows carrying different best-match sources, + // the first in collections-list order supplies the query + let queryRow = this.collectionTreeRows.find(rowIsBestMatchSearch); + let query = queryRow.getBestMatchQuery(); + let source = queryRow.getBestMatchSource(); + // A top-K cutoff is reapplied to the merged candidates below only when + // the source is the transient Advanced Search, which applies uniformly + // to every selected row. A saved search's cutoff is part of that row's + // own membership and must not trim other selected rows' results. + let topK = queryRow.advancedSearch && source + ? source.getBestMatchQuery().topK + : false; + // A best-match quick search shows only the items it can rank. With any + // search source, membership is defined by the selected rows' own + // searches, so keep unscoreable items -- they sort after the ranked + // ones. (A uniform top-K set contains no unscoreable items anyway.) + let keepUnscored = !!source; // Map each item to the item whose embedding scores it let sourceIDByItem = new Map(); for (let item of items) { @@ -211,16 +227,29 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider { throw e; } // Scoring can fail while the model is still downloading or the - // index is being rebuilt -- show no results rather than an - // unranked scope + // index is being rebuilt if (e instanceof Zotero.Embeddings.IndexNotReadyError) { - Zotero.debug("Embeddings: index not ready for best-match search -- showing no results"); + Zotero.debug("Embeddings: index not ready for best-match search"); } else { Zotero.logError(e); } this._bestMatchRanks = new Map(); - return []; + // A rank-only search's membership doesn't depend on the index, so + // show its results unranked; anything else shows no results rather + // than an unranked scope + return keepUnscored ? items : []; + } + // Each selected row's search applies a top-K cutoff to its own scope, + // so trim the merged candidates to K again here, with the same + // deterministic order as search(), so a multi-row selection returns K + // members total rather than K per row + if (topK) { + scores = new Map( + [...scores.entries()] + .sort((a, b) => (b[1] - a[1]) || (a[0] - b[0])) + .slice(0, topK) + ); } let rankOfScore = new Map( [...new Set(scores.values())].sort((a, b) => b - a).map((score, i) => [score, i + 1]) @@ -231,6 +260,9 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider { for (let item of items) { let sourceID = sourceIDByItem.get(item); if (sourceID === undefined || !scores.has(sourceID)) { + if (keepUnscored) { + kept.push(item); + } continue; } kept.push(item); diff --git a/chrome/content/zotero/elements/advancedSearchPane.js b/chrome/content/zotero/elements/advancedSearchPane.js index 65b701cb3b..73211a168a 100644 --- a/chrome/content/zotero/elements/advancedSearchPane.js +++ b/chrome/content/zotero/elements/advancedSearchPane.js @@ -284,7 +284,7 @@ // position-independent and stay at the top level, and the rest, which may get // wrapped in a group below. The search always comes from updateSearch(), so // these are the only markers/flags that can appear at the top level. - const FLAGS = ['resultLevel', 'recursive', 'includeParentsAndChildren']; + const FLAGS = ['resultLevel', 'bestMatch', 'recursive', 'includeParentsAndChildren']; let joinMode = 'all'; let flags = []; let rest = []; diff --git a/chrome/content/zotero/elements/quickSearchTextbox.js b/chrome/content/zotero/elements/quickSearchTextbox.js index 7c8ae768ea..c51ed9f19d 100644 --- a/chrome/content/zotero/elements/quickSearchTextbox.js +++ b/chrome/content/zotero/elements/quickSearchTextbox.js @@ -248,12 +248,6 @@ .setAttribute('checked', 'true'); document.l10n.setAttributes(this.searchTextbox.inputField, "quicksearch-input", { placeholder: this._searchModes[mode] }); - // A best-match search can't be converted into Advanced Search - // conditions, so hide the button in best-match mode - if (this._advancedButton) { - this._advancedButton.hidden = mode === 'bestMatch'; - } - let advancedSearchDeck = document.getElementById('zotero-advanced-search-pane-deck'); if (advancedSearchDeck) { let state = advancedSearchDeck.state; diff --git a/chrome/content/zotero/elements/zoteroSearch.js b/chrome/content/zotero/elements/zoteroSearch.js index fedad7b73b..211567549e 100644 --- a/chrome/content/zotero/elements/zoteroSearch.js +++ b/chrome/content/zotero/elements/zoteroSearch.js @@ -151,6 +151,11 @@ stack[stack.length - 1].resultLevel = condition.operator; continue; + case 'bestMatch': + stack[stack.length - 1].bestMatch = condition.value; + stack[stack.length - 1].bestMatchTopK = parseInt(condition.operator) || false; + continue; + case 'groupStart': { let group = document.createXULElement('search-condition-group'); stack[stack.length - 1].conditionsContainer.appendChild(group); @@ -305,6 +310,16 @@ if (group.resultLevel && group.resultLevel != 'any') { flat.push({ condition: 'resultLevel', operator: group.resultLevel, value: null }); } + // The best-match query is a root-level modifier, offered only for + // top-level item results. The operator carries the optional top-K + // cutoff; 'contains' means rank-only. + if (isRoot && group.bestMatch && group.resultLevel == 'item') { + flat.push({ + condition: 'bestMatch', + operator: group.bestMatchTopK ? String(group.bestMatchTopK) : 'contains', + value: group.bestMatch + }); + } for (let child of group.conditionsContainer.children) { if (child.localName == 'zoterosearchcondition') { let data = child.getConditionData(); @@ -478,6 +493,12 @@ + @@ -491,6 +512,19 @@ this.conditionsContainer = this.querySelector('.conditions'); // The group's own warning element, stashed at init to avoid re-querying. this.levelWarning = this.querySelector('.level-warning'); + this.bestMatchRow = this.querySelector('.best-match-row'); + this.bestMatchInput = this.querySelector('.best-match-input'); + this.bestMatchTopKInput = this.querySelector('.best-match-topk-input'); + // The cutoff has no zero -- empty means "all" -- so route 0 by + // direction: stepping down from 1 goes back to empty, and stepping + // up from empty (which the browser floors at min) goes to 1 + this._lastTopKValue = this.bestMatchTopKInput.value; + this.bestMatchTopKInput.addEventListener('input', () => { + if (this.bestMatchTopKInput.value === '0') { + this.bestMatchTopKInput.value = this._lastTopKValue === '' ? '1' : ''; + } + this._lastTopKValue = this.bestMatchTopKInput.value; + }); // The result level is tracked here and reflected to whichever control is active: the root's // result-level menu ("Find ..."), or a nested group's binding menu ("... in the @@ -523,8 +557,10 @@ this.addEventListener('command', (event) => { if (this.resultLevelControl && this.resultLevelControl.contains(event.target)) { this._resultLevel = this.resultLevelControl.value || 'any'; + this.updateBestMatchRow(); } }); + this.updateBestMatchRow(); // At init the group has no nested groups yet, so these resolve to its own // caption buttons this.addConditionButton = this.querySelector('.add-condition'); @@ -576,6 +612,42 @@ return this._resultLevel; } + /** + * The root group's best-match query, from the "Sort results by + * best match for" field + */ + get bestMatch() { + return this.bestMatchInput.value.trim(); + } + + set bestMatch(val) { + this.bestMatchInput.value = val || ''; + this.updateBestMatchRow(); + } + + /** + * Optional top-K cutoff for the best-match query: with a value, only + * the K most similar results match; empty means rank-only + */ + get bestMatchTopK() { + let val = parseInt(this.bestMatchTopKInput.value); + return val > 0 ? val : false; + } + + set bestMatchTopK(val) { + this.bestMatchTopKInput.value = val || ''; + this._lastTopKValue = this.bestMatchTopKInput.value; + } + + // The best-match field is a root-level modifier, offered only for + // top-level item results and only when semantic search is enabled (or + // the search already carries a query, so it stays editable) + updateBestMatchRow() { + this.bestMatchRow.hidden = !this.isRoot + || this.resultLevel != 'item' + || !(Zotero.Embeddings.isEnabled() || this.bestMatchInput.value); + } + set resultLevel(val) { this._resultLevel = val || 'any'; // Reflect to the active control if it currently offers a matching option; the @@ -584,6 +656,7 @@ if (popup && [...popup.children].some(item => item.value == this._resultLevel)) { this.resultLevelControl.value = this._resultLevel; } + this.updateBestMatchRow(); } // Build the nested-group binding menu ("... in the same attachment"), shown when @@ -795,6 +868,8 @@ clear() { this.joinMode = 'all'; this.resultLevel = 'any'; + this.bestMatch = ''; + this.bestMatchTopK = false; while (this.conditionsContainer.firstChild) { this.conditionsContainer.removeChild(this.conditionsContainer.firstChild); } diff --git a/chrome/content/zotero/xpcom/collectionTreeRow.js b/chrome/content/zotero/xpcom/collectionTreeRow.js index 845c83514e..29d39090a3 100644 --- a/chrome/content/zotero/xpcom/collectionTreeRow.js +++ b/chrome/content/zotero/xpcom/collectionTreeRow.js @@ -495,7 +495,8 @@ Zotero.CollectionTreeRow.prototype.getSearchObject = async function (options = { if (this.searchText && !this.advancedSearch) { let mode = this.searchMode || Zotero.Prefs.get('search.quicksearch-mode'); // The best-match mode isn't a SQL condition -- the base search returns - // the full scope and getSearchResults() re-ranks it semantically. + // the full scope and the items view's row provider ranks it + // semantically. if (mode !== 'bestMatch') { s2.addCondition('quicksearch-' + mode, 'contains', this.searchText); } @@ -702,16 +703,62 @@ Zotero.CollectionTreeRow.prototype.isSearchMode = function () { } /** - * Whether an active quick search on this row is in the semantic similarity - * mode -- i.e. the results are a scored set that the items list orders by - * the Relevance column rather than a plain filter. The scoring itself is - * applied to the merged results by the items view's row provider. + * The search whose root-level bestMatch condition drives semantic ranking on + * this row: the transient Advanced Search when one is set (it replaces the + * row's own filtering, so it takes precedence even on a saved-search row), or + * a selected saved search whose definition carries the marker. False when + * neither does -- including for a best-match quick search, which ranks by the + * quick search text instead. + * + * @return {Zotero.Search|false} + */ +Zotero.CollectionTreeRow.prototype.getBestMatchSource = function () { + if (this.advancedSearch) { + return this.advancedSearch.getBestMatchQuery() ? this.advancedSearch : false; + } + // An active best-match quick search overrides a selected saved search's + // own marker, so the user's typed query wins + if (this.searchText + && (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode')) === 'bestMatch') { + return false; + } + if (this.isSearch() && this.ref.getBestMatchQuery()) { + return this.ref; + } + return false; +}; + +/** + * Whether an active search on this row ranks semantically -- a quick search + * in best-match mode, or an advanced or saved search with a root-level + * bestMatch condition. The results are a scored set that the items list + * orders by the Relevance column; the scoring itself is applied to the + * merged results by the items view's row provider. */ Zotero.CollectionTreeRow.prototype.isBestMatchSearch = function () { - return !!this.searchText && !this.advancedSearch + if (this.advancedSearch) { + return !!this.advancedSearch.getBestMatchQuery(); + } + if (this.getBestMatchSource()) { + return true; + } + return !!this.searchText && (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode')) === 'bestMatch'; }; +/** + * Query text an active best-match search ranks by (see isBestMatchSearch()) + * + * @return {String|false} + */ +Zotero.CollectionTreeRow.prototype.getBestMatchQuery = function () { + if (!this.isBestMatchSearch()) { + return false; + } + let source = this.getBestMatchSource(); + return source ? source.getBestMatchQuery().query : this.searchText; +}; + Zotero.CollectionTreeRow.prototype.isSortable = function () { return !this.isFeedsOrFeed() && !this.isRecentlyRead(); } diff --git a/chrome/content/zotero/xpcom/data/search.js b/chrome/content/zotero/xpcom/data/search.js index 551f3389c3..1334cecda3 100644 --- a/chrome/content/zotero/xpcom/data/search.js +++ b/chrome/content/zotero/xpcom/data/search.js @@ -555,7 +555,12 @@ Zotero.Search.prototype.getConditions = function (){ Zotero.Search.prototype.hasPostSearchFilter = function () { this._requireData('conditions'); for (let i of Object.values(this._conditions)) { - if (i.condition == 'fulltextContent'){ + // Applied in search() after the SQL runs, so uses of this search as a + // scope have to route through search() to include them. A rank-only + // bestMatch condition (no cutoff) doesn't affect membership, so it + // doesn't count. + if (i.condition == 'fulltextContent' + || (i.condition == 'bestMatch' && this.getBestMatchQuery()?.topK)) { return true; } } @@ -823,11 +828,37 @@ Zotero.Search.prototype.search = async function (asTempTable) { //Zotero.debug('Final result set'); //Zotero.debug(ids); - + + // A root-level 'bestMatch' condition with a top-K cutoff makes membership + // semantic: only the K results most similar to the query match, so the + // saved search returns the same set when used as a source (scopes, counts, + // the API). Without a cutoff, best match is only a ranking in the items + // list and membership is untouched. If the index isn't usable (no model, + // or mid-switch), a cutoff search matches nothing rather than an arbitrary + // set. + let bestMatch = this.getBestMatchQuery(); + if (ids && ids.length && bestMatch && bestMatch.topK) { + try { + let scores = await Zotero.Embeddings.scoreItemIDs(bestMatch.query, ids); + ids = [...scores.entries()] + // Deterministic order: by score, then by itemID for equal scores + .sort((a, b) => (b[1] - a[1]) || (a[0] - b[0])) + .slice(0, bestMatch.topK) + .map(([itemID]) => itemID); + } + catch (e) { + if (!(e instanceof Zotero.Embeddings.IndexNotReadyError)) { + throw e; + } + Zotero.debug("Embeddings index not ready -- best-match cutoff search matches nothing"); + ids = []; + } + } + if (!ids || !ids.length) { return []; } - + if (asTempTable) { return Zotero.Search.idsToTempTable(ids); } @@ -835,6 +866,32 @@ Zotero.Search.prototype.search = async function (asTempTable) { }; +/** + * The root-level 'bestMatch' condition, or false if none + * + * @return {Object|false} - { query, topK }, with topK false when the + * condition is rank-only (operator 'contains') rather than a cutoff + */ +Zotero.Search.prototype.getBestMatchQuery = function () { + let depth = 0; + for (let condition of Object.values(this._conditions)) { + if (condition.condition == 'groupStart') { + depth++; + } + else if (condition.condition == 'groupEnd') { + depth--; + } + else if (depth == 0 && condition.condition == 'bestMatch' && condition.value) { + return { + query: condition.value, + topK: parseInt(condition.operator) || false + }; + } + } + return false; +}; + + /** * Populate the object's data from an API JSON data object * @@ -1199,6 +1256,11 @@ Zotero.Search.prototype._buildQuery = async function () { lastCondition = null; conditions.push({ name: 'resultLevel', operator: condition.operator }); continue; + // Applied as a filter at the end of search() and as a ranking by the + // items list, not as part of the condition tree + case 'bestMatch': + lastCondition = null; + continue; case 'groupStart': lastCondition = null; loopDepth++; diff --git a/chrome/content/zotero/xpcom/data/searchConditions.js b/chrome/content/zotero/xpcom/data/searchConditions.js index 6f2a0d9e8f..33724479c9 100644 --- a/chrome/content/zotero/xpcom/data/searchConditions.js +++ b/chrome/content/zotero/xpcom/data/searchConditions.js @@ -266,6 +266,17 @@ Zotero.SearchConditions = new function () { } }, + // Root-level modifier rather than a regular condition: restricts the + // results to items with a stored embedding, and the items list ranks + // them by semantic similarity to the value (see + // CollectionViewItemTreeRowProvider._applyBestMatch()) + { + name: 'bestMatch', + operators: { + contains: true + } + }, + // Shortcuts for adding collections and searches by id { name: 'collectionID', @@ -894,21 +905,27 @@ Zotero.SearchConditions = new function () { */ function hasOperator(condition, operator){ var [condition, mode] = this.parseCondition(condition); - + if (!_conditions) { throw new Zotero.Exception.UnloadedDataException("Search conditions not yet loaded"); } - + if (!_conditions[condition]){ let e = new Error("Invalid condition '" + condition + "' in hasOperator()"); e.name = "ZoteroInvalidDataError"; throw e; } - + if (!operator && typeof _conditions[condition]['operators'] == 'undefined'){ return true; } - + + // The bestMatch marker's operator can carry a top-K result cutoff (any + // positive integer) in place of 'contains' (rank-only, no cutoff) + if (condition == 'bestMatch' && /^[1-9][0-9]*$/.test(operator)) { + return true; + } + return !!_conditions[condition]['operators'][operator]; } diff --git a/chrome/content/zotero/xpcom/embeddings.js b/chrome/content/zotero/xpcom/embeddings.js index 7bde528cb9..4cd7760300 100644 --- a/chrome/content/zotero/xpcom/embeddings.js +++ b/chrome/content/zotero/xpcom/embeddings.js @@ -681,6 +681,10 @@ Zotero.Embeddings = new function () { return _toVectors(data, dims); }; + // The last embedded query, reused across the scoring passes a single + // search triggers (per-row membership cutoffs plus the merged ranking) + let _queryCache = null; + /** * Embed a search query string, applying the active model's query prefix. * @@ -688,7 +692,22 @@ Zotero.Embeddings = new function () { * @return {Promise} */ this.embedQuery = function (text) { - return this.embed(_getModel().queryPrefix + text); + let modelVersion = this.getModelVersion(); + if (_queryCache && _queryCache.modelVersion === modelVersion + && _queryCache.text === text) { + return _queryCache.promise; + } + // Cache the in-flight promise, so the concurrent per-row scoring + // passes of a multi-collection search share one embed + let promise = this.embed(_getModel().queryPrefix + text); + _queryCache = { modelVersion, text, promise }; + // Don't cache a failed embed + promise.catch(() => { + if (_queryCache && _queryCache.promise === promise) { + _queryCache = null; + } + }); + return promise; }; /** diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js index 32feb429af..96cc5718b6 100644 --- a/chrome/content/zotero/zoteroPane.js +++ b/chrome/content/zotero/zoteroPane.js @@ -2027,9 +2027,14 @@ var ZoteroPane = new function () { * @param {String} [mode='fields'] - The quick search mode to reproduce */ this.openAdvancedSearchFromQuickSearch = async function (searchText, mode = 'fields') { - // Split into words (keeping quoted phrases intact), as the quick search does - let parts = Zotero.SearchConditions.parseSearchString(searchText); - if (!parts.length) { + // A best-match search keeps what was typed as one semantic string; other + // modes split it into words (keeping quoted phrases intact), as the + // quick search does + let parts = mode === 'bestMatch' + ? [] + : Zotero.SearchConditions.parseSearchString(searchText); + let hasText = mode === 'bestMatch' ? !!searchText.trim() : !!parts.length; + if (!hasText) { await this.toggleAdvancedSearchState('open'); return; } @@ -2044,10 +2049,14 @@ var ZoteroPane = new function () { // Reproduce the quick search mode as editable conditions, one per word joined // with "all": Title/Creator/Year and All Fields & Tags each map to a single // condition, Everything to an "any" group of Any Field plus full-text. - // Title/Creator/Year matches only top-level items, so set the result level to item. - if (mode === 'titleCreatorYear') { + // Title/Creator/Year and Best Match match only top-level items, so set the + // result level to item. + if (mode === 'titleCreatorYear' || mode === 'bestMatch') { search.addCondition('resultLevel', 'item'); } + if (mode === 'bestMatch') { + search.addCondition('bestMatch', 'contains', searchText.trim()); + } for (let part of parts) { if (mode === 'everything') { search.addCondition('groupStart', 'true', ''); diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index b1671d3d8c..fd83184fc3 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -960,6 +960,16 @@ advanced-search-binding-hint-note = .value = These conditions can match separate notes. advanced-search-binding-hint-annotation = .value = These conditions can match separate annotations. +advanced-search-best-match-prefix = + .value = Sort results by best match for: +advanced-search-best-match-input = + .aria-label = Sort results by best match for + .placeholder = Enter a topic or phrase +advanced-search-best-match-topk-prefix = + .value = Keep top: +advanced-search-best-match-topk-input = + .aria-label = Number of results to keep + .placeholder = all advanced-search-level-warning-mixed = These conditions cannot all match the same item, so this search will never return results. Try matching “{ $matchAny }” of them, or set the result type to “{ $topLevelItems }”. advanced-search-level-warning-unreachable = This search has a condition that cannot apply to the chosen result type. Set the result type to “{ $topLevelItems }” or remove the incompatible condition. advanced-search-group-warning-unreachable = diff --git a/scss/elements/_zoteroSearch.scss b/scss/elements/_zoteroSearch.scss index 7eca317668..86bfa2b8e7 100644 --- a/scss/elements/_zoteroSearch.scss +++ b/scss/elements/_zoteroSearch.scss @@ -165,6 +165,26 @@ zoterosearch { margin-block-end: 6px; } + .best-match-row { + // The semantic ranking modifier: a sentence-styled strip below the root group's + // conditions, spaced by the group's own gap + gap: 6px; + + // Let the row gap govern the label-to-input spacing instead of adding + // the XUL label's default inline-end margin to it + label { + margin-inline-end: 0; + } + + .best-match-input { + flex: 1; + } + + .best-match-topk-input { + width: 60px; + } + } + #search-binding-hint { // A distinct suggestion callout under the conditions (not plain body text): a light // blue, rounded block with slightly smaller text, one "[statement] [button]" line per diff --git a/test/tests/searchTest.js b/test/tests/searchTest.js index 4d55cba267..ac46cf5fd5 100644 --- a/test/tests/searchTest.js +++ b/test/tests/searchTest.js @@ -220,6 +220,39 @@ describe("Zotero.Search", function () { }); + describe("bestMatch condition", function () { + it("shouldn't affect membership without a cutoff and should return the top K with one", async function () { + let itemA = await createDataObject('item', { title: "bestmatchcondtest A" }); + let itemB = await createDataObject('item', { title: "bestmatchcondtest B" }); + + // Rank-only (no cutoff): membership is unchanged + let s = new Zotero.Search(); + s.libraryID = itemA.libraryID; + s.addCondition('resultLevel', 'item'); + s.addCondition('title', 'contains', 'bestmatchcondtest'); + s.addCondition('bestMatch', 'contains', 'some query'); + assert.deepEqual(s.getBestMatchQuery(), { query: 'some query', topK: false }); + assert.sameMembers(await s.search(), [itemA.id, itemB.id]); + + // With a cutoff, membership is the K most similar + let stub = sinon.stub(Zotero.Embeddings, 'scoreItemIDs').callsFake( + async (query, itemIDs) => new Map(itemIDs.map(id => [id, id == itemB.id ? 0.9 : 0.5])) + ); + try { + let s2 = new Zotero.Search(); + s2.libraryID = itemA.libraryID; + s2.addCondition('resultLevel', 'item'); + s2.addCondition('title', 'contains', 'bestmatchcondtest'); + s2.addCondition('bestMatch', '1', 'some query'); + assert.deepEqual(s2.getBestMatchQuery(), { query: 'some query', topK: 1 }); + assert.sameMembers(await s2.search(), [itemB.id]); + } + finally { + stub.restore(); + } + }); + }); + describe("#search()", function () { var userLibraryID; var fooItem;