scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-09-12 23:01:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add semantic ranking to Advanced Search

Browse source 
A root-level 'bestMatch' condition -- serialized as a marker like
joinMode and resultLevel -- ranks results with the Relevance column via
a "Sort results by best match for" field below the root group's
conditions, composing semantic ranking with Boolean filtering. A
best-match quick search converts to it via the Advanced Search button,
and a selected saved search's own marker activates ranking too,
overridden by an active best-match quick search.

Without a cutoff the condition is rank-only and membership is untouched
-- including while the index is unavailable -- so a saved search acts
identically as a source, and unscoreable items just sort last. With the
optional "keeping top" cutoff, carried in the marker's operator,
membership becomes the K most similar results, applied in search() so
scopes, counts, and the API see the same set. A transient search's
cutoff, which applies uniformly to every selected row, is reapplied
over the merged results so a multi-collection selection returns K
members total; a saved search's cutoff is part of its own membership
and never trims other selected rows. The query embedding is cached
in-flight, so the per-row membership passes and the merged ranking
share one worker embed.
This commit is contained in:
Dan Stillman 2026-07-20 15:58:13 -04:00
parent 772189a71e
commit 486bac342f
12 changed files with 349 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
chrome/content/zotero/collectionViewItemTree.jsx
View file

@ -185,7 +185,23 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
* @return {Promise<Zotero.Item[]>} - The scoreable items
*/
async _applyBestMatch(items) {
let query = this.collectionTreeRows.find(rowIsBestMatchSearch).searchText;
// With multiple selected rows carrying different best-match sources,
// the first in collections-list order supplies the query
let queryRow = this.collectionTreeRows.find(rowIsBestMatchSearch);
let query = queryRow.getBestMatchQuery();
let source = queryRow.getBestMatchSource();
// A top-K cutoff is reapplied to the merged candidates below only when
// the source is the transient Advanced Search, which applies uniformly
// to every selected row. A saved search's cutoff is part of that row's
// own membership and must not trim other selected rows' results.
let topK = queryRow.advancedSearch && source
? source.getBestMatchQuery().topK
: false;
// A best-match quick search shows only the items it can rank. With any
// search source, membership is defined by the selected rows' own
// searches, so keep unscoreable items -- they sort after the ranked
// ones. (A uniform top-K set contains no unscoreable items anyway.)
let keepUnscored = !!source;
// Map each item to the item whose embedding scores it
let sourceIDByItem = new Map();
for (let item of items) {
@ -211,16 +227,29 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
throw e;
}
// Scoring can fail while the model is still downloading or the
// index is being rebuilt -- show no results rather than an
// unranked scope
// index is being rebuilt
if (e instanceof Zotero.Embeddings.IndexNotReadyError) {
Zotero.debug("Embeddings: index not ready for best-match search -- showing no results");
Zotero.debug("Embeddings: index not ready for best-match search");
}
else {
Zotero.logError(e);
}
this._bestMatchRanks = new Map();
return [];
// A rank-only search's membership doesn't depend on the index, so
// show its results unranked; anything else shows no results rather
// than an unranked scope
return keepUnscored ? items : [];
}
// Each selected row's search applies a top-K cutoff to its own scope,
// so trim the merged candidates to K again here, with the same
// deterministic order as search(), so a multi-row selection returns K
// members total rather than K per row
if (topK) {
scores = new Map(
[...scores.entries()]
.sort((a, b) => (b[1] - a[1]) || (a[0] - b[0]))
.slice(0, topK)
);
}
let rankOfScore = new Map(
[...new Set(scores.values())].sort((a, b) => b - a).map((score, i) => [score, i + 1])
@ -231,6 +260,9 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
for (let item of items) {
let sourceID = sourceIDByItem.get(item);
if (sourceID === undefined || !scores.has(sourceID)) {
if (keepUnscored) {
kept.push(item);
}
continue;
}
kept.push(item);

2
chrome/content/zotero/elements/advancedSearchPane.js
View file

@ -284,7 +284,7 @@
// position-independent and stay at the top level, and the rest, which may get
// wrapped in a group below. The search always comes from updateSearch(), so
// these are the only markers/flags that can appear at the top level.
const FLAGS = ['resultLevel', 'recursive', 'includeParentsAndChildren'];
const FLAGS = ['resultLevel', 'bestMatch', 'recursive', 'includeParentsAndChildren'];
let joinMode = 'all';
let flags = [];
let rest = [];

6
chrome/content/zotero/elements/quickSearchTextbox.js
View file

@ -248,12 +248,6 @@
.setAttribute('checked', 'true');
document.l10n.setAttributes(this.searchTextbox.inputField, "quicksearch-input", { placeholder: this._searchModes[mode] });
// A best-match search can't be converted into Advanced Search
// conditions, so hide the button in best-match mode
if (this._advancedButton) {
this._advancedButton.hidden = mode === 'bestMatch';
}
let advancedSearchDeck = document.getElementById('zotero-advanced-search-pane-deck');
if (advancedSearchDeck) {
let state = advancedSearchDeck.state;

75
chrome/content/zotero/elements/zoteroSearch.js
View file

@ -151,6 +151,11 @@
stack[stack.length - 1].resultLevel = condition.operator;
continue;
case 'bestMatch':
stack[stack.length - 1].bestMatch = condition.value;
stack[stack.length - 1].bestMatchTopK = parseInt(condition.operator) || false;
continue;
case 'groupStart': {
let group = document.createXULElement('search-condition-group');
stack[stack.length - 1].conditionsContainer.appendChild(group);
@ -305,6 +310,16 @@
if (group.resultLevel && group.resultLevel != 'any') {
flat.push({ condition: 'resultLevel', operator: group.resultLevel, value: null });
}
// The best-match query is a root-level modifier, offered only for
// top-level item results. The operator carries the optional top-K
// cutoff; 'contains' means rank-only.
if (isRoot && group.bestMatch && group.resultLevel == 'item') {
flat.push({
condition: 'bestMatch',
operator: group.bestMatchTopK ? String(group.bestMatchTopK) : 'contains',
value: group.bestMatch
});
}
for (let child of group.conditionsContainer.children) {
if (child.localName == 'zoterosearchcondition') {
let data = child.getConditionData();
@ -478,6 +493,12 @@
</hbox>
</caption>
<vbox class="conditions"/>
<hbox class="best-match-row" align="center" hidden="true">
<label class="best-match-prefix" data-l10n-id="advanced-search-best-match-prefix"/>
<html:input class="best-match-input" type="text" data-l10n-id="advanced-search-best-match-input"/>
<label class="best-match-topk-prefix" data-l10n-id="advanced-search-best-match-topk-prefix"/>
<html:input class="best-match-topk-input" type="number" min="0" data-l10n-id="advanced-search-best-match-topk-input"/>
</hbox>
<hbox class="level-warning" hidden="true">
<description/>
</hbox>
@ -491,6 +512,19 @@
this.conditionsContainer = this.querySelector('.conditions');
// The group's own warning element, stashed at init to avoid re-querying.
this.levelWarning = this.querySelector('.level-warning');
this.bestMatchRow = this.querySelector('.best-match-row');
this.bestMatchInput = this.querySelector('.best-match-input');
this.bestMatchTopKInput = this.querySelector('.best-match-topk-input');
// The cutoff has no zero -- empty means "all" -- so route 0 by
// direction: stepping down from 1 goes back to empty, and stepping
// up from empty (which the browser floors at min) goes to 1
this._lastTopKValue = this.bestMatchTopKInput.value;
this.bestMatchTopKInput.addEventListener('input', () => {
if (this.bestMatchTopKInput.value === '0') {
this.bestMatchTopKInput.value = this._lastTopKValue === '' ? '1' : '';
}
this._lastTopKValue = this.bestMatchTopKInput.value;
});
// The result level is tracked here and reflected to whichever control is active: the root's
// result-level menu ("Find ..."), or a nested group's binding menu ("... in the
@ -523,8 +557,10 @@
this.addEventListener('command', (event) => {
if (this.resultLevelControl && this.resultLevelControl.contains(event.target)) {
this._resultLevel = this.resultLevelControl.value || 'any';
this.updateBestMatchRow();
}
});
this.updateBestMatchRow();
// At init the group has no nested groups yet, so these resolve to its own
// caption buttons
this.addConditionButton = this.querySelector('.add-condition');
@ -576,6 +612,42 @@
return this._resultLevel;
}
/**
* The root group's best-match query, from the "Sort results by
* best match for" field
*/
get bestMatch() {
return this.bestMatchInput.value.trim();
}
set bestMatch(val) {
this.bestMatchInput.value = val || '';
this.updateBestMatchRow();
}
/**
* Optional top-K cutoff for the best-match query: with a value, only
* the K most similar results match; empty means rank-only
*/
get bestMatchTopK() {
let val = parseInt(this.bestMatchTopKInput.value);
return val > 0 ? val : false;
}
set bestMatchTopK(val) {
this.bestMatchTopKInput.value = val || '';
this._lastTopKValue = this.bestMatchTopKInput.value;
}
// The best-match field is a root-level modifier, offered only for
// top-level item results and only when semantic search is enabled (or
// the search already carries a query, so it stays editable)
updateBestMatchRow() {
this.bestMatchRow.hidden = !this.isRoot
|| this.resultLevel != 'item'
|| !(Zotero.Embeddings.isEnabled() || this.bestMatchInput.value);
}
set resultLevel(val) {
this._resultLevel = val || 'any';
// Reflect to the active control if it currently offers a matching option; the
@ -584,6 +656,7 @@
if (popup && [...popup.children].some(item => item.value == this._resultLevel)) {
this.resultLevelControl.value = this._resultLevel;
}
this.updateBestMatchRow();
}
// Build the nested-group binding menu ("... in the same attachment"), shown when
@ -795,6 +868,8 @@
clear() {
this.joinMode = 'all';
this.resultLevel = 'any';
this.bestMatch = '';
this.bestMatchTopK = false;
while (this.conditionsContainer.firstChild) {
this.conditionsContainer.removeChild(this.conditionsContainer.firstChild);
}

59
chrome/content/zotero/xpcom/collectionTreeRow.js
View file

@ -495,7 +495,8 @@ Zotero.CollectionTreeRow.prototype.getSearchObject = async function (options = {
if (this.searchText && !this.advancedSearch) {
let mode = this.searchMode || Zotero.Prefs.get('search.quicksearch-mode');
// The best-match mode isn't a SQL condition -- the base search returns
// the full scope and getSearchResults() re-ranks it semantically.
// the full scope and the items view's row provider ranks it
// semantically.
if (mode !== 'bestMatch') {
s2.addCondition('quicksearch-' + mode, 'contains', this.searchText);
}
@ -702,16 +703,62 @@ Zotero.CollectionTreeRow.prototype.isSearchMode = function () {
}
/**
* Whether an active quick search on this row is in the semantic similarity
* mode -- i.e. the results are a scored set that the items list orders by
* the Relevance column rather than a plain filter. The scoring itself is
* applied to the merged results by the items view's row provider.
* The search whose root-level bestMatch condition drives semantic ranking on
* this row: the transient Advanced Search when one is set (it replaces the
* row's own filtering, so it takes precedence even on a saved-search row), or
* a selected saved search whose definition carries the marker. False when
* neither does -- including for a best-match quick search, which ranks by the
* quick search text instead.
*
* @return {Zotero.Search|false}
*/
Zotero.CollectionTreeRow.prototype.getBestMatchSource = function () {
if (this.advancedSearch) {
return this.advancedSearch.getBestMatchQuery() ? this.advancedSearch : false;
}
// An active best-match quick search overrides a selected saved search's
// own marker, so the user's typed query wins
if (this.searchText
&& (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode')) === 'bestMatch') {
return false;
}
if (this.isSearch() && this.ref.getBestMatchQuery()) {
return this.ref;
}
return false;
};
/**
* Whether an active search on this row ranks semantically -- a quick search
* in best-match mode, or an advanced or saved search with a root-level
* bestMatch condition. The results are a scored set that the items list
* orders by the Relevance column; the scoring itself is applied to the
* merged results by the items view's row provider.
*/
Zotero.CollectionTreeRow.prototype.isBestMatchSearch = function () {
return !!this.searchText && !this.advancedSearch
if (this.advancedSearch) {
return !!this.advancedSearch.getBestMatchQuery();
}
if (this.getBestMatchSource()) {
return true;
}
return !!this.searchText
&& (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode')) === 'bestMatch';
};
/**
* Query text an active best-match search ranks by (see isBestMatchSearch())
*
* @return {String|false}
*/
Zotero.CollectionTreeRow.prototype.getBestMatchQuery = function () {
if (!this.isBestMatchSearch()) {
return false;
}
let source = this.getBestMatchSource();
return source ? source.getBestMatchQuery().query : this.searchText;
};
Zotero.CollectionTreeRow.prototype.isSortable = function () {
return !this.isFeedsOrFeed() && !this.isRecentlyRead();
}

68
chrome/content/zotero/xpcom/data/search.js
View file

@ -555,7 +555,12 @@ Zotero.Search.prototype.getConditions = function (){
Zotero.Search.prototype.hasPostSearchFilter = function () {
this._requireData('conditions');
for (let i of Object.values(this._conditions)) {
if (i.condition == 'fulltextContent'){
// Applied in search() after the SQL runs, so uses of this search as a
// scope have to route through search() to include them. A rank-only
// bestMatch condition (no cutoff) doesn't affect membership, so it
// doesn't count.
if (i.condition == 'fulltextContent'
|| (i.condition == 'bestMatch' && this.getBestMatchQuery()?.topK)) {
return true;
}
}
@ -823,11 +828,37 @@ Zotero.Search.prototype.search = async function (asTempTable) {
//Zotero.debug('Final result set');
//Zotero.debug(ids);
// A root-level 'bestMatch' condition with a top-K cutoff makes membership
// semantic: only the K results most similar to the query match, so the
// saved search returns the same set when used as a source (scopes, counts,
// the API). Without a cutoff, best match is only a ranking in the items
// list and membership is untouched. If the index isn't usable (no model,
// or mid-switch), a cutoff search matches nothing rather than an arbitrary
// set.
let bestMatch = this.getBestMatchQuery();
if (ids && ids.length && bestMatch && bestMatch.topK) {
try {
let scores = await Zotero.Embeddings.scoreItemIDs(bestMatch.query, ids);
ids = [...scores.entries()]
// Deterministic order: by score, then by itemID for equal scores
.sort((a, b) => (b[1] - a[1]) || (a[0] - b[0]))
.slice(0, bestMatch.topK)
.map(([itemID]) => itemID);
}
catch (e) {
if (!(e instanceof Zotero.Embeddings.IndexNotReadyError)) {
throw e;
}
Zotero.debug("Embeddings index not ready -- best-match cutoff search matches nothing");
ids = [];
}
}
if (!ids || !ids.length) {
return [];
}
if (asTempTable) {
return Zotero.Search.idsToTempTable(ids);
}
@ -835,6 +866,32 @@ Zotero.Search.prototype.search = async function (asTempTable) {
};
/**
* The root-level 'bestMatch' condition, or false if none
*
* @return {Object|false} - { query, topK }, with topK false when the
* condition is rank-only (operator 'contains') rather than a cutoff
*/
Zotero.Search.prototype.getBestMatchQuery = function () {
let depth = 0;
for (let condition of Object.values(this._conditions)) {
if (condition.condition == 'groupStart') {
depth++;
}
else if (condition.condition == 'groupEnd') {
depth--;
}
else if (depth == 0 && condition.condition == 'bestMatch' && condition.value) {
return {
query: condition.value,
topK: parseInt(condition.operator) || false
};
}
}
return false;
};
/**
* Populate the object's data from an API JSON data object
*
@ -1199,6 +1256,11 @@ Zotero.Search.prototype._buildQuery = async function () {
lastCondition = null;
conditions.push({ name: 'resultLevel', operator: condition.operator });
continue;
// Applied as a filter at the end of search() and as a ranking by the
// items list, not as part of the condition tree
case 'bestMatch':
lastCondition = null;
continue;
case 'groupStart':
lastCondition = null;
loopDepth++;

25
chrome/content/zotero/xpcom/data/searchConditions.js
View file

@ -266,6 +266,17 @@ Zotero.SearchConditions = new function () {
}
},
// Root-level modifier rather than a regular condition: restricts the
// results to items with a stored embedding, and the items list ranks
// them by semantic similarity to the value (see
// CollectionViewItemTreeRowProvider._applyBestMatch())
{
name: 'bestMatch',
operators: {
contains: true
}
},
// Shortcuts for adding collections and searches by id
{
name: 'collectionID',
@ -894,21 +905,27 @@ Zotero.SearchConditions = new function () {
*/
function hasOperator(condition, operator){
var [condition, mode] = this.parseCondition(condition);
if (!_conditions) {
throw new Zotero.Exception.UnloadedDataException("Search conditions not yet loaded");
}
if (!_conditions[condition]){
let e = new Error("Invalid condition '" + condition + "' in hasOperator()");
e.name = "ZoteroInvalidDataError";
throw e;
}
if (!operator && typeof _conditions[condition]['operators'] == 'undefined'){
return true;
}
// The bestMatch marker's operator can carry a top-K result cutoff (any
// positive integer) in place of 'contains' (rank-only, no cutoff)
if (condition == 'bestMatch' && /^[1-9][0-9]*$/.test(operator)) {
return true;
}
return !!_conditions[condition]['operators'][operator];
}

21
chrome/content/zotero/xpcom/embeddings.js
View file

@ -681,6 +681,10 @@ Zotero.Embeddings = new function () {
return _toVectors(data, dims);
};
// The last embedded query, reused across the scoring passes a single
// search triggers (per-row membership cutoffs plus the merged ranking)
let _queryCache = null;
/**
* Embed a search query string, applying the active model's query prefix.
*
@ -688,7 +692,22 @@ Zotero.Embeddings = new function () {
* @return {Promise<Float32Array>}
*/
this.embedQuery = function (text) {
return this.embed(_getModel().queryPrefix + text);
let modelVersion = this.getModelVersion();
if (_queryCache && _queryCache.modelVersion === modelVersion
&& _queryCache.text === text) {
return _queryCache.promise;
}
// Cache the in-flight promise, so the concurrent per-row scoring
// passes of a multi-collection search share one embed
let promise = this.embed(_getModel().queryPrefix + text);
_queryCache = { modelVersion, text, promise };
// Don't cache a failed embed
promise.catch(() => {
if (_queryCache && _queryCache.promise === promise) {
_queryCache = null;
}
});
return promise;
};
/**

19
chrome/content/zotero/zoteroPane.js
View file

@ -2027,9 +2027,14 @@ var ZoteroPane = new function () {
* @param {String} [mode='fields'] - The quick search mode to reproduce
*/
this.openAdvancedSearchFromQuickSearch = async function (searchText, mode = 'fields') {
// Split into words (keeping quoted phrases intact), as the quick search does
let parts = Zotero.SearchConditions.parseSearchString(searchText);
if (!parts.length) {
// A best-match search keeps what was typed as one semantic string; other
// modes split it into words (keeping quoted phrases intact), as the
// quick search does
let parts = mode === 'bestMatch'
? []
: Zotero.SearchConditions.parseSearchString(searchText);
let hasText = mode === 'bestMatch' ? !!searchText.trim() : !!parts.length;
if (!hasText) {
await this.toggleAdvancedSearchState('open');
return;
}
@ -2044,10 +2049,14 @@ var ZoteroPane = new function () {
// Reproduce the quick search mode as editable conditions, one per word joined
// with "all": Title/Creator/Year and All Fields & Tags each map to a single
// condition, Everything to an "any" group of Any Field plus full-text.
// Title/Creator/Year matches only top-level items, so set the result level to item.
if (mode === 'titleCreatorYear') {
// Title/Creator/Year and Best Match match only top-level items, so set the
// result level to item.
if (mode === 'titleCreatorYear' || mode === 'bestMatch') {
search.addCondition('resultLevel', 'item');
}
if (mode === 'bestMatch') {
search.addCondition('bestMatch', 'contains', searchText.trim());
}
for (let part of parts) {
if (mode === 'everything') {
search.addCondition('groupStart', 'true', '');

10
chrome/locale/en-US/zotero/zotero.ftl
View file

@ -960,6 +960,16 @@ advanced-search-binding-hint-note =
.value = These conditions can match separate notes.
advanced-search-binding-hint-annotation =
.value = These conditions can match separate annotations.
advanced-search-best-match-prefix =
.value = Sort results by best match for:
advanced-search-best-match-input =
.aria-label = Sort results by best match for
.placeholder = Enter a topic or phrase
advanced-search-best-match-topk-prefix =
.value = Keep top:
advanced-search-best-match-topk-input =
.aria-label = Number of results to keep
.placeholder = all
advanced-search-level-warning-mixed = These conditions cannot all match the same item, so this search will never return results. Try matching “{ $matchAny }” of them, or set the result type to “{ $topLevelItems }”.
advanced-search-level-warning-unreachable = This search has a condition that cannot apply to the chosen result type. Set the result type to “{ $topLevelItems }” or remove the incompatible condition.
advanced-search-group-warning-unreachable =

20
scss/elements/_zoteroSearch.scss
View file

@ -165,6 +165,26 @@ zoterosearch {
margin-block-end: 6px;
}
.best-match-row {
// The semantic ranking modifier: a sentence-styled strip below the root group's
// conditions, spaced by the group's own gap
gap: 6px;
// Let the row gap govern the label-to-input spacing instead of adding
// the XUL label's default inline-end margin to it
label {
margin-inline-end: 0;
}
.best-match-input {
flex: 1;
}
.best-match-topk-input {
width: 60px;
}
}
#search-binding-hint {
// A distinct suggestion callout under the conditions (not plain body text): a light
// blue, rounded block with slightly smaller text, one "[statement] [button]" line per

33
test/tests/searchTest.js
View file

@ -220,6 +220,39 @@ describe("Zotero.Search", function () {
});
describe("bestMatch condition", function () {
it("shouldn't affect membership without a cutoff and should return the top K with one", async function () {
let itemA = await createDataObject('item', { title: "bestmatchcondtest A" });
let itemB = await createDataObject('item', { title: "bestmatchcondtest B" });
// Rank-only (no cutoff): membership is unchanged
let s = new Zotero.Search();
s.libraryID = itemA.libraryID;
s.addCondition('resultLevel', 'item');
s.addCondition('title', 'contains', 'bestmatchcondtest');
s.addCondition('bestMatch', 'contains', 'some query');
assert.deepEqual(s.getBestMatchQuery(), { query: 'some query', topK: false });
assert.sameMembers(await s.search(), [itemA.id, itemB.id]);
// With a cutoff, membership is the K most similar
let stub = sinon.stub(Zotero.Embeddings, 'scoreItemIDs').callsFake(
async (query, itemIDs) => new Map(itemIDs.map(id => [id, id == itemB.id ? 0.9 : 0.5]))
);
try {
let s2 = new Zotero.Search();
s2.libraryID = itemA.libraryID;
s2.addCondition('resultLevel', 'item');
s2.addCondition('title', 'contains', 'bestmatchcondtest');
s2.addCondition('bestMatch', '1', 'some query');
assert.deepEqual(s2.getBestMatchQuery(), { query: 'some query', topK: 1 });
assert.sameMembers(await s2.search(), [itemB.id]);
}
finally {
stub.restore();
}
});
});
describe("#search()", function () {
var userLibraryID;
var fooItem;