scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-10-08 03:08:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

handle all HighWire journals (Oxford, Science, etc.) with one translator

Browse source
This commit is contained in:
Simon Kornblith 2006-12-20 11:45:00 +00:00
parent a33fbd0834
commit 060512a379
1 changed files with 35 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
scrapers.sql
View file

@ -1,4 +1,4 @@
-- 152
-- 153
-- ***** BEGIN LICENSE BLOCK *****
--
@ -22,7 +22,7 @@
-- Set the following timestamp to the most recent scraper update date
REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-12-20 06:05:01'));
REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-12-20 06:40:52'));
REPLACE INTO translators VALUES ('96b9f483-c44d-5784-cdad-ce21b984fe01', '1.0.0b3.r1', '', '2006-12-15 03:40:00', 1, 100, 4, 'Amazon.com', 'Sean Takats', '^https?://(?:www\.)?amazon',
'function detectWeb(doc, url) {
@ -5883,18 +5883,20 @@ function doWeb(doc, url) {
}
}');
REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b2.r2', '', '2006-12-19 20:33:07', '1', '100', '4', 'Oxford Journals', 'Simon Kornblith', '^http://[^/]+/(?:cgi/searchresults|cgi/content/(?:abstract|full)/[0-9]+/[0-9]+/[0-9]+|current.dtl|content/vol[0-9]+/issue[0-9]+/index.dtl)',
REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b3.r1', '', '2006-12-20 06:40:52', '1', '100', '4', 'HighWire', 'Simon Kornblith', '^http://[^/]+/(?:cgi/searchresults|cgi/search|cgi/content/(?:abstract|full)/[0-9]+/[0-9]+/[0-9]+|current.dtl$|content/vol[0-9]+/issue[0-9]+/(?:index.dt)?$)',
'function detectWeb(doc, url) {
var namespace = doc.documentElement.namespaceURI;
var nsResolver = namespace ? function(prefix) {
if (prefix == ''x'') return namespace; else return null;
} : null;
if(doc.title == "Oxford Journals -- Search Result") {
if(doc.evaluate(''//div[@id="hw"]/table/tbody/tr/td/p/font/table/tbody/tr[td/input[@type="checkbox"]]'', doc,
if(doc.title.indexOf(" -- Search Result") == doc.title.length-17 ||
doc.title == "Science/AAAS | Search Results") {
if(doc.evaluate(''//table/tbody/tr[td/input[@type="checkbox"][@name="gca"]]'', doc,
nsResolver, XPathResult.ANY_TYPE, null).iterateNext()) return "multiple";
} else if(doc.title.indexOf(" -- Table of Contents") != -1) {
if(doc.evaluate(''//div[@id="hw"]/form/dl'', doc, nsResolver, XPathResult.ANY_TYPE,null).iterateNext()) return "multiple";
} else if(doc.title.indexOf(" -- Table of Contents") != -1||
doc.title == "Science/AAAS | Science Magazine Search Results") {
if(doc.evaluate(''//form/dl'', doc, nsResolver, XPathResult.ANY_TYPE,null).iterateNext()) return "multiple";
} else {
if(doc.evaluate(''//a[substring(@href, 1, 16) = "/cgi/citmgr?gca="]'', doc, nsResolver,
XPathResult.ANY_TYPE, null).iterateNext()) return "journalArticle";
@ -5928,10 +5930,10 @@ REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b
if(snapshot) {
if(typeof(snapshot) == "string") {
// string snapshot (from search)
item.attachments.push({title:"Oxford Journals Snapshot", mimeType:"text/html", url:snapshot});
item.attachments.push({title:"HighWire Snapshot", mimeType:"text/html", url:snapshot});
} else {
// document object
item.attachments.push({title:"Oxford Journals Snapshot", document:snapshot});
item.attachments.push({title:"HighWire Snapshot", document:snapshot});
}
}
if(pdf) {
@ -5956,8 +5958,9 @@ function doWeb(doc, url) {
var requests = new Array();
var hostRe = /https?:\/\/[^\/]+/;
var isSearch = doc.title == "Oxford Journals -- Search Result";
var isSearch = doc.title.indexOf("Search Result") != -1
var isTOC = doc.title.indexOf(" -- Table of Contents") != -1;
var isScience = doc.title.indexOf("Science Magazine Search Results") != -1;
if(isSearch || isTOC) {
// search page
var items = new Object();
@ -5965,10 +5968,13 @@ function doWeb(doc, url) {
var pdfs = new Object();
if(isTOC) {
var gcaRe = /^https?:\/\/[^\/]+\/cgi\/content\/abstract\/([0-9]+\/[0-9]+\/[0-9]+)/;
var tableRows = doc.evaluate(''//div[@id="hw"]/form/dl'', doc, nsResolver, XPathResult.ANY_TYPE, null);
var gcaRe = /^https?:\/\/[^\/]+\/cgi\/reprint\/([0-9]+\/[0-9]+\/[0-9]+)/;
var tableRows = doc.evaluate(''//form/dl'', doc, nsResolver, XPathResult.ANY_TYPE, null);
} else if(isScience) {
var tableRows = doc.evaluate(''//form/dl/dd'', doc, nsResolver, XPathResult.ANY_TYPE, null);
var tableDTs = doc.evaluate(''//form/dl/dt'', doc, nsResolver, XPathResult.ANY_TYPE, null);
} else {
var tableRows = doc.evaluate(''//div[@id="hw"]/table/tbody/tr/td/p/font/table/tbody/tr[td/input[@type="checkbox"]]'', doc,
var tableRows = doc.evaluate(''//table/tbody/tr[td/input[@type="checkbox"]]'', doc,
nsResolver, XPathResult.ANY_TYPE, null);
}
@ -5985,25 +5991,32 @@ function doWeb(doc, url) {
// prefer Full Text snapshots, but take abstracts
if(link.textContent == "[Abstract]") {
if(!snapshot) snapshot = link.href;
var m = gcaRe.exec(link.href);
var gca = m[1];
} else if (link.textContent == "[Full Text]") {
snapshot = link.href;
} else if(link.textContent == "[PDF]") {
pdf = link.href;
var m = gcaRe.exec(link.href);
var gca = m[1];
}
}
} else {
var gca = doc.evaluate(''./td/input[@type="checkbox"]'', tableRow, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value;
var title = doc.evaluate(''./td/font/strong'', tableRow, nsResolver, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
title = title.snapshotItem(title.snapshotLength-1).textContent;
if(isScience) {
var tableDT = tableDTs.iterateNext();
var gca = doc.evaluate(''./input[@type="checkbox"]'', tableDT, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value;
var title = doc.evaluate(''./label'', tableDT, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().textContent;
} else {
var gca = doc.evaluate(''./td/input[@type="checkbox"]'', tableRow, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value;
var title = doc.evaluate(''./td/font/strong'', tableRow, nsResolver, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
title = title.snapshotItem(title.snapshotLength-1).textContent;
}
var links = doc.evaluate(''.//font/strong/a'', tableRow, nsResolver, XPathResult.ANY_TYPE, null);
var links = doc.evaluate(''.//a'', tableRow, nsResolver, XPathResult.ANY_TYPE, null);
while(link = links.iterateNext()) {
// prefer Full Text snapshots, but take abstracts
if((link.textContent == "Abstract" && !snapshot) || link.textContent == "Full Text") {
var textContent = link.textContent;
if((textContent.substr(0, 8) == "Abstract" && !snapshot) || textContent.substr(0, 9) == "Full Text") {
snapshot = link.href;
} else if(link.textContent == "PDF") {
} else if(textContent.substr(0, 3) == "PDF") {
pdf = link.href;
}
}
@ -6020,7 +6033,7 @@ function doWeb(doc, url) {
var requests = new Array();
for(var gca in items) {
var m = hostRe.exec(snapshots[gca]);
var m = hostRe.exec(pdfs[gca]);
var baseURL = m[0]+"/cgi/citmgr?type=refman";
var thisRequest = null;