From 060512a379569bb14f5f962297cc12a533468c57 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Wed, 20 Dec 2006 11:45:00 +0000 Subject: [PATCH] handle all HighWire journals (Oxford, Science, etc.) with one translator --- scrapers.sql | 57 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/scrapers.sql b/scrapers.sql index 84443ba735..d52336200f 100644 --- a/scrapers.sql +++ b/scrapers.sql @@ -1,4 +1,4 @@ --- 152 +-- 153 -- ***** BEGIN LICENSE BLOCK ***** -- @@ -22,7 +22,7 @@ -- Set the following timestamp to the most recent scraper update date -REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-12-20 06:05:01')); +REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-12-20 06:40:52')); REPLACE INTO translators VALUES ('96b9f483-c44d-5784-cdad-ce21b984fe01', '1.0.0b3.r1', '', '2006-12-15 03:40:00', 1, 100, 4, 'Amazon.com', 'Sean Takats', '^https?://(?:www\.)?amazon', 'function detectWeb(doc, url) { @@ -5883,18 +5883,20 @@ function doWeb(doc, url) { } }'); -REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b2.r2', '', '2006-12-19 20:33:07', '1', '100', '4', 'Oxford Journals', 'Simon Kornblith', '^http://[^/]+/(?:cgi/searchresults|cgi/content/(?:abstract|full)/[0-9]+/[0-9]+/[0-9]+|current.dtl|content/vol[0-9]+/issue[0-9]+/index.dtl)', +REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b3.r1', '', '2006-12-20 06:40:52', '1', '100', '4', 'HighWire', 'Simon Kornblith', '^http://[^/]+/(?:cgi/searchresults|cgi/search|cgi/content/(?:abstract|full)/[0-9]+/[0-9]+/[0-9]+|current.dtl$|content/vol[0-9]+/issue[0-9]+/(?:index.dt)?$)', 'function detectWeb(doc, url) { var namespace = doc.documentElement.namespaceURI; var nsResolver = namespace ? function(prefix) { if (prefix == ''x'') return namespace; else return null; } : null; - if(doc.title == "Oxford Journals -- Search Result") { - if(doc.evaluate(''//div[@id="hw"]/table/tbody/tr/td/p/font/table/tbody/tr[td/input[@type="checkbox"]]'', doc, + if(doc.title.indexOf(" -- Search Result") == doc.title.length-17 || + doc.title == "Science/AAAS | Search Results") { + if(doc.evaluate(''//table/tbody/tr[td/input[@type="checkbox"][@name="gca"]]'', doc, nsResolver, XPathResult.ANY_TYPE, null).iterateNext()) return "multiple"; - } else if(doc.title.indexOf(" -- Table of Contents") != -1) { - if(doc.evaluate(''//div[@id="hw"]/form/dl'', doc, nsResolver, XPathResult.ANY_TYPE,null).iterateNext()) return "multiple"; + } else if(doc.title.indexOf(" -- Table of Contents") != -1|| + doc.title == "Science/AAAS | Science Magazine Search Results") { + if(doc.evaluate(''//form/dl'', doc, nsResolver, XPathResult.ANY_TYPE,null).iterateNext()) return "multiple"; } else { if(doc.evaluate(''//a[substring(@href, 1, 16) = "/cgi/citmgr?gca="]'', doc, nsResolver, XPathResult.ANY_TYPE, null).iterateNext()) return "journalArticle"; @@ -5928,10 +5930,10 @@ REPLACE INTO translators VALUES ('5eacdb93-20b9-4c46-a89b-523f62935ae4', '1.0.0b if(snapshot) { if(typeof(snapshot) == "string") { // string snapshot (from search) - item.attachments.push({title:"Oxford Journals Snapshot", mimeType:"text/html", url:snapshot}); + item.attachments.push({title:"HighWire Snapshot", mimeType:"text/html", url:snapshot}); } else { // document object - item.attachments.push({title:"Oxford Journals Snapshot", document:snapshot}); + item.attachments.push({title:"HighWire Snapshot", document:snapshot}); } } if(pdf) { @@ -5956,8 +5958,9 @@ function doWeb(doc, url) { var requests = new Array(); var hostRe = /https?:\/\/[^\/]+/; - var isSearch = doc.title == "Oxford Journals -- Search Result"; + var isSearch = doc.title.indexOf("Search Result") != -1 var isTOC = doc.title.indexOf(" -- Table of Contents") != -1; + var isScience = doc.title.indexOf("Science Magazine Search Results") != -1; if(isSearch || isTOC) { // search page var items = new Object(); @@ -5965,10 +5968,13 @@ function doWeb(doc, url) { var pdfs = new Object(); if(isTOC) { - var gcaRe = /^https?:\/\/[^\/]+\/cgi\/content\/abstract\/([0-9]+\/[0-9]+\/[0-9]+)/; - var tableRows = doc.evaluate(''//div[@id="hw"]/form/dl'', doc, nsResolver, XPathResult.ANY_TYPE, null); + var gcaRe = /^https?:\/\/[^\/]+\/cgi\/reprint\/([0-9]+\/[0-9]+\/[0-9]+)/; + var tableRows = doc.evaluate(''//form/dl'', doc, nsResolver, XPathResult.ANY_TYPE, null); + } else if(isScience) { + var tableRows = doc.evaluate(''//form/dl/dd'', doc, nsResolver, XPathResult.ANY_TYPE, null); + var tableDTs = doc.evaluate(''//form/dl/dt'', doc, nsResolver, XPathResult.ANY_TYPE, null); } else { - var tableRows = doc.evaluate(''//div[@id="hw"]/table/tbody/tr/td/p/font/table/tbody/tr[td/input[@type="checkbox"]]'', doc, + var tableRows = doc.evaluate(''//table/tbody/tr[td/input[@type="checkbox"]]'', doc, nsResolver, XPathResult.ANY_TYPE, null); } @@ -5985,25 +5991,32 @@ function doWeb(doc, url) { // prefer Full Text snapshots, but take abstracts if(link.textContent == "[Abstract]") { if(!snapshot) snapshot = link.href; - var m = gcaRe.exec(link.href); - var gca = m[1]; } else if (link.textContent == "[Full Text]") { snapshot = link.href; } else if(link.textContent == "[PDF]") { pdf = link.href; + var m = gcaRe.exec(link.href); + var gca = m[1]; } } } else { - var gca = doc.evaluate(''./td/input[@type="checkbox"]'', tableRow, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value; - var title = doc.evaluate(''./td/font/strong'', tableRow, nsResolver, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); - title = title.snapshotItem(title.snapshotLength-1).textContent; + if(isScience) { + var tableDT = tableDTs.iterateNext(); + var gca = doc.evaluate(''./input[@type="checkbox"]'', tableDT, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value; + var title = doc.evaluate(''./label'', tableDT, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().textContent; + } else { + var gca = doc.evaluate(''./td/input[@type="checkbox"]'', tableRow, nsResolver, XPathResult.ANY_TYPE, null).iterateNext().value; + var title = doc.evaluate(''./td/font/strong'', tableRow, nsResolver, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); + title = title.snapshotItem(title.snapshotLength-1).textContent; + } - var links = doc.evaluate(''.//font/strong/a'', tableRow, nsResolver, XPathResult.ANY_TYPE, null); + var links = doc.evaluate(''.//a'', tableRow, nsResolver, XPathResult.ANY_TYPE, null); while(link = links.iterateNext()) { // prefer Full Text snapshots, but take abstracts - if((link.textContent == "Abstract" && !snapshot) || link.textContent == "Full Text") { + var textContent = link.textContent; + if((textContent.substr(0, 8) == "Abstract" && !snapshot) || textContent.substr(0, 9) == "Full Text") { snapshot = link.href; - } else if(link.textContent == "PDF") { + } else if(textContent.substr(0, 3) == "PDF") { pdf = link.href; } } @@ -6020,7 +6033,7 @@ function doWeb(doc, url) { var requests = new Array(); for(var gca in items) { - var m = hostRe.exec(snapshots[gca]); + var m = hostRe.exec(pdfs[gca]); var baseURL = m[0]+"/cgi/citmgr?type=refman"; var thisRequest = null;