From ca3a0e6e5d6fa29936cc376885448504d0707d3f Mon Sep 17 00:00:00 2001
From: Simon Kornblith <simon@simonster.com>
Date: Thu, 22 Jun 2006 02:43:40 +0000
Subject: [PATCH] Beginnings of search result scraping (does not yet actually
 do the scraping, but does present the menu)

---
 .../content/scholar/ingester/browser.js       | 30 +++------
 .../content/scholar/ingester/browser.xul      |  4 --
 .../content/scholar/ingester/selectitems.js   | 44 ++++++++++++
 .../content/scholar/ingester/selectitems.xul  | 23 +++++++
 .../content/scholar/xpcom/ingester.js         | 12 +++-
 .../locale/en-US/scholar/scholar.dtd          |  7 +-
 scrapers.sql                                  | 67 ++++++++++++++-----
 7 files changed, 144 insertions(+), 43 deletions(-)
 create mode 100644 chrome/chromeFiles/content/scholar/ingester/selectitems.js
 create mode 100755 chrome/chromeFiles/content/scholar/ingester/selectitems.xul

diff --git a/chrome/chromeFiles/content/scholar/ingester/browser.js b/chrome/chromeFiles/content/scholar/ingester/browser.js
index baf76dab4c..8813c35828 100644
--- a/chrome/chromeFiles/content/scholar/ingester/browser.js
+++ b/chrome/chromeFiles/content/scholar/ingester/browser.js
@@ -61,11 +61,21 @@ Scholar_Ingester_Interface.chromeUnload = function() {
 Scholar_Ingester_Interface.scrapeThisPage = function() {
 	var documentObject = Scholar_Ingester_Interface._getDocument(Scholar_Ingester_Interface.tabBrowser.selectedBrowser);
 	if(documentObject.scraper) {
+		if(documentObject.scrapeURLList) {
+			// In the case that there are multiple scrapable URLs, make the user choose
+			Scholar_Ingester_Interface.chooseURL(documentObject);
+		}
 		Scholar_Ingester_Interface.scrapeProgress = new Scholar_Ingester_Interface.Progress(window, Scholar_Ingester_Interface.tabBrowser.selectedBrowser.contentDocument, Scholar.getString("ingester.scraping"));
 		documentObject.scrapePage(Scholar_Ingester_Interface._finishScraping);
 	}
 }
 
+Scholar_Ingester_Interface.chooseURL = function(documentObject) {
+	Scholar.debug("chooseURL called");
+	var newDialog = window.openDialog("chrome://scholar/content/ingester/selectitems.xul",
+		"_blank","chrome,modal,centerscreen,resizable=yes", documentObject);
+}
+
 /*
  * Updates the status of the capture icon to reflect the scrapability or lack
  * thereof of the current page
@@ -108,7 +118,6 @@ Scholar_Ingester_Interface.Listener.onStateChange = function() {}
  * appropriate status indicator for the current tab, and to free useless objects
  */
 Scholar_Ingester_Interface.Listener.onLocationChange = function(progressObject) {
-	Scholar.debug("onLocationChange called");
     var browsers = Scholar_Ingester_Interface.tabBrowser.browsers;
 
     // Remove document object of any browser that no longer exists
@@ -130,25 +139,6 @@ Scholar_Ingester_Interface.Listener.onLocationChange = function(progressObject)
             Scholar_Ingester_Interface._deleteDocument(browser);
         }
     }
-    
-    /*// Add a collector to any new browser
-    for (var i = 0; i < browsers.length; i++) {
-        var browser = browsers[i];
-        var exists = false;
-
-        for (var j = 0; j < Scholar_Ingester_Interface.browsers.length; j++) {
-            if (browser == Scholar_Ingester_Interface.browsers[j]) {
-                exists = true;
-                break;
-            }
-        }
-
-        if (!exists) {
-            Scholar_Ingester_Interface.browsers.splice(i,0,browser);
-            
-        	// To execute if window is new
-        }
-    }*/
 
     Scholar_Ingester_Interface.updateStatus(
     	Scholar_Ingester_Interface.tabBrowser.selectedBrowser
diff --git a/chrome/chromeFiles/content/scholar/ingester/browser.xul b/chrome/chromeFiles/content/scholar/ingester/browser.xul
index 9e5e80e803..b9b230b1e1 100755
--- a/chrome/chromeFiles/content/scholar/ingester/browser.xul
+++ b/chrome/chromeFiles/content/scholar/ingester/browser.xul
@@ -1,12 +1,8 @@
 <?xml version="1.0"?>
-
-
 <!-- Note: Contains Firefox-specific overlay -->
-
 <overlay id="scholar-ingester-overlay" xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
 
 	<script src="../include.js"/>
-
     <script src="browser.js"/>
     
 	<script type="application/x-javascript">
diff --git a/chrome/chromeFiles/content/scholar/ingester/selectitems.js b/chrome/chromeFiles/content/scholar/ingester/selectitems.js
new file mode 100644
index 0000000000..4f78d63891
--- /dev/null
+++ b/chrome/chromeFiles/content/scholar/ingester/selectitems.js
@@ -0,0 +1,44 @@
+//////////////////////////////////////////////////////////////////////////////
+//
+// Scholar_Ingester_Interface_SelectItems
+//
+//////////////////////////////////////////////////////////////////////////////
+
+// Class to interface with the browser when ingesting data
+
+Scholar_Ingester_Interface_SelectItems = function() {}
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// Public Scholar_Ingester_Interface_SelectItems methods
+//
+//////////////////////////////////////////////////////////////////////////////
+
+/*
+ * Initialize some variables and prepare event listeners for when chrome is done
+ * loading
+ */
+Scholar_Ingester_Interface_SelectItems.init = function() {
+	this.documentObject = window.arguments[0];
+	this.listbox = document.getElementById("scholar-selectitems-links");
+	
+	for(i in this.documentObject.scrapeURLList) {	// we could use a tree for this if we wanted to
+		var itemNode = document.createElement("listitem");
+		itemNode.setAttribute("type", "checkbox");
+		itemNode.setAttribute("value", i);
+		itemNode.setAttribute("label", this.documentObject.scrapeURLList[i]);
+		itemNode.setAttribute("checked", false);
+		this.listbox.appendChild(itemNode);
+	}
+}
+
+Scholar_Ingester_Interface_SelectItems.acceptSelection = function() {
+	// clear scrapeURLList
+	this.documentObject.scrapeURLList = new Object();
+	
+	// collect scrapeURLList from listbox
+	for(var i=0; i<this.listbox.length; i++) {
+		var itemNode = this.listbox[i];
+		this.documentObject.scrapeURLList[itemNode.getAttribute("value")] = itemNode.getAttribute("label");
+	}
+}
\ No newline at end of file
diff --git a/chrome/chromeFiles/content/scholar/ingester/selectitems.xul b/chrome/chromeFiles/content/scholar/ingester/selectitems.xul
new file mode 100755
index 0000000000..31f65f326d
--- /dev/null
+++ b/chrome/chromeFiles/content/scholar/ingester/selectitems.xul
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<!--
+Borrowed from Linky, originally MPL/GPL/LGPL (now GPL, and modified into oblivion)
+-->
+<?xml-stylesheet href="chrome://global/skin/" type="text/css"?>
+<!DOCTYPE window SYSTEM "chrome://scholar/locale/scholar.dtd">
+<dialog xmlns:html="http://www.w3.org/1999/xhtml"
+	xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
+	title="&selectitems.title;" width="400" height="330"
+	persist="width height screenX screenY"
+	buttons="cancel,accept"
+	ondialogaccept="Scholar_Ingester_Interface_SelectItems.acceptSelection()"
+	ondialogcancel="self.close()"
+	id="scholar-selectitems"
+	onload="Scholar_Ingester_Interface_SelectItems.init()">
+	
+	<script src="../include.js"/>
+	<script src="selectitems.js"/>
+	<caption label="&selectitems.intro.label;" id="scholar-selectitems-intro"/>
+	<box flex="1">
+		<listbox id="scholar-selectitems-links" flex="1" context="scholarSelectContext"></listbox>
+	</box>
+</dialog>
\ No newline at end of file
diff --git a/chrome/chromeFiles/content/scholar/xpcom/ingester.js b/chrome/chromeFiles/content/scholar/xpcom/ingester.js
index 7fb450137d..bc58a3f8ad 100644
--- a/chrome/chromeFiles/content/scholar/xpcom/ingester.js
+++ b/chrome/chromeFiles/content/scholar/xpcom/ingester.js
@@ -283,7 +283,7 @@ Scholar.Ingester.Utilities.prototype.cleanAuthor = function(author) {
  */
 Scholar.Ingester.Utilities.prototype.cleanString = function(s) {
 	s = this.trimString(s);
-	return s.replace(/ +/g, " ");
+	return s.replace(/[ \xA0]+/g, " ");
 }
 
 /*
@@ -569,7 +569,15 @@ Scholar.Ingester.Document.prototype.canScrape = function(currentScraper) {
 							   "\n})()", scraperSandbox);
 		} catch(e) {
 			Scholar.debug(e+' in scraperDetectCode for '+currentScraper.label);
-			canScrape = false;
+			return false;
+		}
+				
+		// scraperDetectCode returns an associative array (object) in the case of a search result
+		if(typeof(canScrape) == "object") {
+			Scholar.debug("scraperDetectCode returned a URL list");
+			this.scrapeURLList = canScrape;
+		} else {
+			Scholar.debug("canScrape was a "+typeof(canScrape));
 		}
 	}
 	return canScrape;
diff --git a/chrome/chromeFiles/locale/en-US/scholar/scholar.dtd b/chrome/chromeFiles/locale/en-US/scholar/scholar.dtd
index 24ef7e934b..e351a1107d 100644
--- a/chrome/chromeFiles/locale/en-US/scholar/scholar.dtd
+++ b/chrome/chromeFiles/locale/en-US/scholar/scholar.dtd
@@ -24,4 +24,9 @@
 <!ENTITY toolbar.newCollection.label   	"New Project">
 <!ENTITY toolbar.renameCollection.label   "Rename Project...">
 <!ENTITY toolbar.removeCollection.label   "Remove Project...">
-<!ENTITY toolbar.search.label   			"Search:">
\ No newline at end of file
+<!ENTITY toolbar.search.label   			"Search:">
+
+<!ENTITY selectitems.title   			"Select Items">
+<!ENTITY selectitems.intro.label   		"Select which items you'd like to add to your library">
+<!ENTITY selectitems.cancel.label   	"Cancel">
+<!ENTITY selectitems.select.label   	"OK">
\ No newline at end of file
diff --git a/scrapers.sql b/scrapers.sql
index 3afa019d95..526552e1d4 100644
--- a/scrapers.sql
+++ b/scrapers.sql
@@ -175,24 +175,59 @@ utilities.HTTPUtilities.doPost(newUri, ''exportselect=record&exporttype=plaintex
 wait();');
 
 REPLACE INTO "scrapers" VALUES('88915634-1af6-c134-0171-56fd198235ed', '2006-06-18 11:02:00', 'LOC/Voyager WebVoyage Scraper', 'Simon Kornblith', 'Pwebrecon\.cgi',
-'try {
-	if(doc.forms.namedItem(''frm'').elements.namedItem(''RC'')) {
-		return false;
-	}
-	var export_options = doc.forms.namedItem(''frm'').elements.namedItem(''RD'').options;
-	for(i in export_options) {
-		if(export_options[i].text == ''Latin1 MARC''
-		|| export_options[i].text == ''Raw MARC''
-		|| export_options[i].text == ''UTF-8''
-		|| export_options[i].text == ''MARC (Unicode/UTF-8)''
-		|| export_options[i].text == ''MARC (non-Unicode/MARC-8)'') {
-			return true;
+'if(doc.forms.namedItem(''frm'').elements.namedItem(''RC'')) {	
+	// We have search results
+	
+	var namespace = doc.documentElement.namespaceURI;
+	var nsResolver = namespace ? function(prefix) {
+		if (prefix == ''x'') return namespace; else return null;
+	} : null;
+	
+	var availableItems = new Object();	// Technically, associative arrays are objects
+	
+	// Require link to match this
+	var tagRegexp = new RegExp();
+	tagRegexp.compile(''Pwebrecon\\.cgi\\?.*v1=[0-9]+\\&.*ti='');
+	// Do not allow text to match this
+	var rejectRegexp = new RegExp();
+	rejectRegexp.compile(''\[ [0-9]+ \]'');
+	
+	var links = doc.getElementsByTagName("a");
+	for(var i=0; i<links.length; i++) {
+		if(tagRegexp.test(links[i].href)) {
+			var text = utilities.getNodeString(doc, links[i], ''.//text()'', nsResolver);
+			if(text) {
+				text = utilities.cleanString(text);
+				if(!rejectRegexp.test(text)) {
+					if(availableItems[links[i].href]) {
+						availableItems[links[i].href] += " "+text;
+					} else {
+						availableItems[links[i].href] = text;
+					}
+				}
+			}
 		}
 	}
-	return false;
-} catch(e) {
-	return false;
-}',
+	
+	if(availableItems) {
+		return availableItems;
+	} else {
+		return false;
+	}
+}
+
+var export_options = doc.forms.namedItem(''frm'').elements.namedItem(''RD'').options;
+for(i in export_options) {
+	if(export_options[i].text == ''Latin1 MARC''
+	|| export_options[i].text == ''Raw MARC''
+	|| export_options[i].text == ''UTF-8''
+	|| export_options[i].text == ''MARC (Unicode/UTF-8)''
+	|| export_options[i].text == ''MARC (non-Unicode/MARC-8)'') {
+		// We have an exportable single record
+		return true;
+	}
+}
+return false;',
 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
 var prefixDC = ''http://purl.org/dc/elements/1.1/'';
 var prefixDCMI = ''http://purl.org/dc/dcmitype/'';