From 0225f6e798f2c3a3d0616369b266c4764afeea5a Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 26 Nov 2014 10:27:53 +0000
Subject: [PATCH] added more logging and a panic detector to try and figure out
 why process dieing

---
 commands/pilosa-cruncher/cruncher.go | 18 ++++++++++++++++++
 core/etcd.go                         |  4 ++--
 core/http.go                         |  5 +++--
 deps.json                            |  5 +++++
 executor/executor.go                 |  3 ++-
 5 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/commands/pilosa-cruncher/cruncher.go b/commands/pilosa-cruncher/cruncher.go
index 1ac215f62..3158f31ce 100644
--- a/commands/pilosa-cruncher/cruncher.go
+++ b/commands/pilosa-cruncher/cruncher.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"github.com/mitchellh/panicwrap"
 	"log"
 	"os"
 	"pilosa/core"
@@ -15,6 +16,16 @@ var (
 )
 
 func main() {
+	exitStatus, err := panicwrap.BasicWrap(panicHandler)
+	if err != nil {
+		// Something went wrong setting up the panic wrapper. Unlikely,
+		// but possible.
+		panic(err)
+	}
+
+	if exitStatus >= 0 {
+		os.Exit(exitStatus)
+	}
 	core.Build = Build
 
 	flag.Parse()
@@ -30,3 +41,10 @@ func main() {
 	cruncher.Run()
 	log.Println("STOP")
 }
+
+func panicHandler(output string) {
+	// output contains the full output (including stack traces) of the
+	// panic. Put it in a file or something.
+	log.Printf("The child panicked:\n\n%s\n", output)
+	os.Exit(1)
+}
diff --git a/core/etcd.go b/core/etcd.go
index 145dcefc4..69475c838 100644
--- a/core/etcd.go
+++ b/core/etcd.go
@@ -450,8 +450,8 @@ func (self *ProcessMapper) Run() {
 	for _, node := range flatten(response.Node) {
 		err := self.handlenode(node)
 		if err != nil {
-			spew.Dump(node)
-			log.Println(err)
+			out := spew.Sdump(node)
+			log.Println(err, out)
 		}
 	}
 
diff --git a/core/http.go b/core/http.go
index 58a8e3dca..df1b69697 100644
--- a/core/http.go
+++ b/core/http.go
@@ -309,7 +309,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if results == nil {
-		log.Println("Empty results:", pql)
+		log.Println("Empty results:", database_name, pql)
 		http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -636,7 +636,8 @@ func (self *WebService) streamer(writer func(map[string]interface{}) error) {
 func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		err := recover()
-		spew.Dump(err)
+		out := spew.Sdump(err)
+		log.Println(out)
 	}()
 	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
 	if _, ok := err.(websocket.HandshakeError); ok {
diff --git a/deps.json b/deps.json
index a8c1389d5..31a8530a2 100644
--- a/deps.json
+++ b/deps.json
@@ -64,6 +64,11 @@
         "version": "f9e07770bd9b5142de517b05a85ffbb42d1895da",
         "type": "git"
     },
+    "panicwrapper": {
+        "repo": "github.com/mitchellh/panicwrap",
+        "version": "master",
+        "type": "git"
+    },
     "rbtree": {
         "repo": "github.com/yasushi-saito/rbtree",
         "version": "571e2538414bf914c7e2909b61217b4e3e5508f4",
diff --git a/executor/executor.go b/executor/executor.go
index ae1db3673..ad15ac8c2 100644
--- a/executor/executor.go
+++ b/executor/executor.go
@@ -166,7 +166,8 @@ func (self *Executor) RunPQL(database_name string, pql string) (interface{}, err
 				}{final, q.Label, err}
 			}(query_list[i], result)
 			if err != nil {
-				spew.Dump(err)
+				out := spew.Sdump(err)
+				log.Println(out)
 			}
 		}
 		for z := 0; z < x; z++ {