added more logging and a panic detector to try and figure out why process dieing

commands/pilosa-cruncher/cruncher.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"github.com/mitchellh/panicwrap"
 	"log"
 	"os"
 	"pilosa/core"
@@ -15,6 +16,16 @@ var (
 )
 
 func main() {
+	exitStatus, err := panicwrap.BasicWrap(panicHandler)
+	if err != nil {
+		// Something went wrong setting up the panic wrapper. Unlikely,
+		// but possible.
+		panic(err)
+	}
+
+	if exitStatus >= 0 {
+		os.Exit(exitStatus)
+	}
 	core.Build = Build
 
 	flag.Parse()
@@ -30,3 +41,10 @@ func main() {
 	cruncher.Run()
 	log.Println("STOP")
 }
+
+func panicHandler(output string) {
+	// output contains the full output (including stack traces) of the
+	// panic. Put it in a file or something.
+	log.Printf("The child panicked:\n\n%s\n", output)
+	os.Exit(1)
+}

core/etcd.go

@@ -450,8 +450,8 @@ func (self *ProcessMapper) Run() {
 	for _, node := range flatten(response.Node) {
 		err := self.handlenode(node)
 		if err != nil {
-			spew.Dump(node)
-			log.Println(err)
+			out := spew.Sdump(node)
+			log.Println(err, out)
 		}
 	}
 

core/http.go

@@ -309,7 +309,7 @@ func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if results == nil {
-		log.Println("Empty results:", pql)
+		log.Println("Empty results:", database_name, pql)
 		http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -636,7 +636,8 @@ func (self *WebService) streamer(writer func(map[string]interface{}) error) {
 func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		err := recover()
-		spew.Dump(err)
+		out := spew.Sdump(err)
+		log.Println(out)
 	}()
 	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
 	if _, ok := err.(websocket.HandshakeError); ok {

deps.json

@@ -64,6 +64,11 @@
         "version": "f9e07770bd9b5142de517b05a85ffbb42d1895da",
         "type": "git"
     },
+    "panicwrapper": {
+        "repo": "github.com/mitchellh/panicwrap",
+        "version": "master",
+        "type": "git"
+    },
     "rbtree": {
         "repo": "github.com/yasushi-saito/rbtree",
         "version": "571e2538414bf914c7e2909b61217b4e3e5508f4",

executor/executor.go

@@ -166,7 +166,8 @@ func (self *Executor) RunPQL(database_name string, pql string) (interface{}, err
 				}{final, q.Label, err}
 			}(query_list[i], result)
 			if err != nil {
-				spew.Dump(err)
+				out := spew.Sdump(err)
+				log.Println(out)
 			}
 		}
 		for z := 0; z < x; z++ {