.

cmd/sauron/design.org

@@ -0,0 +1,199 @@
+
+* Oracle Design
+- setup server that intersepts PQL queries and proxies to live pilosa server
+- implement all the pilosa functions using SQL, specifically the sqlite flavor.  This will
+  provide validation against pilosa results
+- provide timings for each query path, PQL and SQL and compare
+- this is intended to be a correctness check and not a performance check.  the
+  sql solution is likely to be pretty slow with realitively small numbers ~1millions
+  #+BEGIN_SRC 
+   .─────.                                              
+ ,'       `.             ┌────────────┐       ┌────────┐
+;pilosa json:            │   Sauron   │   ┌──▶│ pilosa │
+:  client   ;──────────▶ │            │───┤   └────────┘
+ ╲         ╱             │            │   │   ┌────────┐
+  `.     ,'              └────────────┘   └──▶│ sqlite │
+    `───'                                     └────────┘
+  #+END_SRC
+* Setup Sql (sqlite)
+#+name: setup test
+#+header: :results silent
+#+header: :db poc.db
+  #+BEGIN_SRC sqlite
+    create table bits (field string,row string, column bigint);
+    create table bsi (field string,column bigint , val bigint);
+    CREATE VIEW columns as
+	   select distinct column from bits
+	   UNION
+	   select distinct column from bsi;
+   
+    INSERT INTO bits VALUES( 'color','red',1);
+    INSERT INTO bits VALUES( 'color','red',2);
+    INSERT INTO bits VALUES( 'color','red',3);
+    INSERT INTO bits VALUES( 'color','red',4);
+    INSERT INTO bits VALUES( 'color','red',5);
+    INSERT INTO bits VALUES( 'color','green',2);
+    INSERT INTO bits VALUES( 'color','green',4);
+    INSERT INTO bits VALUES( 'color','green',6);
+    INSERT INTO bits VALUES( 'color','green',8);
+    INSERT INTO bits VALUES( 'color','yellow',1);
+    INSERT INTO bits VALUES( 'color','yellow',3);
+    INSERT INTO bits VALUES( 'color','yellow',5);
+    INSERT INTO bits VALUES( 'color','yellow',7);
+    INSERT INTO bits VALUES( 'color','orange',5);
+    INSERT INTO bits VALUES( 'color','orange',6);
+    INSERT INTO bits VALUES( 'color','orange',7);
+    INSERT INTO bits VALUES( 'color','orange',8);
+
+    INSERT INTO bsi VALUES( 'luminosity',1, 1);
+    INSERT INTO bsi VALUES( 'luminosity',2, 2);
+    INSERT INTO bsi VALUES( 'luminosity',3, 4);
+    INSERT INTO bsi VALUES( 'luminosity',4, 5);
+    INSERT INTO bsi VALUES( 'luminosity',5, 4);
+    INSERT INTO bsi VALUES( 'luminosity',6, 3);
+    INSERT INTO bsi VALUES( 'luminosity',7, 2);
+    INSERT INTO bsi VALUES( 'luminosity',8, 1);
+  #+END_SRC
+  
+* TODO PQL to SQL TODO [10/11]
+- [X] Row(color=red)
+  #+begin_src go
+(*pql.Query)(&pql.Query{
+        Calls: ([]*pql.Call)([]*pql.Call{
+                (*pql.Call)(&pql.Call{
+                        Name: (string)("Row"),
+                        Args: (map[string]interface{})(map[string]interface{}{
+                                (string)("color"): (string)("red"),
+                        }),
+                        Children:    ([]*pql.Call)(nil),
+                        Type:        (pql.CallType)(0),
+                        Precomputed: (map[uint64]interface{})(nil),
+                }),
+        }),
+        callStack:   ([]*pql.callStackElem)([]*pql.callStackElem{}),
+        conditional: ([]string)(nil),
+})
+  #+end_src
+  #+BEGIN_SRC sqlite
+    select column from bits where field="color" AND row="red"
+  #+END_SRC
+- [X] Count(Row(color=red))
+  #+begin_src go
+(*pql.Query)(&pql.Query{
+        Calls: ([]*pql.Call)([]*pql.Call{
+                (*pql.Call)(&pql.Call{
+                        Name: (string)("Count"),
+                        Args: (map[string]interface{})(nil),
+                        Children: ([]*pql.Call)([]*pql.Call{
+                                (*pql.Call)(&pql.Call{
+                                        Name: (string)("Row"),
+                                        Args: (map[string]interface{})(map[string]interface{}{
+                                                (string)("color"): (string)("red"),
+                                        }),
+                                        Children:    ([]*pql.Call)(nil),
+                                        Type:        (pql.CallType)(0),
+                                        Precomputed: (map[uint64]interface{})(nil),
+                                }),
+                        }),
+                        Type:        (pql.CallType)(0),
+                        Precomputed: (map[uint64]interface{})(nil),
+                }),
+        }),
+        callStack:   ([]*pql.callStackElem)([]*pql.callStackElem{}),
+        conditional: ([]string)(nil),
+})
+
+  #+end_src
+  #+BEGIN_SRC sql
+    select count(*) from(
+    select column from bits where field="color" AND row="red"
+    )
+  #+END_SRC
+- [X] Intersect(Row(color=red),Row(color=green))
+  #+BEGIN_SRC sql
+    select column from bits where field="color" AND row="red"
+    intersect
+    select column from bits where field="color" AND row="green"
+  #+END_SRC
+- [X] Union(Row(color=red),Row(color=green))
+  #+BEGIN_SRC sql
+    select column from bits where field="color" AND row="red"
+    union
+    select column from bits where field="color" AND row="green"
+  #+END_SRC
+- [X] Difference(Row(color=red),Row(color=green))
+  #+BEGIN_SRC sql
+    select column from bits where field="color" AND row="red"
+    except
+    select column from bits where field="color" AND row="green"
+  #+END_SRC
+- [X] Not(Row(color=red))
+  #+BEGIN_SRC sql
+    select column from columns
+    except
+    select column from bits where field="color" AND row="red"
+  #+END_SRC
+- [X] Xor(Row(color=red),Row(color=green))
+  #+BEGIN_SRC sql
+    select column from (
+      select column from bits where field="color" AND row="green"
+      union
+      select column from bits where field="color" AND row="red"
+    )
+    except
+      select column from (
+      select column from bits where field="color" AND row="green"
+      intersect
+      select column from bits where field="color" AND row="red"
+    )
+  #+END_SRC
+- [X] Distinct(field=luminosity)
+  #+BEGIN_SRC sql
+    select distinct val from bsi where field="lumin"
+  #+END_SRC
+- [X] Distinct(Row(color="red"),field=luminosity)
+  #+BEGIN_SRC sql
+  select distinct val from bsi
+      where field="luminosity" ANDcolumn in (
+	        select column from bits where row="color" AND row="red"
+    )
+  #+END_SRC
+- [X] Distinct(Intersect(Row(color="red"),Row(color="green")),luminosity)
+  #+BEGIN_SRC sql
+      select distinct val from bsi
+      where column in (
+	select column from bits where field="color" AND row="red"
+	intersect
+	select column from bits where field="color" AND row="green"
+    )
+     AND field="luminosity"
+  #+END_SRC
+
+- [X] Count(Union(Intersect(Row(color=red),Row(color=green)),Intersect(Row(color=yellow),Row(color=orange))))
+#+begin_src sqlite
+select count(*) from (
+select column from (
+    select column from bits where field="color" AND row="green"
+    intersect
+    select column from bits where field="color" AND row="red"
+)
+union
+select column from (
+    select column from bits where field="color" AND row="yellow"
+    intersect
+    select column from bits where field="color" AND row="orange"
+))
+
+- [ ] Rows(color)
+#+begin_src sqlite
+select distinct row from bits where field="color"
+#+end_src
+
+- [ ] GroupBy(Rows(color))
+  #+BEGIN_SRC sql
+    select row,count(*) from bits where field="color" group by row;
+  #+END_SRC
+- [ ] GroupBy(Rows(color), Rows(other), limit=7)
+  #+BEGIN_SRC sql
+    select row,count(*) from bits where field="color" group by row;
+  #+END_SRC

cmd/sauron/httputil.go

@@ -0,0 +1,274 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"mime"
+	"net/http"
+	"strings"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pkg/errors"
+)
+
+// validHeaderAcceptJSON returns false if one or more Accept
+// headers are present, but none of them are "application/json"
+// (or any matching wildcard). Otherwise returns true.
+func validHeaderAcceptJSON(header http.Header) bool {
+	return validHeaderAcceptType(header, "application", "json")
+}
+
+func validHeaderAcceptType(header http.Header, typ, subtyp string) bool {
+	if v, found := header["Accept"]; found {
+		for _, v := range v {
+			t, _, err := mime.ParseMediaType(v)
+			if err != nil {
+				switch err {
+				case mime.ErrInvalidMediaParameter:
+					// This is an optional feature, so we can keep going anyway.
+				default:
+					continue
+				}
+			}
+			spl := strings.SplitN(t, "/", 2)
+			if len(spl) < 2 {
+				continue
+			}
+			switch {
+			case spl[0] == typ && spl[1] == subtyp:
+				return true
+			case spl[0] == "*" && spl[1] == subtyp:
+				return true
+			case spl[0] == typ && spl[1] == "*":
+				return true
+			case spl[0] == "*" && spl[1] == "*":
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}
+
+// successResponse is a general success/error struct for http responses.
+type successResponse struct {
+	//h         *Handler
+	Success   bool   `json:"success"`
+	Name      string `json:"name,omitempty"`
+	CreatedAt int64  `json:"createdAt,omitempty"`
+	//Error     *Error `json:"error,omitempty"`
+	Error error
+}
+
+// check determines success or failure based on the error.
+// It also returns the corresponding http status code.
+func (r *successResponse) check(err error) (statusCode int) {
+	if err == nil {
+		r.Success = true
+		return 0
+	}
+
+	cause := errors.Cause(err)
+
+	// Determine HTTP status code based on the error type.
+	switch cause.(type) {
+	case pilosa.BadRequestError:
+		statusCode = http.StatusBadRequest
+	case pilosa.ConflictError:
+		statusCode = http.StatusConflict
+	case pilosa.NotFoundError:
+		statusCode = http.StatusNotFound
+	default:
+		statusCode = http.StatusInternalServerError
+	}
+
+	r.Success = false
+	r.Error = err // = &Error{Message: err.Error()}
+
+	return statusCode
+}
+
+// write sends a response to the http.ResponseWriter based on the success
+// status and the error.
+func (r *successResponse) write(w http.ResponseWriter, err error) {
+	// Apply the error and get the status code.
+	statusCode := r.check(err)
+
+	// Marshal the json response.
+	msg, err := json.Marshal(r)
+	if err != nil {
+		http.Error(w, string(msg), http.StatusInternalServerError)
+		return
+	}
+
+	// Write the response.
+	if statusCode == 0 {
+		w.Header().Set("Content-Type", "application/json")
+		_, err := w.Write(msg)
+		if err != nil {
+			log.Printf("error writing response: %v", err)
+			return
+		}
+		_, err = w.Write([]byte("\n"))
+		if err != nil {
+			log.Printf("error writing newline after response: %v", err)
+			return
+		}
+	} else {
+		http.Error(w, string(msg), statusCode)
+	}
+}
+
+type postFieldRequest struct {
+	Options fieldOptions `json:"options"`
+}
+
+// fieldOptions tracks pilosa.FieldOptions. It is made up of pointers to values,
+// and used for input validation.
+type fieldOptions struct {
+	Type           string              `json:"type,omitempty"`
+	CacheType      *string             `json:"cacheType,omitempty"`
+	CacheSize      *uint32             `json:"cacheSize,omitempty"`
+	Min            *pql.Decimal        `json:"min,omitempty"`
+	Max            *pql.Decimal        `json:"max,omitempty"`
+	Scale          *int64              `json:"scale,omitempty"`
+	TimeQuantum    *pilosa.TimeQuantum `json:"timeQuantum,omitempty"`
+	Keys           *bool               `json:"keys,omitempty"`
+	NoStandardView bool                `json:"noStandardView,omitempty"`
+	ForeignIndex   *string             `json:"foreignIndex,omitempty"`
+}
+
+func (o *fieldOptions) ToPilosaFieldOptions() *pilosa.FieldOptions {
+
+	fo := &pilosa.FieldOptions{
+		Type: o.Type,
+
+		/*
+			//Base           int64       `json:"base,omitempty"`
+			//BitDepth       uint        `json:"bitDepth,omitempty"`
+			Min:            *o.Min,
+			Max:            *o.Max,
+			Scale:          *o.Scale,
+			NoStandardView: o.NoStandardView,
+			CacheSize:      *o.CacheSize,
+			CacheType:      *o.CacheType,
+			TimeQuantum:    *o.TimeQuantum,
+			ForeignIndex:   *o.ForeignIndex,
+		*/
+	}
+	if o.Keys != nil {
+		fo.Keys = *o.Keys
+	}
+	return fo
+}
+
+func (o *fieldOptions) validate() error {
+	// Pointers to default values.
+	defaultCacheType := pilosa.DefaultCacheType
+	defaultCacheSize := uint32(pilosa.DefaultCacheSize)
+
+	switch o.Type {
+	case pilosa.FieldTypeSet, "":
+		// Because FieldTypeSet is the default, its arguments are
+		// not required. Instead, the defaults are applied whenever
+		// a value does not exist.
+		if o.Type == "" {
+			o.Type = pilosa.FieldTypeSet
+		}
+		if o.CacheType == nil {
+			o.CacheType = &defaultCacheType
+		}
+		if o.CacheSize == nil {
+			o.CacheSize = &defaultCacheSize
+		}
+		if o.Min != nil {
+			return pilosa.NewBadRequestError(errors.New("min does not apply to field type set"))
+		} else if o.Max != nil {
+			return pilosa.NewBadRequestError(errors.New("max does not apply to field type set"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type set"))
+		}
+	case pilosa.FieldTypeInt:
+		if o.CacheType != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int"))
+		} else if o.CacheSize != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
+		} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
+			return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
+		}
+	case pilosa.FieldTypeDecimal:
+		if o.Scale == nil {
+			return pilosa.NewBadRequestError(errors.New("decimal field requires a scale argument"))
+		} else if o.CacheType != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type int"))
+		} else if o.CacheSize != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type int"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type int"))
+		} else if o.ForeignIndex != nil && o.Type == pilosa.FieldTypeDecimal {
+			return pilosa.NewBadRequestError(errors.New("decimal field cannot be a foreign key"))
+		}
+	case pilosa.FieldTypeTime:
+		if o.CacheType != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type time"))
+		} else if o.CacheSize != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type time"))
+		} else if o.Min != nil {
+			return pilosa.NewBadRequestError(errors.New("min does not apply to field type time"))
+		} else if o.Max != nil {
+			return pilosa.NewBadRequestError(errors.New("max does not apply to field type time"))
+		} else if o.TimeQuantum == nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum is required for field type time"))
+		}
+	case pilosa.FieldTypeMutex:
+		if o.CacheType == nil {
+			o.CacheType = &defaultCacheType
+		}
+		if o.CacheSize == nil {
+			o.CacheSize = &defaultCacheSize
+		}
+		if o.Min != nil {
+			return pilosa.NewBadRequestError(errors.New("min does not apply to field type mutex"))
+		} else if o.Max != nil {
+			return pilosa.NewBadRequestError(errors.New("max does not apply to field type mutex"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex"))
+		}
+	case pilosa.FieldTypeBool:
+		if o.CacheType != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool"))
+		} else if o.CacheSize != nil {
+			return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool"))
+		} else if o.Min != nil {
+			return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool"))
+		} else if o.Max != nil {
+			return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool"))
+		} else if o.TimeQuantum != nil {
+			return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool"))
+		} else if o.Keys != nil {
+			return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool"))
+		} else if o.ForeignIndex != nil {
+			return pilosa.NewBadRequestError(errors.New("bool field cannot be a foreign key"))
+		}
+	default:
+		return errors.Errorf("invalid field type: %s", o.Type)
+	}
+	return nil
+}

cmd/sauron/populate.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+mod=20
+for i in `seq 100 1`;do
+b=$(dc -e "$i $mod %p")
+[[ $b == 0 ]] && mod=$(($mod+1)) 
+r=$(($b+30))
+payload="Set($i, j=$r)"
+echo $payload
+curl -XPOST "localhost:10101/index/i/query" -d "$payload"
+done

cmd/sauron/sauron_test.go

@@ -0,0 +1,396 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"testing"
+
+	pb "github.com/golang/protobuf/proto" //nolint:staticcheck
+	pbuf "github.com/molecula/go-pilosa/v2/gopilosa_pbuf"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/encoding/proto"
+	"github.com/pkg/errors"
+)
+
+func TestSauron(t *testing.T) {
+
+	cfg := NewSauronTestConfig(t)
+	proxy := NewSauron(cfg)
+	proxy.sql.MustRemove() //remove the existing db file if present
+	url, err := proxy.Start()
+	panicOn(err)
+	defer proxy.Stop()
+
+	schemaJson := `{"indexes":[{"name":"scratch","createdAt":1611185870149882000,"options":{"keys":false,"trackExistence":true}, "fields":[{"name":"luminosity","createdAt":1595896639332730413,"options":{"type":"int","base":0,"bitDepth":31,"min":-9223372036854775808,"max":9223372036854775807,"keys":false,"foreignIndex":""}}]}]}`
+
+	client := &http.Client{}
+	t.Run("Schema", func(t *testing.T) {
+		_, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
+		panicOn(err)
+		// verify that we can fetch it back
+		resp, err := client.Get(url + "/schema")
+		panicOn(err)
+		body, err := ioutil.ReadAll(resp.Body)
+		panicOn(err)
+		sbody := string(body)
+		if sbody[:62] != schemaJson[:62] {
+			fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson)
+			panic("unexpected schema back")
+		}
+	})
+	// add a set field "color"
+	//POST localhost:10101/index/scratch/field/color
+	t.Run("Create Non-Keyed SetField", func(t *testing.T) {
+		resp, err := client.Post(url+"/index/scratch/field/rating", "application/text", bytes.NewBuffer(nil))
+		panicOn(err)
+		if resp.StatusCode != 200 {
+			panic(fmt.Sprintf("expected 200 status, got '%v'", resp))
+		}
+	})
+	t.Run("Create Keyed SetField", func(t *testing.T) {
+		resp, err := client.Post(url+"/index/scratch/field/color", "application/text", bytes.NewBuffer([]byte(`{"options": {"keys": true}}`)))
+		panicOn(err)
+		if resp.StatusCode != 200 {
+			panic(fmt.Sprintf("expected 200 status, got '%v'", resp))
+		}
+	})
+	t.Run("Create Time Field", func(t *testing.T) {
+		resp, err := client.Post(url+"/index/scratch/field/event", "application/text", bytes.NewBuffer([]byte(`{"options": { "type": "time", "timeQuantum": "YMDH" }}`)))
+		panicOn(err)
+		if resp.StatusCode != 200 {
+			panic(fmt.Sprintf("expected 200 status, got '%v'", resp))
+		}
+	})
+
+	// set some bits
+
+	// set some bits
+
+	t.Run("Set", func(t *testing.T) {
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`))
+		panicOn(err)
+		// test for dups
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(0, color=red)`))
+		panicOn(err)
+		//
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=red)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=red)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=red)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=red)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=red)`))
+		panicOn(err)
+
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, color=green)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, color=green)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=green)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=green)`))
+		panicOn(err)
+
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, color=yellow)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, color=yellow)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=yellow)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=yellow)`))
+		panicOn(err)
+
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, color=orange)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, color=orange)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, color=orange)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, color=orange)`))
+		panicOn(err)
+
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(1, luminosity=1)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, luminosity=2)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, luminosity=4)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, luminosity=5)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, luminosity=4)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, luminosity=3)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, luminosity=2)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(8, luminosity=1)`))
+		panicOn(err)
+
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(7, rating=1)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(10, rating=1)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(6, rating=2)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, rating=2)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, rating=3)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, rating=4)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, rating=4)`))
+		panicOn(err)
+
+		//time fields
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T01:00)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(5, event=1,2021-02-05T02:00)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(4, event=1,2021-02-05T02:05)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(3, event=1,2021-02-05T03:00)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Set(2, event=1,2021-02-05T04:00)`))
+		panicOn(err)
+	})
+	t.Run("Clear", func(t *testing.T) {
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Clear(1, color=purple)`))
+		panicOn(err)
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`ClearRow(color=purple)`))
+		panicOn(err)
+	})
+	t.Run("Store", func(t *testing.T) {
+		_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(`Store(Row(color=red), color=purple)`))
+		panicOn(err)
+	})
+	// verify that we can fetch it back
+	var schema pilosa.Schema
+	t.Run("Fetch Schema", func(t *testing.T) {
+		resp, err := client.Get(url + "/schema")
+		panicOn(err)
+		body, err := ioutil.ReadAll(resp.Body)
+		panicOn(err)
+		err = json.Unmarshal(body, &schema)
+		if err != nil {
+			panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err))
+		}
+	})
+	t.Run("Import Roaring", func(t *testing.T) {
+		//sets 12 bits in shard 0
+		roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100")
+		idx := schema.Indexes[0]
+		var fld *pilosa.FieldInfo
+		for _, f := range idx.Fields {
+			if f.Name == "rating" {
+				fld = f
+			}
+		}
+		if fld == nil {
+			panic("field color not found")
+		}
+		msg := pilosa.ImportRoaringRequest{
+			IndexCreatedAt: idx.CreatedAt,
+			FieldCreatedAt: fld.CreatedAt,
+			Clear:          false,
+			Views: map[string][]byte{
+				"": roaringData,
+			},
+			UpdateExistence: true,
+		}
+		ser := proto.Serializer{}
+		data, err := ser.Marshal(&msg)
+		if err != nil {
+			t.Fatal(err)
+		}
+		furl := url + "/index/scratch/field/rating/import-roaring/0"
+		req, err := http.NewRequest("POST", furl, bytes.NewBuffer(data))
+		panicOn(err)
+		req.Header.Set("Content-Type", "application/x-protobuf")
+		req.Header.Set("Accept", "application/x-protobuf")
+
+		// maybe need to Accept: "application/x-protobuf"
+		resp, err := client.Do(req)
+		panicOn(err)
+		_ = resp
+	})
+	t.Run("Read Query", func(t *testing.T) {
+		for _, tst := range GetTests() {
+			_, err = client.Post(url+"/index/scratch/query", "application/text", bytes.NewBufferString(tst.Pql))
+			panicOn(err)
+		}
+	})
+}
+
+func makeImportRequest(field *FieldInfo2, shard uint64, rows, columns []uint64, clear bool) (path string, data []byte, err error) {
+	msg := &pbuf.ImportRequest{
+		Index:          field.GetIndexName(),
+		IndexCreatedAt: field.GetIndexCreatedAt(),
+		Field:          field.GetName(),
+		FieldCreatedAt: field.GetCreatedAt(),
+		Shard:          shard,
+		RowIDs:         rows,
+		ColumnIDs:      columns,
+	}
+	data, err = pb.Marshal(msg)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "marshaling Import to protobuf")
+	}
+	path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear))
+	return path, data, nil
+}
+func makeImportKeyRequest(field *FieldInfo2, shard uint64, rows []string, columns []uint64, clear bool) (path string, data []byte, err error) {
+	msg := &pbuf.ImportRequest{
+		Index:          field.GetIndexName(),
+		IndexCreatedAt: field.GetIndexCreatedAt(),
+		Field:          field.GetName(),
+		FieldCreatedAt: field.GetCreatedAt(),
+		Shard:          shard,
+		RowKeys:        rows,
+		ColumnIDs:      columns,
+	}
+	data, err = pb.Marshal(msg)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "marshaling Import to protobuf")
+	}
+	path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear))
+	return path, data, nil
+}
+
+func makeImportValueRequest(field *FieldInfo2, shard uint64, values []int64, columns []uint64, clear bool) (path string, data []byte, err error) {
+	msg := &pbuf.ImportValueRequest{
+		Index:          field.GetIndexName(),
+		IndexCreatedAt: field.GetIndexCreatedAt(),
+		Field:          field.GetName(),
+		FieldCreatedAt: field.GetCreatedAt(),
+		Shard:          shard,
+		Values:         values,
+		ColumnIDs:      columns,
+	}
+	data, err = pb.Marshal(msg)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "marshaling Import to protobuf")
+	}
+	path = fmt.Sprintf("/index/%s/field/%s/import?clear=%s&ignoreKeyCheck=true", field.GetIndexName(), field.GetName(), strconv.FormatBool(clear))
+	return path, data, nil
+}
+func TestSauronImport(t *testing.T) {
+
+	cfg := NewSauronTestConfig(t)
+	proxy := NewSauron(cfg)
+	proxy.sql.MustRemove() //remove the existing db file if present
+	url, err := proxy.Start()
+	panicOn(err)
+	defer proxy.Stop()
+	schemaJson := `{
+	"indexes": [{
+		"name": "scratch",
+		"createdAt": 1611185870149882000,
+		"options": {
+			"keys": false,
+			"trackExistence": true
+		},
+		"fields": [{
+			"name": "bsi",
+			"createdAt": 1595896639332730413,
+			"options": {
+				"type": "int",
+				"base": 0,
+				"bitDepth": 31,
+				"min": -9223372036854775808,
+				"max": 9223372036854775807,
+				"keys": false,
+				"foreignIndex": ""
+			}
+		}, {
+			"name": "decimal",
+			"createdAt": 1613177295654228860,
+			"options": {
+				"type": "decimal",
+				"base": 0,
+				"scale": 1,
+				"bitDepth": 0,
+				"min": -922337203685477580.8,
+				"max": 922337203685477580.7,
+				"keys": false
+			}
+		},{
+			"name": "setkeyfield",
+			"createdAt": 1613345793598357200,
+			"options": {
+			  "type": "set",
+			  "cacheType": "ranked",
+			  "cacheSize": 50000,
+			  "keys": true
+			}
+		  },{
+			"name": "setfield",
+			"createdAt": 1613345793598357200,
+			"options": {
+			  "type": "set",
+			  "cacheType": "ranked",
+			  "cacheSize": 50000,
+			  "keys": false
+			}
+		  }]
+	}]
+}`
+	client := &http.Client{}
+	_, err = client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
+	panicOn(err)
+
+	t.Run("basic row/column", func(t *testing.T) {
+		f, err := proxy.sql.GetField("scratch", "setfield")
+		panicOn(err)
+		clear := false
+		cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3}
+		rows := []uint64{0, 0, 0, 1, 1, 1, 2, 2, 2}
+		shard := uint64(0)
+		path, payload, err := makeImportRequest(f, shard, rows, cols, clear)
+		_, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload))
+		panicOn(err)
+	})
+	t.Run("basic rowkey/column", func(t *testing.T) {
+		f, err := proxy.sql.GetField("scratch", "setkeyfield")
+		panicOn(err)
+		clear := false
+		cols := []uint64{0, 1, 2, 2, 3, 4, 1, 2, 3}
+		rows := []string{"red", "red", "red", "blue", "blue", "blue", "green", "green", "green"}
+		shard := uint64(0)
+		path, payload, err := makeImportKeyRequest(f, shard, rows, cols, clear)
+		_, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload))
+		panicOn(err)
+	})
+	t.Run("basic import values", func(t *testing.T) {
+		f, err := proxy.sql.GetField("scratch", "bsi")
+		panicOn(err)
+		clear := false
+		cols := []uint64{0, 1, 2, 3, 4}
+		values := []int64{40, 30, 20, 10, 5}
+		shard := uint64(0)
+		path, payload, err := makeImportValueRequest(f, shard, values, cols, clear)
+		_, err = client.Post(url+path, "application/json", bytes.NewBuffer(payload))
+		panicOn(err)
+	})
+
+}

cmd/sauron/schema_test.go

@@ -0,0 +1,353 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2"
+)
+
+func NewSauronTestConfig(t *testing.T) *SauronConfig {
+
+	bind := fmt.Sprintf("127.0.0.1:%d", GetAvailPort())
+	bindGRPC := fmt.Sprintf("127.0.0.1:%d", GetAvailPort())
+	gossipPort := fmt.Sprintf("%d", GetAvailPort())
+	dataDir, err := ioutil.TempDir("", "pilosa-sauron-target-*")
+	panicOn(err)
+
+	return &SauronConfig{
+		DatabaseName: strings.ToLower(t.Name()),
+		Bind:         bind,
+		BindGRPC:     bindGRPC,
+		GossipPort:   gossipPort,
+		DataDir:      dataDir,
+	}
+}
+
+func TestProxyCanHandlePostSchema(t *testing.T) {
+
+	cfg := NewSauronTestConfig(t)
+	proxy := NewSauron(cfg)
+	proxy.sql.MustRemove() //start with
+	url, err := proxy.Start()
+	panicOn(err)
+	defer proxy.Stop()
+
+	// prep for test by cleaning up leftovers from any old run.
+	//panicOn(proxy.sql.DropTablesWithPrefix(t.Name()))
+
+	//schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } }, { "name ": "color", "options ": {} } ] }] }`
+	/*
+		schemaJson := `{ "indexes": [{ "name": "simple", "options": { "trackExistence": true }, "fields": [{ "name": "luminosity", "options": { "type": "int" } } ] }] }`
+
+
+		vv("Posting err = '%v'", schemaJson)
+		resp, err := client.Post(url+"/schema", "application/json", bytes.NewBufferString(schemaJson))
+		if resp.StatusCode != http.StatusNoContent {
+			panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode))
+		}
+		// verify that we can fetch it back
+		resp2, err := client.Get(url + "/schema")
+		panicOn(err)
+		body, err := ioutil.ReadAll(resp2.Body)
+		panicOn(err)
+		sbody := string(body)
+		vv("body = '%v'", sbody)
+		if sbody[:62] != schemaJson[:62] {
+			fmt.Printf("did not get posted schema back: observed:\n\n%v\n\nexpected:\n\n%v\n\n", sbody, schemaJson)
+			panic("unxpected schema back")
+		}
+	*/
+	// load large schema too
+	client := &http.Client{}
+	schm, err := ioutil.ReadFile("sample_schema.json")
+	panicOn(err)
+
+	resp, err := client.Post(url+"/schema", "application/json", bytes.NewBuffer(schm))
+	panicOn(err)
+	if resp.StatusCode != http.StatusNoContent {
+		panic(fmt.Sprintf("expecting %v got %v", http.StatusNoContent, resp.StatusCode))
+	}
+
+	// verify all tables expected are present
+	obs := make(map[string]bool)
+	for i, table := range proxy.sql.ListIndexFields() {
+		_ = i
+		obs[table] = true
+		if !expectedTables[table] {
+			panic(fmt.Sprintf("observed table '%v' but was not expected", table))
+		}
+	}
+	for table := range expectedTables {
+		if !obs[table] {
+			panic(fmt.Sprintf("expected table '%v' but was not observed", table))
+		}
+	}
+
+}
+
+func TestProxyCanCreateIndexFieldViaPost(t *testing.T) {
+
+	cfg := NewSauronTestConfig(t)
+	proxy := NewSauron(cfg)
+	proxy.sql.MustRemove() //start with
+	url, err := proxy.Start()
+	panicOn(err)
+	defer proxy.Stop()
+	client := &http.Client{}
+	//create non-keyed or "simple" index
+	resp, err := client.Post(url+"/index/simple", "application/json", bytes.NewBuffer(nil))
+	panicOn(err)
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode))
+	}
+
+	fields := []struct {
+		name    string
+		options []byte
+	}{
+		{name: "set", options: []byte{}},
+		{name: "keyset", options: []byte{}},
+		{name: "bsi", options: []byte{}},
+	}
+	for _, field := range fields {
+		//create set field
+		resp, err = client.Post(fmt.Sprintf("%v/index/simple/field/%v", url, field.name), "application/json", bytes.NewBuffer(field.options))
+		panicOn(err)
+		if resp.StatusCode != http.StatusOK {
+			panic(fmt.Sprintf("expecting %v got %v", http.StatusOK, resp.StatusCode))
+		}
+	}
+	// verify that we can fetch it back
+	resp2, err := client.Get(url + "/schema")
+	panicOn(err)
+	body, err := ioutil.ReadAll(resp2.Body)
+	panicOn(err)
+	var schema pilosa.Schema
+	err = json.Unmarshal(body, &schema)
+	if err != nil {
+		panic(fmt.Sprintf("decoding request as JSON Pilosa schema: %v", err))
+	}
+
+	expected := map[string]map[string]bool{
+		"simple": {"bsi": true, "set": true, "keyset": true},
+	}
+	for _, i := range schema.Indexes {
+		ll, ok := expected[i.Name]
+		if !ok {
+			panic(fmt.Sprintf("expected index not present: %v", i.Name))
+		}
+		for _, f := range i.Fields {
+			_, ok := ll[f.Name]
+			if !ok {
+				panic(fmt.Sprintf("expected field '%v' not present in index'%v'", f.Name, i.Name))
+			}
+		}
+	}
+}
+
+var expectedTables = map[string]bool{
+	"trait_store/aba":                                                                true,
+	"trait_store/ach_batch_recency":                                                  true,
+	"trait_store/ach_pass_thru_recency":                                              true,
+	"trait_store/ach_payment_recency":                                                true,
+	"trait_store/bools":                                                              true,
+	"trait_store/bools-exists":                                                       true,
+	"trait_store/central_group":                                                      true,
+	"trait_store/custom_audiences":                                                   true,
+	"trait_store/days_since_last_logon":                                              true,
+	"trait_store/db":                                                                 true,
+	"trait_store/desktop_recency":                                                    true,
+	"trait_store/domestic_wire_recency":                                              true,
+	"trait_store/external_transfer_recency":                                          true,
+	"trait_store/fields":                                                             true,
+	"trait_store/funds_transfer_recency":                                             true,
+	"trait_store/international_wire_recency":                                         true,
+	"trait_store/mobile_remote_deposit_recency":                                      true,
+	"trait_store/payroll_recency":                                                    true,
+	"trait_store/pfm_category_total_current_balance__401k_investment":                true,
+	"trait_store/pfm_category_total_current_balance__403b_investment":                true,
+	"trait_store/pfm_category_total_current_balance__529_investment":                 true,
+	"trait_store/pfm_category_total_current_balance__any":                            true,
+	"trait_store/pfm_category_total_current_balance__auto_loan":                      true,
+	"trait_store/pfm_category_total_current_balance__brokerage_account":              true,
+	"trait_store/pfm_category_total_current_balance__certificate_of_deposit":         true,
+	"trait_store/pfm_category_total_current_balance__checking":                       true,
+	"trait_store/pfm_category_total_current_balance__credit_card":                    true,
+	"trait_store/pfm_category_total_current_balance__home_equity_loan":               true,
+	"trait_store/pfm_category_total_current_balance__ira_investment":                 true,
+	"trait_store/pfm_category_total_current_balance__line_of_credit":                 true,
+	"trait_store/pfm_category_total_current_balance__loan":                           true,
+	"trait_store/pfm_category_total_current_balance__money_market":                   true,
+	"trait_store/pfm_category_total_current_balance__mortgage":                       true,
+	"trait_store/pfm_category_total_current_balance__personal_loan":                  true,
+	"trait_store/pfm_category_total_current_balance__roth_ira_investment":            true,
+	"trait_store/pfm_category_total_current_balance__savings":                        true,
+	"trait_store/pfm_category_total_current_balance__simple_ira":                     true,
+	"trait_store/pfm_category_total_current_balance__student_loan":                   true,
+	"trait_store/pfm_category_total_current_balance__taxable_investment":             true,
+	"trait_store/phone_recency":                                                      true,
+	"trait_store/product_count__brokerage_account":                                   true,
+	"trait_store/product_count__commercial_cd_or_share_certificate":                  true,
+	"trait_store/product_count__commercial_checking_or_share_draft":                  true,
+	"trait_store/product_count__commercial_credit_card":                              true,
+	"trait_store/product_count__commercial_indirect_auto_loan":                       true,
+	"trait_store/product_count__commercial_line_of_credit":                           true,
+	"trait_store/product_count__commercial_loan":                                     true,
+	"trait_store/product_count__commercial_money_market":                             true,
+	"trait_store/product_count__commercial_new_auto_loan":                            true,
+	"trait_store/product_count__commercial_real_estate_loan":                         true,
+	"trait_store/product_count__commercial_savings_or_share":                         true,
+	"trait_store/product_count__commercial_secured_credit_card":                      true,
+	"trait_store/product_count__commercial_secured_loan":                             true,
+	"trait_store/product_count__commercial_special_checking_or_share_draft":          true,
+	"trait_store/product_count__commercial_special_credit_card":                      true,
+	"trait_store/product_count__commercial_special_savings_or_share":                 true,
+	"trait_store/product_count__commercial_special_vehicle_loan":                     true,
+	"trait_store/product_count__commercial_used_auto_loan":                           true,
+	"trait_store/product_count__consumer_basic_checking_or_share_draft":              true,
+	"trait_store/product_count__consumer_cd_or_share_certificate":                    true,
+	"trait_store/product_count__consumer_credit_card":                                true,
+	"trait_store/product_count__consumer_indirect_auto_loan":                         true,
+	"trait_store/product_count__consumer_interest_bearing_checking_or_share_draft":   true,
+	"trait_store/product_count__consumer_line_of_credit":                             true,
+	"trait_store/product_count__consumer_loan":                                       true,
+	"trait_store/product_count__consumer_money_market":                               true,
+	"trait_store/product_count__consumer_new_auto_loan":                              true,
+	"trait_store/product_count__consumer_premium_checking_or_share_draft":            true,
+	"trait_store/product_count__consumer_premium_savings_or_share":                   true,
+	"trait_store/product_count__consumer_real_estate_loan":                           true,
+	"trait_store/product_count__consumer_restricted_checking_or_share_draft":         true,
+	"trait_store/product_count__consumer_restricted_savings_or_share":                true,
+	"trait_store/product_count__consumer_savings_or_share":                           true,
+	"trait_store/product_count__consumer_secured_credit_card":                        true,
+	"trait_store/product_count__consumer_secured_loan":                               true,
+	"trait_store/product_count__consumer_special_checking_or_share_draft":            true,
+	"trait_store/product_count__consumer_special_credit_card":                        true,
+	"trait_store/product_count__consumer_special_savings_or_share":                   true,
+	"trait_store/product_count__consumer_special_vehicle_loan":                       true,
+	"trait_store/product_count__consumer_used_auto_loan":                             true,
+	"trait_store/product_count__debit_card":                                          true,
+	"trait_store/product_count__education_savings_or_share":                          true,
+	"trait_store/product_count__escrow_account":                                      true,
+	"trait_store/product_count__health_savings_account":                              true,
+	"trait_store/product_count__home_equity_line_of_credit":                          true,
+	"trait_store/product_count__home_equity_loan":                                    true,
+	"trait_store/product_count__insurance_product":                                   true,
+	"trait_store/product_count__interest_only_legal_trust_account":                   true,
+	"trait_store/product_count__membership_share":                                    true,
+	"trait_store/product_count__other_service":                                       true,
+	"trait_store/product_count__plan_401k":                                           true,
+	"trait_store/product_count__plan_403b":                                           true,
+	"trait_store/product_count__plan_529":                                            true,
+	"trait_store/product_count__roth_ira":                                            true,
+	"trait_store/product_count__roth_ira_cd_or_share_certificate":                    true,
+	"trait_store/product_count__small_business_checking_or_share_draft":              true,
+	"trait_store/product_count__special_indirect_vehicle_loan":                       true,
+	"trait_store/product_count__student_education_loan":                              true,
+	"trait_store/product_count__traditional_ira":                                     true,
+	"trait_store/product_count__traditional_ira_cd_or_share_certificate":             true,
+	"trait_store/product_count__trust":                                               true,
+	"trait_store/product_count__vacation_club_holiday_savings":                       true,
+	"trait_store/product_recency__brokerage_account":                                 true,
+	"trait_store/product_recency__commercial_cd_or_share_certificate":                true,
+	"trait_store/product_recency__commercial_checking_or_share_draft":                true,
+	"trait_store/product_recency__commercial_credit_card":                            true,
+	"trait_store/product_recency__commercial_indirect_auto_loan":                     true,
+	"trait_store/product_recency__commercial_line_of_credit":                         true,
+	"trait_store/product_recency__commercial_loan":                                   true,
+	"trait_store/product_recency__commercial_money_market":                           true,
+	"trait_store/product_recency__commercial_new_auto_loan":                          true,
+	"trait_store/product_recency__commercial_real_estate_loan":                       true,
+	"trait_store/product_recency__commercial_savings_or_share":                       true,
+	"trait_store/product_recency__commercial_secured_credit_card":                    true,
+	"trait_store/product_recency__commercial_secured_loan":                           true,
+	"trait_store/product_recency__commercial_special_checking_or_share_draft":        true,
+	"trait_store/product_recency__commercial_special_credit_card":                    true,
+	"trait_store/product_recency__commercial_special_savings_or_share":               true,
+	"trait_store/product_recency__commercial_special_vehicle_loan":                   true,
+	"trait_store/product_recency__commercial_used_auto_loan":                         true,
+	"trait_store/product_recency__consumer_basic_checking_or_share_draft":            true,
+	"trait_store/product_recency__consumer_cd_or_share_certificate":                  true,
+	"trait_store/product_recency__consumer_credit_card":                              true,
+	"trait_store/product_recency__consumer_indirect_auto_loan":                       true,
+	"trait_store/product_recency__consumer_interest_bearing_checking_or_share_draft": true,
+	"trait_store/product_recency__consumer_line_of_credit":                           true,
+	"trait_store/product_recency__consumer_loan":                                     true,
+	"trait_store/product_recency__consumer_money_market":                             true,
+	"trait_store/product_recency__consumer_new_auto_loan":                            true,
+	"trait_store/product_recency__consumer_premium_checking_or_share_draft":          true,
+	"trait_store/product_recency__consumer_premium_savings_or_share":                 true,
+	"trait_store/product_recency__consumer_real_estate_loan":                         true,
+	"trait_store/product_recency__consumer_restricted_checking_or_share_draft":       true,
+	"trait_store/product_recency__consumer_restricted_savings_or_share":              true,
+	"trait_store/product_recency__consumer_savings_or_share":                         true,
+	"trait_store/product_recency__consumer_secured_credit_card":                      true,
+	"trait_store/product_recency__consumer_secured_loan":                             true,
+	"trait_store/product_recency__consumer_special_checking_or_share_draft":          true,
+	"trait_store/product_recency__consumer_special_credit_card":                      true,
+	"trait_store/product_recency__consumer_special_savings_or_share":                 true,
+	"trait_store/product_recency__consumer_special_vehicle_loan":                     true,
+	"trait_store/product_recency__consumer_used_auto_loan":                           true,
+	"trait_store/product_recency__debit_card":                                        true,
+	"trait_store/product_recency__education_savings_or_share":                        true,
+	"trait_store/product_recency__escrow_account":                                    true,
+	"trait_store/product_recency__health_savings_account":                            true,
+	"trait_store/product_recency__home_equity_line_of_credit":                        true,
+	"trait_store/product_recency__home_equity_loan":                                  true,
+	"trait_store/product_recency__insurance_product":                                 true,
+	"trait_store/product_recency__interest_only_legal_trust_account":                 true,
+	"trait_store/product_recency__membership_share":                                  true,
+	"trait_store/product_recency__other_service":                                     true,
+	"trait_store/product_recency__plan_401k":                                         true,
+	"trait_store/product_recency__plan_403b":                                         true,
+	"trait_store/product_recency__plan_529":                                          true,
+	"trait_store/product_recency__roth_ira":                                          true,
+	"trait_store/product_recency__roth_ira_cd_or_share_certificate":                  true,
+	"trait_store/product_recency__small_business_checking_or_share_draft":            true,
+	"trait_store/product_recency__special_indirect_vehicle_loan":                     true,
+	"trait_store/product_recency__student_education_loan":                            true,
+	"trait_store/product_recency__traditional_ira":                                   true,
+	"trait_store/product_recency__traditional_ira_cd_or_share_certificate":           true,
+	"trait_store/product_recency__trust":                                             true,
+	"trait_store/product_recency__vacation_club_holiday_savings":                     true,
+	"trait_store/stop_payment_recency":                                               true,
+	"trait_store/survey_2053c210_adfc_4e14_afd3_9abc68a70719":                        true,
+	"trait_store/survey_2679bd7e_cdb8_4c24_a1da_e904799382eb":                        true,
+	"trait_store/survey_2f336120_7130_492b_b116_d93d560baa0a":                        true,
+	"trait_store/survey_36e229ff_c0a6_48ad_928e_19b5abf69f11":                        true,
+	"trait_store/survey_3773950d_a3a6_419a_b9e8_7ed9e33f93aa":                        true,
+	"trait_store/survey_38aeaf3f_35ab_4df5_b213_5bd7dc29b796":                        true,
+	"trait_store/survey_587ae933_b8e3_49a2_a23d_989df9ad119f":                        true,
+	"trait_store/survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab":                        true,
+	"trait_store/survey_7bc3b9f0_fb8c_4a79_9a22_14be4ed71949":                        true,
+	"trait_store/survey_9cebb459_162e_4518_88f6_bc1cdd630cd0":                        true,
+	"trait_store/survey_a05de6d3_6620_4eb2_b9a2_511936680b57":                        true,
+	"trait_store/survey_b570363c_ba94_4a7a_9891_141a1befc42b":                        true,
+	"trait_store/survey_c358d6e0_4116_4230_84ed_869b315e89a1":                        true,
+	"trait_store/survey_cce773ce_14e8_4fae_90cd_213a821a3bc0":                        true,
+	"trait_store/survey_e49df858_4903_40ac_bd9f_96f764940f2c":                        true,
+	"trait_store/survey_eab7aa49_26e0_4bef_a25f_ddabc9ee2bd9":                        true,
+	"trait_store/tablet_recency":                                                     true,
+	"trait_store/user_id":                                                            true,
+	"trait_store/zip_code":                                                           true,
+}

cmd/sauron/sqlgen_test.go

@@ -0,0 +1,260 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/shurcooL/go-goon"
+)
+
+/*
+sqlite db this test validated this against
+create table bits (field ,row , column , timestamp);
+           create table λbsi (field string,column bigint , val bigint);
+          CREATE VIEW columns as
+            select distinct column from λbits
+            UNION
+					  select distinct column from λbsi;
+
+	         INSERT INTO λbits VALUES( 'color','red',1);
+          INSERT INTO λbits VALUES( 'color','red',2);
+          INSERT INTO λbits VALUES( 'color','red',3);
+          INSERT INTO λbits VALUES( 'color','red',4);
+          INSERT INTO λbits VALUES( 'color','red',5);
+          INSERT INTO λbits VALUES( 'color','green',2);
+          INSERT INTO λbits VALUES( 'color','green',4);
+          INSERT INTO λbits VALUES( 'color','green',6);
+          INSERT INTO λbits VALUES( 'color','green',8);
+         INSERT INTO λbits VALUES( 'color','yellow',1);
+          INSERT INTO λbits VALUES( 'color','yellow',3);
+          INSERT INTO λbits VALUES( 'color','yellow',5);
+          INSERT INTO λbits VALUES( 'color','yellow',7);
+          INSERT INTO λbits VALUES( 'color','orange',5);
+          INSERT INTO λbits VALUES( 'color','orange',6);
+          INSERT INTO λbits VALUES( 'color','orange',7);
+          INSERT INTO λbits VALUES( 'color','orange',8);
+
+          INSERT INTO λbits VALUES( 'rating',1,8);
+          INSERT INTO λbits VALUES( 'rating',1,7);
+          INSERT INTO λbits VALUES( 'rating',1,10);
+          INSERT INTO λbits VALUES( 'rating',2,6);
+          INSERT INTO λbits VALUES( 'rating',2,5);
+          INSERT INTO λbits VALUES( 'rating',3,4);
+          INSERT INTO λbits VALUES( 'rating',4,3);
+          INSERT INTO λbits VALUES( 'rating',4,2);
+
+          INSERT INTO λbsi VALUES( 'luminosity',1, 1);
+          INSERT INTO λbsi VALUES( 'luminosity',2, 2);
+          INSERT INTO λbsi VALUES( 'luminosity',3, 4);
+          INSERT INTO λbsi VALUES( 'luminosity',4, 5);
+          INSERT INTO λbsi VALUES( 'luminosity',5, 4);
+          INSERT INTO λbsi VALUES( 'luminosity',6, 3);
+          INSERT INTO λbsi VALUES( 'luminosity',7, 2);
+          INSERT INTO λbsi VALUES( 'luminosity',8, 1);
+*/
+func GetTests() []struct {
+	Pql string
+	Sql string
+} {
+	return []struct {
+		Pql string
+		Sql string
+	}{
+		{
+			Pql: "Row(rating=0)",
+			Sql: `select distinct column from λbits where field="rating" AND row="0"`,
+		},
+		{
+			Pql: "Row(color=red)",
+			Sql: `select distinct column from λbits where field="color" AND row="red"`,
+		},
+		{
+			Pql: "Row(luminosity>2)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val>2`,
+		},
+		{
+			Pql: "Row(luminosity<2)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val<2`,
+		},
+		{
+			Pql: "Row(luminosity==4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val=4`,
+		},
+		{
+			Pql: "Row(luminosity<=4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val<=4`,
+		},
+		{
+			Pql: "Row(luminosity>=4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val>=4`,
+		},
+		{
+			Pql: "Row(luminosity!=4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val!=4`,
+		},
+		{
+			Pql: "Row(2<=luminosity<=4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val>=2 and val<=4`,
+		},
+		{
+			Pql: "Row(2<luminosity<=4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val>2 and val<=4`,
+		},
+		{
+			Pql: "Row(2<luminosity<4)",
+			Sql: `select distinct column from λbsi where field="luminosity" and val>2 and val<4`,
+		},
+		{
+			Pql: `Row(event=1, from='2021-02-01T00:00', to='2021-02-05T03:00')`,
+			Sql: `select distinct column from λbits where field="event" AND row="1" AND timestamp >= "2021-02-01 00:00" AND timestamp < "2021-02-05 03:00"`,
+		},
+		{
+			Pql: "Count(row(color=red))",
+			Sql: `select count(*) from( select distinct column from λbits where field="color" AND row="red" )`,
+		},
+		{
+			Pql: `Intersect(Row(color=red),Row(color=green))`,
+			Sql: `select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`,
+		},
+		{
+			Pql: "Count(Union(Row(color=red),Row(color=green)))",
+			Sql: `select count(*) from( select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") )`,
+		},
+		{
+			Pql: "Union(Row(color=red),Row(color=green))",
+			Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green")`,
+		},
+		{
+			Pql: "Difference(Row(color=red),Row(color=green))",
+			Sql: `select column from(select distinct column from λbits where field="color" AND row="red" except select distinct column from λbits where field="color" AND row="green")`,
+		},
+		{
+			Pql: `All()`,
+			Sql: `select column from λcolumns`,
+		},
+		{
+			Pql: "Not(Row(color=red))",
+			Sql: `select column from λcolumns except select distinct column from λbits where field="color" AND row="red"`,
+		},
+		{
+			Pql: "Xor(Row(color=red),Row(color=green))",
+			Sql: `select column from(select distinct column from λbits where field="color" AND row="red" union select distinct column from λbits where field="color" AND row="green") except select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green")`,
+		},
+		{
+			Pql: "Distinct(field=luminosity)",
+			Sql: `select distinct val from λbsi where field="luminosity"`,
+		},
+		{
+			Pql: "Distinct(Row(color=red),field=luminosity)",
+			Sql: `select distinct val from λbsi where field="luminosity" AND column in (select distinct column from λbits where field="color" AND row="red")`,
+		},
+		{
+			Pql: "Distinct(Intersect(Row(color=red),Row(color=green)),field=luminosity)",
+			Sql: `select distinct val from λbsi where field="luminosity" AND column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="green"))`,
+		},
+		{
+			Pql: "Rows(color)",
+			Sql: `select distinct field, row from λbits where field="color" order by row`,
+		},
+		{
+			Pql: "Rows(color, limit=10)",
+			Sql: `select distinct field, row from λbits where field="color" order by row limit 10`,
+		},
+		{
+			Pql: "Rows(color, column=1)",
+			Sql: `select distinct field, row from λbits where field="color" and column='1' order by row`,
+		},
+		//		{ // TODO (twg) Not supported by this test, can't reliably get key ordering
+		//		Pql: "Rows(color, previous=green )",
+		//	    Sql: `select distinct field, row from λbits where field="color" and row>'green' order by row`,
+		//		},
+		{
+			Pql: "GroupBy(Rows(color))",
+			Sql: `select  f1.field, f1.row,count(*) as cnt from λbits f1 where f1.field='color' group by f1.field,f1.row order by cnt desc`,
+		},
+		{
+			Pql: "GroupBy(Rows(color),Rows(rating))",
+			Sql: `select  f1.field, f1.row,f2.field, f2.row,count(*) as cnt from λbits f1 inner join λbits f2 on f1.column = f2.column where f1.field='color'and f2.field='rating' group by f1.field,f1.row,f2.field,f2.row order by cnt desc`,
+		},
+		{
+			Pql: "TopN(color)",
+			Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`,
+		},
+		{
+			Pql: "TopN(color,n=2)",
+			Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc limit 2`,
+		},
+		{
+			Pql: "TopN(color,Row(color=red),n=2)",
+			Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="color" AND row="red") group by row order by cnt desc limit 2`,
+		},
+		{
+			Pql: "Topk(color)",
+			Sql: `select row,count(*) as cnt from λbits where field="color" group by row order by cnt desc`,
+		},
+		{
+			Pql: "TopK(color, k=2, filter=Row(rating=1))",
+			Sql: `select row,count(*) as cnt from λbits where field="color" and column in (select distinct column from λbits where field="rating" AND row="1") group by row order by cnt desc limit 2`,
+		},
+		{
+			Pql: `Min(field="luminosity")`,
+			Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity")`,
+		},
+		{
+			Pql: `Max(field="luminosity")`,
+			Sql: `select val,count(*) from λbsi where val=(select max(val) from λbsi where field="luminosity")`,
+		},
+		{
+			Pql: `Min(Row(color=orange),field="luminosity")`,
+			Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select distinct column from λbits where field="color" AND row="orange")) and column in (select distinct column from λbits where field="color" AND row="orange")`,
+		},
+		{
+			Pql: `Min(Intersect(Row(color=red),Row(color=orange)),field="luminosity")`,
+			Sql: `select val,count(*) from λbsi where val=(select min(val) from λbsi where field="luminosity" and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))) and column in (select column from(select distinct column from λbits where field="color" AND row="red" intersect select distinct column from λbits where field="color" AND row="orange"))`,
+		},
+		{
+			Pql: `Sum(field="luminosity")`,
+			Sql: `select sum(val),count(*) from λbsi where field="luminosity"`,
+		},
+		{
+			Pql: `ConstRow(columns=[1,3,5])`,
+			Sql: `select column1 as column from (values (1),(3),(5))`,
+		},
+		{
+			Pql: `UnionRows(Rows(color))`,
+			Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`,
+		},
+		{
+			Pql: `UnionRows(Rows(color),Rows(rating))`,
+			Sql: `select distinct b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row`,
+		},
+		{
+			Pql: `IncludesColumn(Row(color=red), column=1)`,
+			Sql: `select count(*)>0 from (select column from(select distinct column from λbits where field="color" AND row="red") where column='1')`,
+		},
+		{
+			Pql: `Extract(Row(color=red),Rows(color),Rows(rating))`,
+			Sql: `select sq.field, sq.row, b.column from (select field,row from(select distinct field, row from λbits where field="color" order by row)union all select field,row from (select distinct field, row from λbits where field="rating" order by row)) sq inner join λbits b on sq.field = b.field and sq.row = b.row where b.column in (select distinct column from λbits where field="color" AND row="red")`,
+		},
+		{
+			Pql: `Extract(ConstRow(columns=[1,2,3]))`,
+			Sql: `select '' as field, '' as row,column from (select column1 as column from (values (1),(2),(3)))`,
+		},
+	}
+}
+func TestParse(t *testing.T) {
+
+	for _, tst := range GetTests() {
+
+		q, err := pql.NewParser(strings.NewReader(tst.Pql)).Parse()
+		panicOn(err)
+		sql, _, _, err := ToSql(q.Calls, "")
+		panicOn(err)
+		if sql != tst.Sql {
+			vv("\npql:%v\ngenerated:\n%v\nexpected:\n%v\n", tst.Pql, sql, tst.Sql)
+			goon.Dump(q)
+		}
+
+	}
+}

cmd/sauron/test_sqlite.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+db="poc2.db"
+sqlite3 ${db} <<EOF
+create table λbits (field ,row , column , timestamp);
+create table λbsi (field string,column bigint , val bigint);
+
+CREATE VIEW λcolumns as
+	select distinct column from λbits
+        UNION
+	select distinct column from λbsi;
+INSERT INTO λbsi VALUES( 'luminosity',1, 1);
+INSERT INTO λbsi VALUES( 'luminosity',2, 2);
+INSERT INTO λbsi VALUES( 'luminosity',3, 4);
+INSERT INTO λbsi VALUES( 'luminosity',4, 5);
+INSERT INTO λbsi VALUES( 'luminosity',5, 4);
+INSERT INTO λbsi VALUES( 'luminosity',6, 3);
+INSERT INTO λbsi VALUES( 'luminosity',7, 2);
+INSERT INTO λbsi VALUES( 'luminosity',8, 1);
+
+INSERT INTO λbits (field,row,column)VALUES( 'color','red',1);
+INSERT INTO λbits (field,row,column)VALUES( 'color','red',2);
+INSERT INTO λbits (field,row,column)VALUES( 'color','red',3);
+INSERT INTO λbits (field,row,column)VALUES( 'color','red',4);
+INSERT INTO λbits (field,row,column)VALUES( 'color','red',5);
+INSERT INTO λbits (field,row,column)VALUES( 'color','green',2);
+INSERT INTO λbits (field,row,column)VALUES( 'color','green',4);
+INSERT INTO λbits (field,row,column)VALUES( 'color','green',6);
+INSERT INTO λbits (field,row,column)VALUES( 'color','green',8);
+INSERT INTO λbits (field,row,column)VALUES( 'color','yellow',1);
+INSERT INTO λbits (field,row,column)VALUES( 'color','yellow',3);
+INSERT INTO λbits (field,row,column)VALUES( 'color','yellow',5);
+INSERT INTO λbits (field,row,column)VALUES( 'color','yellow',7);
+INSERT INTO λbits (field,row,column)VALUES( 'color','orange',5);
+INSERT INTO λbits (field,row,column)VALUES( 'color','orange',6);
+INSERT INTO λbits (field,row,column)VALUES( 'color','orange',7);
+INSERT INTO λbits (field,row,column)VALUES( 'color','orange',8);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','1',8);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','1',7);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','1',10);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','2',6);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','2',5);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','3',4);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','4',3);
+INSERT INTO λbits (field,row,column)VALUES( 'rating','4',2);
+
+INSERT INTO λbits (field,row,column,timestamp)VALUES( 'event','1',5,'2021-02-05 01:00');
+INSERT INTO λbits (field,row,column,timestamp)VALUES( 'event','1',5,'2021-02-05 02:00');
+INSERT INTO λbits (field,row,column,timestamp)VALUES( 'event','1',4,'2021-02-05 02:05');
+INSERT INTO λbits (field,row,column,timestamp)VALUES( 'event','1',3,'2021-02-05 03:00');
+INSERT INTO λbits (field,row,column,timestamp)VALUES( 'event','1',2,'2021-02-05 04:00');
+EOF

cmd/sauron/vprint.go

@@ -0,0 +1,167 @@
+// home: https://github.com/glycerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}