From d57dae3749973fc5c95a333f98cb2658e9c6e2d4 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 6 Sep 2018 11:32:50 -0500
Subject: [PATCH 01/11] implement NotNull field with index option trackNotNull

---
 encoding/proto/proto.go |   4 +-
 executor.go             |   7 ++
 executor_test.go        |  29 +++++++
 holder.go               |   4 +
 index.go                |  37 +++++++-
 internal/private.pb.go  | 182 ++++++++++++++++++++++++----------------
 internal/private.proto  |   1 +
 7 files changed, 190 insertions(+), 74 deletions(-)

diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index a19f1e548..c00e71b40 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -509,7 +509,8 @@ func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateInde
 
 func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta {
 	return &internal.IndexMeta{
-		Keys: m.Keys,
+		Keys:         m.Keys,
+		TrackNotNull: m.TrackNotNull,
 	}
 }
 
@@ -741,6 +742,7 @@ func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateI
 
 func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) {
 	m.Keys = pb.Keys
+	m.TrackNotNull = pb.TrackNotNull
 }
 
 func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) {
diff --git a/executor.go b/executor.go
index 2888b9270..1149f8c0e 100644
--- a/executor.go
+++ b/executor.go
@@ -1131,6 +1131,13 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
 		return false, ErrFieldNotFound
 	}
 
+	// Set column on not-null field.
+	if nnf := idx.unprotectedNotNullField(); nnf != nil {
+		if _, err := nnf.SetBit(0, colID, nil); err != nil {
+			return false, errors.Wrap(err, "setting not-null column")
+		}
+	}
+
 	if f.Type() == FieldTypeInt {
 		// Read remaining fields using labels.
 		rowVal, ok, err := c.IntArg(fieldName)
diff --git a/executor_test.go b/executor_test.go
index b6fa5ccfa..03ac09102 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -17,6 +17,7 @@ package pilosa_test
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"reflect"
 	"strconv"
 	"strings"
@@ -1495,3 +1496,31 @@ func TestExecutor_QueryCall(t *testing.T) {
 		}
 	})
 }
+
+func benchmarkNotNull(nn bool, b *testing.B) {
+	c := test.MustRunCluster(b, 1)
+	defer c.Close()
+	hldr := test.Holder{Holder: c[0].Server.Holder()}
+
+	indexName := "i"
+	fieldName := "f"
+
+	index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackNotNull: nn})
+	// Create field.
+	if _, err := index.CreateFieldIfNotExists(fieldName); err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		colID := uint64(rand.Intn(1 << 20))
+		rowID := uint64(rand.Intn(100000))
+		qry := fmt.Sprintf(`Set(%d, %s=%d)`, colID, fieldName, rowID)
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: qry}); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkExecutor_NotNull_True(b *testing.B)  { benchmarkNotNull(true, b) }
+func BenchmarkExecutor_NotNull_False(b *testing.B) { benchmarkNotNull(false, b) }
diff --git a/holder.go b/holder.go
index 957765511..252fca2db 100644
--- a/holder.go
+++ b/holder.go
@@ -38,6 +38,9 @@ const (
 
 	// fileLimit is the maximum open file limit (ulimit -n) to automatically set.
 	fileLimit = 262144 // (512^2)
+
+	// notNullFieldName is the name of the internal field used to store not-null values.
+	notNullFieldName = "notnull" // TODO: use a name less likely to collide?
 )
 
 // Holder represents a container for indexes.
@@ -354,6 +357,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	}
 
 	index.keys = opt.Keys
+	index.trackNotNull = opt.TrackNotNull
 
 	if err := index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
diff --git a/index.go b/index.go
index 0547d07d2..adbc6810e 100644
--- a/index.go
+++ b/index.go
@@ -36,6 +36,10 @@ type Index struct {
 	name string
 	keys bool // use string keys
 
+	// Not-null tracking.
+	trackNotNull bool
+	notNullField *Field
+
 	// Fields by name.
 	fields map[string]*Field
 
@@ -91,7 +95,10 @@ func (i *Index) Options() IndexOptions {
 }
 
 func (i *Index) options() IndexOptions {
-	return IndexOptions{Keys: i.keys}
+	return IndexOptions{
+		Keys:         i.keys,
+		TrackNotNull: i.trackNotNull,
+	}
 }
 
 // Open opens and initializes the index.
@@ -110,6 +117,12 @@ func (i *Index) Open() error {
 		return errors.Wrap(err, "opening fields")
 	}
 
+	if i.trackNotNull {
+		if err := i.openNotNullField(); err != nil {
+			return errors.Wrap(err, "opening not-null field")
+		}
+	}
+
 	if err := i.columnAttrs.Open(); err != nil {
 		return errors.Wrap(err, "opening attrstore")
 	}
@@ -147,6 +160,16 @@ func (i *Index) openFields() error {
 	return nil
 }
 
+// openNotNullField gets or creates the not-null field and associates it to the index.
+func (i *Index) openNotNullField() error {
+	f, err := i.createFieldIfNotExists(notNullFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
+	if err != nil {
+		return errors.Wrap(err, "creating not-null field")
+	}
+	i.notNullField = f
+	return nil
+}
+
 // loadMeta reads meta data for the index, if any.
 func (i *Index) loadMeta() error {
 	var pb internal.IndexMeta
@@ -165,6 +188,7 @@ func (i *Index) loadMeta() error {
 
 	// Copy metadata fields.
 	i.keys = pb.Keys
+	i.trackNotNull = pb.TrackNotNull
 
 	return nil
 }
@@ -173,7 +197,8 @@ func (i *Index) loadMeta() error {
 func (i *Index) saveMeta() error {
 	// Marshal metadata.
 	buf, err := proto.Marshal(&internal.IndexMeta{
-		Keys: i.keys,
+		Keys:         i.keys,
+		TrackNotNull: i.trackNotNull,
 	})
 	if err != nil {
 		return errors.Wrap(err, "marshalling")
@@ -250,6 +275,11 @@ func (i *Index) Fields() []*Field {
 	return a
 }
 
+// unprotectedNotNullField returns the internal field used to track not-null columns.
+func (i *Index) unprotectedNotNullField() *Field {
+	return i.notNullField
+}
+
 // recalculateCaches recalculates caches on every field in the index.
 func (i *Index) recalculateCaches() {
 	for _, field := range i.Fields() {
@@ -408,7 +438,8 @@ func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
 // IndexOptions represents options to set when initializing an index.
 type IndexOptions struct {
-	Keys bool `json:"keys"`
+	Keys         bool `json:"keys"`
+	TrackNotNull bool `json:"trackNotNull"`
 }
 
 // hasTime returns true if a contains a non-nil time.
diff --git a/internal/private.pb.go b/internal/private.pb.go
index 6247c58d4..2d1c0deef 100644
--- a/internal/private.pb.go
+++ b/internal/private.pb.go
@@ -62,7 +62,8 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type IndexMeta struct {
-	Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"`
+	Keys         bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"`
+	TrackNotNull bool `protobuf:"varint,4,opt,name=TrackNotNull,proto3" json:"TrackNotNull,omitempty"`
 }
 
 func (m *IndexMeta) Reset()                    { *m = IndexMeta{} }
@@ -77,6 +78,13 @@ func (m *IndexMeta) GetKeys() bool {
 	return false
 }
 
+func (m *IndexMeta) GetTrackNotNull() bool {
+	if m != nil {
+		return m.TrackNotNull
+	}
+	return false
+}
+
 type FieldOptions struct {
 	Type        string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"`
 	CacheType   string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
@@ -1051,6 +1059,16 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
+	if m.TrackNotNull {
+		dAtA[i] = 0x20
+		i++
+		if m.TrackNotNull {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i++
+	}
 	return i, nil
 }
 
@@ -2300,6 +2318,9 @@ func (m *IndexMeta) Size() (n int) {
 	if m.Keys {
 		n += 2
 	}
+	if m.TrackNotNull {
+		n += 2
+	}
 	return n
 }
 
@@ -2902,6 +2923,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.Keys = bool(v != 0)
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field TrackNotNull", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.TrackNotNull = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPrivate(dAtA[iNdEx:])
@@ -7143,73 +7184,74 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
 
 var fileDescriptorPrivate = []byte{
-	// 1077 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45,
-	0x14, 0x66, 0x7f, 0xec, 0xda, 0xc7, 0x75, 0x9a, 0x6c, 0x69, 0xd8, 0x22, 0x94, 0x9a, 0x51, 0xa5,
-	0x9a, 0x4a, 0x84, 0xaa, 0xbd, 0xe1, 0xaf, 0x52, 0x49, 0x1c, 0x60, 0x29, 0x09, 0x65, 0x36, 0xc9,
-	0x5d, 0x2f, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0xb3, 0xec, 0xce, 0x26, 0x71, 0x2f, 0xb8, 0x05,
-	0x89, 0x17, 0x40, 0x3c, 0x09, 0x8f, 0xc0, 0x25, 0x8f, 0x80, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xfb,
-	0x13, 0xc7, 0x21, 0x55, 0xe0, 0x6e, 0xce, 0x77, 0xce, 0x9c, 0xf3, 0xed, 0xf9, 0x9b, 0x85, 0x7e,
-	0x9a, 0x45, 0xc7, 0x4c, 0xf2, 0xf5, 0x34, 0x13, 0x52, 0x78, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0x58,
-	0x4c, 0xee, 0x41, 0x37, 0x48, 0x26, 0xfc, 0x74, 0x9b, 0x4b, 0xe6, 0x79, 0xe0, 0x3e, 0xe7, 0xb3,
-	0xdc, 0x77, 0x06, 0xd6, 0xb0, 0x43, 0xf1, 0x4c, 0x7e, 0xb7, 0xe0, 0xe6, 0x97, 0x11, 0x8f, 0x27,
-	0xdf, 0xa5, 0x32, 0x12, 0x49, 0xee, 0xbd, 0x07, 0xdd, 0x4d, 0x36, 0x3e, 0xe4, 0xbb, 0xb3, 0x94,
-	0xa3, 0x65, 0x97, 0xd6, 0x40, 0xa5, 0x0d, 0xa3, 0xd7, 0xdc, 0x77, 0x07, 0xd6, 0xb0, 0x4f, 0x6b,
-	0xc0, 0x1b, 0x40, 0x6f, 0x37, 0x9a, 0xf2, 0xef, 0x0b, 0x96, 0xc8, 0x62, 0xea, 0xb7, 0xf0, 0x76,
-	0x13, 0x52, 0x14, 0xd0, 0x71, 0x07, 0x55, 0x78, 0xf6, 0x96, 0xc1, 0xd9, 0x8e, 0x12, 0xbf, 0x3b,
-	0xb0, 0x86, 0x0e, 0x55, 0x47, 0x44, 0xd8, 0xa9, 0x0f, 0x06, 0x61, 0xa7, 0x15, 0xf5, 0x5e, 0x83,
-	0x3a, 0x81, 0xa5, 0x60, 0x9a, 0x8a, 0x4c, 0x52, 0x9e, 0xa7, 0x22, 0xc9, 0xd1, 0xd3, 0x56, 0x96,
-	0xf9, 0x16, 0x3a, 0x57, 0x47, 0xf2, 0x23, 0x2c, 0x6f, 0xc4, 0x62, 0x7c, 0x34, 0x62, 0x92, 0x51,
-	0xfe, 0x43, 0xc1, 0x73, 0xe9, 0xbd, 0x0d, 0x2d, 0xcc, 0x89, 0xb1, 0xd3, 0x82, 0x42, 0x31, 0x0f,
-	0xbe, 0xad, 0x51, 0x14, 0x14, 0x8a, 0xf7, 0x31, 0x13, 0x2e, 0xd5, 0x82, 0x42, 0xc3, 0x43, 0x96,
-	0x4d, 0x30, 0x03, 0x2e, 0xd5, 0x82, 0xe2, 0xb8, 0x1f, 0xf1, 0x13, 0xf3, 0xd9, 0x78, 0x26, 0x01,
-	0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x24, 0x18, 0xe5, 0xbe, 0x35, 0x70, 0x86,
-	0x2e, 0x35, 0x12, 0x26, 0x57, 0xc4, 0xc5, 0x34, 0x51, 0x2a, 0x1b, 0x55, 0x35, 0x40, 0xee, 0x42,
-	0x0b, 0x33, 0xad, 0xbe, 0xb2, 0xbe, 0xab, 0x8e, 0xe4, 0x27, 0x0b, 0xba, 0xdb, 0xec, 0x14, 0x69,
-	0xe4, 0xde, 0x53, 0xe8, 0x84, 0x92, 0x25, 0x13, 0x45, 0x50, 0x19, 0xf5, 0x1e, 0xbf, 0xbf, 0x5e,
-	0x36, 0xc4, 0x7a, 0x65, 0xb6, 0x5e, 0xda, 0x6c, 0x25, 0x32, 0x9b, 0xd1, 0xea, 0xca, 0xbb, 0x9f,
-	0x41, 0xff, 0x9c, 0x4a, 0xc5, 0x3b, 0xe2, 0xb3, 0x32, 0xab, 0x47, 0x7c, 0xa6, 0xbe, 0xff, 0x98,
-	0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xda, 0x1f, 0x5b, 0x64, 0x1f, 0xbc, 0xcd, 0x8c,
-	0x33, 0xc9, 0x31, 0xc8, 0x36, 0xcf, 0x73, 0xf6, 0x8a, 0x5f, 0x9e, 0x71, 0x9d, 0x45, 0xbb, 0x99,
-	0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x21, 0x78, 0x23, 0x1e, 0x73, 0xc9, 0x4d, 0x37, 0xff,
-	0x8b, 0x5f, 0x12, 0x96, 0x1c, 0xae, 0xb6, 0xf5, 0x1e, 0x80, 0xab, 0x46, 0x03, 0x29, 0xf4, 0x1e,
-	0xdf, 0xae, 0xf3, 0x54, 0x4d, 0x0d, 0x45, 0x03, 0x12, 0x97, 0x4e, 0x91, 0xcf, 0x95, 0x1f, 0xb6,
-	0xa0, 0x95, 0x1e, 0x9a, 0x50, 0x0e, 0x86, 0x5a, 0xad, 0x43, 0x35, 0xc7, 0xcf, 0x44, 0x7b, 0x56,
-	0x7e, 0xee, 0x75, 0xa3, 0x91, 0x97, 0x06, 0x55, 0x5d, 0xb9, 0xc3, 0xa6, 0xdc, 0xdc, 0xc1, 0x73,
-	0x45, 0xc5, 0xbe, 0x9a, 0x8a, 0x72, 0xaf, 0x3a, 0x59, 0x6d, 0x0d, 0x47, 0xb9, 0x47, 0x81, 0x3c,
-	0x81, 0x76, 0x38, 0x3e, 0xe4, 0x53, 0xe6, 0x7d, 0x00, 0x37, 0x90, 0x07, 0xcf, 0x4d, 0xb3, 0xdd,
-	0x9a, 0x4b, 0x22, 0x2d, 0xf5, 0x64, 0x64, 0xf8, 0x2f, 0xe4, 0xf4, 0x00, 0xda, 0x18, 0x3d, 0xf7,
-	0xdd, 0x79, 0x37, 0x88, 0x53, 0xa3, 0x26, 0x5b, 0xe0, 0xec, 0xd1, 0x40, 0x0d, 0x11, 0x32, 0x28,
-	0xbd, 0x18, 0x49, 0xf9, 0xfe, 0x5a, 0xe4, 0xd2, 0x64, 0x03, 0xcf, 0x0a, 0x7b, 0x21, 0x32, 0x89,
-	0xa9, 0xef, 0x53, 0x3c, 0x93, 0x97, 0xe0, 0xee, 0x88, 0x09, 0xf7, 0x96, 0xc0, 0x0e, 0x46, 0xc6,
-	0x87, 0x1d, 0x8c, 0xbc, 0x7b, 0xe8, 0xde, 0xa4, 0xa6, 0x5f, 0x93, 0xd8, 0xa3, 0x01, 0xc5, 0xc0,
-	0xf7, 0xa1, 0x1f, 0xe4, 0x9b, 0x42, 0x64, 0x93, 0x28, 0x61, 0x52, 0x64, 0x66, 0x9d, 0x9e, 0x07,
-	0xc9, 0x33, 0x58, 0x56, 0xee, 0x43, 0xc9, 0x24, 0x2f, 0xeb, 0xb7, 0x0a, 0x6d, 0x85, 0x55, 0xe1,
-	0x8c, 0x84, 0x83, 0xa0, 0xec, 0xca, 0x0a, 0xa2, 0x40, 0xbe, 0xd5, 0x1e, 0xb6, 0x8e, 0x79, 0x22,
-	0x1b, 0x1d, 0x80, 0x32, 0x3a, 0xe8, 0x53, 0x2d, 0x78, 0x44, 0x7f, 0x8a, 0xe1, 0xbc, 0x54, 0x73,
-	0x56, 0x28, 0x45, 0x1d, 0xf9, 0xc5, 0x02, 0x28, 0x09, 0x15, 0x79, 0x75, 0xc5, 0xba, 0xfc, 0x8a,
-	0x37, 0x2c, 0x6b, 0x6c, 0x5a, 0x76, 0xb9, 0xb6, 0xd2, 0x38, 0x2d, 0x7b, 0xe0, 0xa3, 0xba, 0x07,
-	0x74, 0xf1, 0xee, 0xcc, 0xf5, 0x80, 0x8e, 0x5a, 0x77, 0xc2, 0x0b, 0xe8, 0x35, 0xf0, 0x85, 0xfd,
-	0xf0, 0x61, 0xd5, 0x0f, 0xf6, 0xbc, 0x4b, 0xc4, 0x8d, 0xcb, 0xb2, 0x2b, 0x9e, 0x43, 0xaf, 0x01,
-	0x2f, 0xf4, 0x38, 0x84, 0x5b, 0x5f, 0x1c, 0xb3, 0x28, 0x66, 0x07, 0xb1, 0x5e, 0x4f, 0xe5, 0x92,
-	0x9d, 0x87, 0x49, 0x04, 0xfd, 0xcd, 0xb8, 0xc8, 0x25, 0xcf, 0x8c, 0x3b, 0xb5, 0x99, 0x35, 0x50,
-	0x15, 0xaf, 0x06, 0x16, 0xd7, 0xcf, 0xbb, 0x0f, 0x2d, 0x95, 0x46, 0x3d, 0x38, 0x17, 0x73, 0xac,
-	0x95, 0x64, 0x1f, 0x3a, 0x1b, 0x61, 0xf0, 0x55, 0x26, 0x8a, 0x74, 0x21, 0xe9, 0xf2, 0xc1, 0xb4,
-	0x2f, 0x3e, 0x98, 0xce, 0x85, 0x07, 0xd3, 0xad, 0x1e, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a,
-	0x5e, 0xaf, 0xb3, 0xae, 0xca, 0xd7, 0xcc, 0x69, 0xbc, 0x66, 0x21, 0xac, 0xe8, 0xb5, 0xf4, 0x7f,
-	0x3a, 0xfd, 0xcd, 0x86, 0x15, 0xca, 0xf3, 0xe8, 0x35, 0x0f, 0x92, 0x5c, 0x66, 0xc5, 0x58, 0x6d,
-	0x1f, 0x75, 0xff, 0x1b, 0x71, 0x60, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x49, 0xa7, 0x7b, 0x8f, 0xa0,
-	0x37, 0x3f, 0x9d, 0x17, 0x4d, 0x9b, 0x26, 0xde, 0x23, 0xb8, 0x11, 0x8a, 0x22, 0x1b, 0x57, 0xed,
-	0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x2b, 0x46, 0xe3, 0xe9,
-	0x5c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x9c, 0x9a, 0x9e, 0xb7, 0x26, 0x3f, 0x5b,
-	0x70, 0xb3, 0x49, 0xe1, 0x8d, 0x06, 0xb7, 0xaa, 0x88, 0xbd, 0xb0, 0x22, 0xce, 0xa2, 0x8a, 0xb8,
-	0x75, 0x45, 0xea, 0x87, 0xb9, 0xd5, 0x78, 0x98, 0xc9, 0x11, 0xdc, 0xbd, 0x50, 0xa6, 0x4d, 0x31,
-	0x4d, 0x55, 0x3f, 0xfc, 0x87, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2,
-	0x09, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x01, 0x38, 0x3b, 0xfc, 0xe4, 0x92, 0xcf,
-	0x57, 0x2a, 0xf2, 0x39, 0xf8, 0x7b, 0xe9, 0x84, 0x49, 0x7e, 0xad, 0xdb, 0x1b, 0xd0, 0xd9, 0x15,
-	0xa9, 0x88, 0xc5, 0xab, 0xd9, 0x15, 0x53, 0xef, 0xc3, 0x0d, 0xbd, 0xbf, 0xf5, 0x1a, 0xe9, 0xd2,
-	0x52, 0x24, 0xb7, 0x55, 0x43, 0x8f, 0x59, 0x3c, 0x2e, 0x62, 0x45, 0x43, 0xfd, 0xb4, 0xe5, 0x1b,
-	0xcb, 0x7f, 0x9c, 0xad, 0x59, 0x7f, 0x9e, 0xad, 0x59, 0x7f, 0x9d, 0xad, 0x59, 0xbf, 0xfe, 0xbd,
-	0xf6, 0xd6, 0x41, 0x1b, 0x7f, 0xd6, 0x9f, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x7c, 0x6d,
-	0x22, 0xbd, 0x0b, 0x00, 0x00,
+	// 1093 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44,
+	0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0x9b, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x95,
+	0x1a, 0x2a, 0x11, 0xaa, 0xf6, 0x86, 0xbf, 0x4a, 0x25, 0xbb, 0x01, 0x4c, 0x49, 0x28, 0xe3, 0x24,
+	0x77, 0xbd, 0x98, 0xec, 0x8e, 0x1a, 0x2b, 0x5e, 0x8f, 0xb1, 0xc7, 0x49, 0xb6, 0x17, 0xdc, 0x82,
+	0xc4, 0x0b, 0x20, 0x9e, 0x84, 0x47, 0xe0, 0x92, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x9c, 0x19, 0xff,
+	0x64, 0xb3, 0x21, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f,
+	0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x66, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c,
+	0x26, 0x03, 0xe8, 0x06, 0xc9, 0x98, 0x9f, 0xed, 0x70, 0xc9, 0x3c, 0x0f, 0xdc, 0x67, 0x7c, 0x9a,
+	0xfb, 0xce, 0xba, 0xb5, 0xd1, 0xa1, 0x78, 0xf6, 0x08, 0x2c, 0xee, 0x65, 0x6c, 0x74, 0xbc, 0x2b,
+	0xe4, 0x6e, 0x11, 0xc7, 0xbe, 0x8b, 0xba, 0x0b, 0x18, 0xf9, 0xc3, 0x82, 0xc5, 0xaf, 0x22, 0x1e,
+	0x8f, 0xbf, 0x4f, 0x65, 0x24, 0x92, 0xdc, 0x7b, 0x0f, 0xba, 0x03, 0x36, 0x3a, 0xe2, 0x7b, 0xd3,
+	0x94, 0xa3, 0xb7, 0x2e, 0xad, 0x81, 0x4a, 0x1b, 0x46, 0xaf, 0x38, 0xfa, 0xeb, 0xd3, 0x1a, 0xf0,
+	0xd6, 0xa1, 0xb7, 0x17, 0x4d, 0xf8, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf1, 0x5b, 0x78, 0xbb, 0x09,
+	0x29, 0x9a, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0xcb, 0xe0, 0xec, 0x44, 0x89, 0xdf, 0x5d, 0xb7,
+	0x36, 0x1c, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0xe7, 0xf5, 0xea, 0xe7,
+	0x11, 0x02, 0x4b, 0xc1, 0x24, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, 0xed, 0x2c,
+	0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xde, 0x8a, 0xc5, 0xe8, 0x78, 0xc8, 0x24, 0xa3,
+	0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x37, 0x63, 0xa7, 0x05, 0x85, 0x62, 0x1e,
+	0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, 0x47, 0x2c,
+	0x1b, 0x63, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, 0x26, 0x01,
+	0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe6, 0xbe, 0xb5, 0xee, 0x6c,
+	0xb8, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0x93, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, 0xb9, 0x0b,
+	0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, 0x19, 0xd2,
+	0xc8, 0xbd, 0x27, 0xd0, 0x09, 0x25, 0x4b, 0xc6, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfa, 0x60, 0xb3,
+	0x6c, 0x9a, 0xcd, 0xca, 0x6c, 0xb3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa5, 0xd5, 0x95, 0x77, 0x3f,
+	0x87, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0xa7, 0x65, 0x56, 0x8f, 0xf9, 0x54, 0xbd, 0xff, 0x84,
+	0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xd9, 0x9f, 0x58, 0xe4, 0x00, 0xbc, 0x41, 0xc6,
+	0x99, 0xe4, 0x18, 0x64, 0x87, 0xe7, 0x39, 0x7b, 0xc9, 0xaf, 0xce, 0xb8, 0xce, 0xa2, 0xdd, 0xcc,
+	0x62, 0x55, 0x07, 0xa7, 0x51, 0x07, 0xf2, 0x00, 0xbc, 0x21, 0x8f, 0xb9, 0xe4, 0xa6, 0xe3, 0xff,
+	0xc3, 0x2f, 0x09, 0x4b, 0x0e, 0xd7, 0xdb, 0x7a, 0xf7, 0xc1, 0x55, 0xe3, 0x83, 0x14, 0x7a, 0x8f,
+	0x6e, 0xd7, 0x79, 0xaa, 0x26, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0xe7, 0xda, 0x87, 0xcd,
+	0x69, 0xa5, 0x07, 0x26, 0x94, 0x83, 0xa1, 0x56, 0xeb, 0x50, 0xcd, 0xf1, 0x33, 0xd1, 0x9e, 0x96,
+	0xcf, 0xbd, 0x69, 0x34, 0xf2, 0xc2, 0xa0, 0xaa, 0x2b, 0x77, 0xd9, 0x84, 0x9b, 0x3b, 0x78, 0xae,
+	0xa8, 0xd8, 0xd7, 0x53, 0x51, 0xee, 0x55, 0x27, 0xab, 0xcd, 0xe2, 0x28, 0xf7, 0x28, 0x90, 0xc7,
+	0xd0, 0x0e, 0x47, 0x47, 0x7c, 0xc2, 0xbc, 0x0f, 0x61, 0x01, 0x79, 0xf0, 0xdc, 0x34, 0xdb, 0xad,
+	0x99, 0x24, 0xd2, 0x52, 0x4f, 0x86, 0x86, 0xff, 0x5c, 0x4e, 0xf7, 0xa1, 0x8d, 0xd1, 0x73, 0xdf,
+	0x9d, 0x75, 0x83, 0x38, 0x35, 0x6a, 0xb2, 0x0d, 0xce, 0x3e, 0x0d, 0xd4, 0x10, 0x21, 0x83, 0xd2,
+	0x8b, 0x91, 0x94, 0xef, 0x6f, 0x44, 0x2e, 0x4d, 0x36, 0xf0, 0xac, 0xb0, 0xe7, 0x22, 0x93, 0x98,
+	0xfa, 0x3e, 0xc5, 0x33, 0x79, 0x01, 0xee, 0xae, 0x18, 0x73, 0x6f, 0x09, 0xec, 0x60, 0x68, 0x7c,
+	0xd8, 0xc1, 0xd0, 0x7b, 0x1f, 0xdd, 0x9b, 0xd4, 0xf4, 0x6b, 0x12, 0xfb, 0x34, 0xa0, 0x18, 0xf8,
+	0x1e, 0xf4, 0x83, 0x7c, 0x20, 0x44, 0x36, 0x8e, 0x12, 0x26, 0x45, 0x66, 0x56, 0xee, 0x45, 0x90,
+	0x3c, 0x85, 0x65, 0xe5, 0x3e, 0x94, 0x4c, 0xf2, 0xb2, 0x7e, 0xab, 0xd0, 0x56, 0x58, 0x15, 0xce,
+	0x48, 0x38, 0x08, 0xca, 0xae, 0xac, 0x20, 0x0a, 0xe4, 0x3b, 0xed, 0x61, 0xfb, 0x84, 0x27, 0xb2,
+	0xd1, 0x01, 0x28, 0xa3, 0x83, 0x3e, 0xd5, 0x82, 0x47, 0xf4, 0x53, 0x0c, 0xe7, 0xa5, 0x9a, 0xb3,
+	0x42, 0x29, 0xea, 0xc8, 0xaf, 0x16, 0x40, 0x49, 0xa8, 0xc8, 0xab, 0x2b, 0xd6, 0xd5, 0x57, 0xbc,
+	0x8d, 0xb2, 0xc6, 0xa6, 0x65, 0x97, 0x6b, 0x2b, 0x8d, 0xd3, 0xb2, 0x07, 0x3e, 0xae, 0x7b, 0x40,
+	0x17, 0xef, 0xce, 0x4c, 0x0f, 0xe8, 0xa8, 0x75, 0x27, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0xdb, 0x0f,
+	0x1f, 0x55, 0xfd, 0x60, 0xcf, 0xba, 0x44, 0xdc, 0xb8, 0x2c, 0xbb, 0xe2, 0x19, 0xf4, 0x1a, 0xf0,
+	0x5c, 0x8f, 0x1b, 0x70, 0xeb, 0xcb, 0x13, 0x16, 0xc5, 0xec, 0x30, 0xd6, 0xeb, 0xa9, 0x5c, 0xb2,
+	0xb3, 0x30, 0x89, 0xa0, 0x3f, 0x88, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, 0x03, 0x55,
+	0xf1, 0x6a, 0x60, 0x7e, 0xfd, 0xbc, 0x7b, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, 0xc7, 0x5a,
+	0x49, 0x0e, 0xa0, 0xb3, 0x15, 0x06, 0x5f, 0x67, 0xa2, 0x48, 0xe7, 0x92, 0x2e, 0x3f, 0x98, 0xf6,
+	0xe5, 0x0f, 0xa6, 0x73, 0xe9, 0x83, 0xe9, 0x56, 0x1f, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a,
+	0x5e, 0x6f, 0xb2, 0xae, 0xca, 0xaf, 0x99, 0xd3, 0xf8, 0x9a, 0x85, 0xb0, 0xa2, 0xd7, 0xd2, 0x9b,
+	0x74, 0xfa, 0xbb, 0x0d, 0x2b, 0x94, 0xe7, 0xd1, 0x2b, 0x1e, 0x24, 0xb9, 0xcc, 0x8a, 0x91, 0xda,
+	0x3e, 0xea, 0xfe, 0xb7, 0xe2, 0xd0, 0x64, 0xdb, 0xa1, 0x5a, 0x78, 0x9d, 0x4e, 0xf7, 0x1e, 0x42,
+	0x6f, 0x76, 0x3a, 0x2f, 0x9b, 0x36, 0x4d, 0xbc, 0x87, 0xb0, 0x10, 0x8a, 0x22, 0x1b, 0x55, 0xed,
+	0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x6b, 0x46, 0xe3, 0xc9,
+	0x4c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x82, 0x9a, 0x5e, 0xb4, 0x26, 0xbf, 0x58,
+	0xb0, 0xd8, 0xa4, 0xf0, 0x5a, 0x83, 0x5b, 0x55, 0xc4, 0x9e, 0x5b, 0x11, 0x67, 0x5e, 0x45, 0xdc,
+	0xba, 0x22, 0xf5, 0x87, 0xb9, 0xd5, 0xf8, 0x30, 0x93, 0x63, 0xb8, 0x7b, 0xa9, 0x4c, 0x03, 0x31,
+	0x49, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2,
+	0x29, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x75, 0x70, 0x76, 0xf9, 0xe9, 0x15, 0xcf,
+	0x57, 0x2a, 0xf2, 0x05, 0xf8, 0xfb, 0xe9, 0x98, 0x49, 0x7e, 0xa3, 0xdb, 0x5b, 0xd0, 0xd9, 0x13,
+	0xa9, 0x88, 0xc5, 0xcb, 0xe9, 0x35, 0x53, 0xef, 0xc3, 0x82, 0xde, 0xdf, 0x7a, 0x8d, 0x74, 0x69,
+	0x29, 0x92, 0xdb, 0xaa, 0xa1, 0x47, 0x2c, 0x1e, 0x15, 0xb1, 0xa2, 0xa1, 0x7e, 0xda, 0xf2, 0xad,
+	0xe5, 0x3f, 0xcf, 0xd7, 0xac, 0xbf, 0xce, 0xd7, 0xac, 0xbf, 0xcf, 0xd7, 0xac, 0xdf, 0xfe, 0x59,
+	0x7b, 0xeb, 0xb0, 0x8d, 0x3f, 0xf4, 0x8f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x4d, 0x5d,
+	0xbe, 0xe1, 0x0b, 0x00, 0x00,
 }
diff --git a/internal/private.proto b/internal/private.proto
index 95f211a81..1e97e29bc 100644
--- a/internal/private.proto
+++ b/internal/private.proto
@@ -4,6 +4,7 @@ package internal;
 
 message IndexMeta {
 	bool Keys = 3;
+	bool TrackNotNull = 4;
 }
 
 message FieldOptions {

From 60e83fc6dbdc25a21d8f0a483dc7f1d19d296635 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 6 Sep 2018 12:47:07 -0500
Subject: [PATCH 02/11] rename bitmap to row in tests

---
 executor_test.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/executor_test.go b/executor_test.go
index 03ac09102..851c88713 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -33,8 +33,8 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Ensure a bitmap query can be executed.
-func TestExecutor_Execute_Bitmap(t *testing.T) {
+// Ensure a row query can be executed.
+func TestExecutor_Execute_Row(t *testing.T) {
 	t.Run("Row", func(t *testing.T) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
@@ -283,7 +283,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 		t.Run("OK", func(t *testing.T) {
 			hldr.ClearBit("i", "f", 11, 1)
 			if n := hldr.Row("i", "f", 11).Count(); n != 0 {
-				t.Fatalf("unexpected bitmap count: %d", n)
+				t.Fatalf("unexpected row count: %d", n)
 			}
 
 			if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil {
@@ -295,7 +295,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 			}
 
 			if n := hldr.Row("i", "f", 11).Count(); n != 1 {
-				t.Fatalf("unexpected bitmap count: %d", n)
+				t.Fatalf("unexpected row count: %d", n)
 			}
 			if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, f=11)`}); err != nil {
 				t.Fatal(err)
@@ -328,7 +328,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 		t.Run("OK", func(t *testing.T) {
 			hldr.SetBit("i", "f", 1, 0)
 			if n := hldr.Row("i", "f", 11).Count(); n != 0 {
-				t.Fatalf("unexpected bitmap count: %d", n)
+				t.Fatalf("unexpected row count: %d", n)
 			}
 
 			if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil {
@@ -340,7 +340,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
 			}
 
 			if n := hldr.Row("i", "f", 11).Count(); n != 1 {
-				t.Fatalf("unexpected bitmap count: %d", n)
+				t.Fatalf("unexpected row count: %d", n)
 			}
 			if res, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("foo", f=11)`}); err != nil {
 				t.Fatal(err)
@@ -477,7 +477,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
 	}
 	t.Run("rowID", func(t *testing.T) {
 		// Set two attrs on f/10.
-		// Also set attrs on other bitmaps and fields to test isolation.
+		// Also set attrs on other rows and fields to test isolation.
 		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
 			t.Fatal(err)
 		}
@@ -495,13 +495,13 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
 		if m, err := f.RowAttrStore().Attrs(10); err != nil {
 			t.Fatal(err)
 		} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
-			t.Fatalf("unexpected bitmap attr: %#v", m)
+			t.Fatalf("unexpected row attr: %#v", m)
 		}
 	})
 
 	t.Run("rowKey", func(t *testing.T) {
 		// Set two attrs on f/10.
-		// Also set attrs on other bitmaps and fields to test isolation.
+		// Also set attrs on other rows and fields to test isolation.
 		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil {
 			t.Fatal(err)
 		}
@@ -664,7 +664,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
 	}
 }
 
-// Ensure a TopN() query with a source bitmap can be executed.
+// Ensure a TopN() query with a source row can be executed.
 func TestExecutor_Execute_TopN_Src(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
@@ -724,7 +724,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
 
 }
 
-//Ensure TopN handles Attribute filters with source bitmap
+//Ensure TopN handles Attribute filters with source row
 func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()

From 4c67cb18edd253df205ba6e8f9f3437012d1af72 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Thu, 6 Sep 2018 22:56:53 -0500
Subject: [PATCH 03/11] update notnull field on imports

---
 api.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/api.go b/api.go
index 9e6ee1daa..8fdb4cfa0 100644
--- a/api.go
+++ b/api.go
@@ -686,6 +686,12 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error {
 		timestamps[i] = &t
 	}
 
+	// Import columnIDs into notnull field.
+	if err := importNotNullColumns(index, req.ColumnIDs); err != nil {
+		api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing notnull columns")
+	}
+
 	// Import into fragment.
 	err = field.Import(req.RowIDs, req.ColumnIDs, timestamps)
 	if err != nil {
@@ -720,6 +726,12 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error {
 		}
 	}
 
+	// Import columnIDs into notnull field.
+	if err := importNotNullColumns(index, req.ColumnIDs); err != nil {
+		api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing notnull columns")
+	}
+
 	// Import into fragment.
 	err = field.importValue(req.ColumnIDs, req.Values)
 	if err != nil {
@@ -728,6 +740,17 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error {
 	return errors.Wrap(err, "importing")
 }
 
+func importNotNullColumns(index *Index, columnIDs []uint64) error {
+	nnf := index.unprotectedNotNullField()
+	if nnf == nil {
+		return nil
+	}
+
+	notNullRowIDs := make([]uint64, len(columnIDs))
+	notNullTimestamps := make([]*time.Time, len(columnIDs))
+	return nnf.Import(notNullRowIDs, columnIDs, notNullTimestamps)
+}
+
 // MaxShards returns the maximum shard number for each index in a map.
 func (api *API) MaxShards(_ context.Context) map[string]uint64 {
 	m := make(map[string]uint64)

From a433862c989862699e7d9faf7af4924b617ea9fb Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Fri, 7 Sep 2018 11:50:25 -0500
Subject: [PATCH 04/11] add tests for the notnull tracking

---
 executor_test.go    | 46 ++++++++++++++++++++++++
 http/client_test.go | 85 ++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 127 insertions(+), 4 deletions(-)

diff --git a/executor_test.go b/executor_test.go
index 851c88713..602be93a5 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -1497,6 +1497,52 @@ func TestExecutor_QueryCall(t *testing.T) {
 	})
 }
 
+// Ensure a notnull field is maintained.
+func TestExecutor_Execute_NotNull(t *testing.T) {
+	t.Run("Row", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackNotNull: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set bits.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
+			fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20),
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
+			t.Fatalf("unexpected notnull columns: %+v", bits)
+		}
+
+		// Reopen cluster to ensure not-null is reloaded.
+		if err := c[0].Reopen(); err != nil {
+			t.Fatal(err)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
+			t.Fatalf("unexpected notnull columns after reopen: %+v", bits)
+		}
+	})
+}
+
 func benchmarkNotNull(nn bool, b *testing.B) {
 	c := test.MustRunCluster(b, 1)
 	defer c.Close()
diff --git a/http/client_test.go b/http/client_test.go
index 131161d3c..33a452f7b 100644
--- a/http/client_test.go
+++ b/http/client_test.go
@@ -196,7 +196,7 @@ func TestClient_Export(t *testing.T) {
 		{RowID: 2, ColumnID: 203, RowKey: "row2", ColumnKey: "col203"},
 	}
 
-	t.Run("Import unkeyed,unkeyedf", func(t *testing.T) {
+	t.Run("Export unkeyed,unkeyedf", func(t *testing.T) {
 		// Populate data.
 		for _, bit := range data {
 			_, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{
@@ -230,7 +230,7 @@ func TestClient_Export(t *testing.T) {
 		}
 	})
 
-	t.Run("Import unkeyed,keyedf", func(t *testing.T) {
+	t.Run("Export unkeyed,keyedf", func(t *testing.T) {
 		// Populate data.
 		for _, bit := range data {
 			_, err := c.Query(context.Background(), "unkeyed", &pilosa.QueryRequest{
@@ -264,7 +264,7 @@ func TestClient_Export(t *testing.T) {
 		}
 	})
 
-	t.Run("Import keyed,unkeyedf", func(t *testing.T) {
+	t.Run("Export keyed,unkeyedf", func(t *testing.T) {
 		// Populate data.
 		for _, bit := range data {
 			_, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{
@@ -298,7 +298,7 @@ func TestClient_Export(t *testing.T) {
 		}
 	})
 
-	t.Run("Import keyed,keyedf", func(t *testing.T) {
+	t.Run("Export keyed,keyedf", func(t *testing.T) {
 		// Populate data.
 		for _, bit := range data {
 			_, err := c.Query(context.Background(), "keyed", &pilosa.QueryRequest{
@@ -655,6 +655,83 @@ func TestClient_ImportValue(t *testing.T) {
 	}
 }
 
+// Ensure client can bulk import data while tracking notnull.
+func TestClient_ImportNotNull(t *testing.T) {
+	cmd := test.MustRunCluster(t, 1)[0]
+	host := cmd.URL()
+	holder := cmd.Server.Holder()
+	hldr := test.Holder{Holder: holder}
+
+	t.Run("Set", func(t *testing.T) {
+		idxName := "iset"
+		fldName := "fset"
+
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true})
+		_, err := index.CreateFieldIfNotExists(fldName)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Send import request.
+		c := MustNewClient(host, http.GetHTTPClient(nil))
+		if err := c.Import(context.Background(), idxName, fldName, 0, []pilosa.Bit{
+			{RowID: 0, ColumnID: 1},
+			{RowID: 0, ColumnID: 5},
+			{RowID: 200, ColumnID: 6},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify data.
+		if a := hldr.Row(idxName, fldName, 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
+			t.Fatalf("unexpected columns: %+v", a)
+		}
+		if a := hldr.Row(idxName, fldName, 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
+			t.Fatalf("unexpected columns: %+v", a)
+		}
+
+		// Verify notnull.
+		if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) {
+			t.Fatalf("unexpected notnull columns: %+v", a)
+		}
+	})
+
+	t.Run("Int", func(t *testing.T) {
+		idxName := "iint"
+		fldName := "fint"
+
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true})
+		field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Send import request.
+		c := MustNewClient(host, http.GetHTTPClient(nil))
+		if err := c.ImportValue(context.Background(), idxName, fldName, 0, []pilosa.FieldValue{
+			{ColumnID: 1, Value: -10},
+			{ColumnID: 2, Value: 20},
+			{ColumnID: 3, Value: 40},
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify Sum.
+		sum, cnt, err := field.Sum(nil, fldName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if sum != 50 || cnt != 3 {
+			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
+		}
+
+		// Verify notnull.
+		if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
+			t.Fatalf("unexpected notnull columns: %+v", a)
+		}
+	})
+}
+
 // Ensure client can retrieve a list of all checksums for blocks in a fragment.
 func TestClient_FragmentBlocks(t *testing.T) {
 	cmd := test.MustRunCluster(t, 1)[0]

From 3d7fc0c1c68cde863bf2c90c96f3b5c136285ff0 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Fri, 7 Sep 2018 11:54:44 -0500
Subject: [PATCH 05/11] remove unnecessary timestamp slice allocation

---
 api.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/api.go b/api.go
index 8fdb4cfa0..e5562602b 100644
--- a/api.go
+++ b/api.go
@@ -747,8 +747,7 @@ func importNotNullColumns(index *Index, columnIDs []uint64) error {
 	}
 
 	notNullRowIDs := make([]uint64, len(columnIDs))
-	notNullTimestamps := make([]*time.Time, len(columnIDs))
-	return nnf.Import(notNullRowIDs, columnIDs, notNullTimestamps)
+	return nnf.Import(notNullRowIDs, columnIDs, nil)
 }
 
 // MaxShards returns the maximum shard number for each index in a map.

From 8b99414029bc9c9a22987b06c471e3507976fcb1 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Mon, 10 Sep 2018 11:49:40 -0500
Subject: [PATCH 06/11] apply fieldName validation to index.CreateField

---
 field.go |  9 +++++++--
 index.go | 12 +++++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/field.go b/field.go
index 9fb352fc1..7a60f3b3f 100644
--- a/field.go
+++ b/field.go
@@ -159,12 +159,17 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption {
 func NewField(path, index, name string, opts FieldOption) (*Field, error) {
 	err := validateName(name)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "validating name")
 	}
 
+	return newField(path, index, name, opts)
+}
+
+// newField returns a new instance of field (without name validation).
+func newField(path, index, name string, opts FieldOption) (*Field, error) {
 	// Apply functional option.
 	fo := FieldOptions{}
-	err = opts(&fo)
+	err := opts(&fo)
 	if err != nil {
 		return nil, errors.Wrap(err, "applying option")
 	}
diff --git a/index.go b/index.go
index adbc6810e..cccf2de69 100644
--- a/index.go
+++ b/index.go
@@ -289,6 +289,11 @@ func (i *Index) recalculateCaches() {
 
 // CreateField creates a field.
 func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
+	err := validateName(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "validating name")
+	}
+
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -311,6 +316,11 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) {
 
 // CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
 func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error) {
+	err := validateName(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "validating name")
+	}
+
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
@@ -379,7 +389,7 @@ func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
 }
 
 func (i *Index) newField(path, name string) (*Field, error) {
-	f, err := NewField(path, i.name, name, OptFieldTypeDefault())
+	f, err := newField(path, i.name, name, OptFieldTypeDefault())
 	if err != nil {
 		return nil, err
 	}

From 9b4c67ee605c80d2bf3db3792ac196bc9fa08762 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Mon, 10 Sep 2018 13:13:11 -0500
Subject: [PATCH 07/11] rename notnull to exists

---
 api.go                  |  24 +++----
 encoding/proto/proto.go |   6 +-
 executor.go             |   6 +-
 executor_test.go        |  24 +++----
 holder.go               |   6 +-
 http/client_test.go     |  20 +++---
 index.go                |  40 +++++------
 internal/private.pb.go  | 156 ++++++++++++++++++++--------------------
 internal/private.proto  |   2 +-
 pilosa.go               |   5 ++
 pilosa_internal_test.go |   1 +
 test/holder.go          |  14 ++++
 12 files changed, 162 insertions(+), 142 deletions(-)

diff --git a/api.go b/api.go
index e5562602b..beb2741d0 100644
--- a/api.go
+++ b/api.go
@@ -686,10 +686,10 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error {
 		timestamps[i] = &t
 	}
 
-	// Import columnIDs into notnull field.
-	if err := importNotNullColumns(index, req.ColumnIDs); err != nil {
-		api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
-		return errors.Wrap(err, "importing notnull columns")
+	// Import columnIDs into existence field.
+	if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
+		api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing existence columns")
 	}
 
 	// Import into fragment.
@@ -726,10 +726,10 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error {
 		}
 	}
 
-	// Import columnIDs into notnull field.
-	if err := importNotNullColumns(index, req.ColumnIDs); err != nil {
-		api.server.logger.Printf("import notnull error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
-		return errors.Wrap(err, "importing notnull columns")
+	// Import columnIDs into existence field.
+	if err := importExistenceColumns(index, req.ColumnIDs); err != nil {
+		api.server.logger.Printf("import existence error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
+		return errors.Wrap(err, "importing existence columns")
 	}
 
 	// Import into fragment.
@@ -740,14 +740,14 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error {
 	return errors.Wrap(err, "importing")
 }
 
-func importNotNullColumns(index *Index, columnIDs []uint64) error {
-	nnf := index.unprotectedNotNullField()
+func importExistenceColumns(index *Index, columnIDs []uint64) error {
+	nnf := index.unprotectedExistenceField()
 	if nnf == nil {
 		return nil
 	}
 
-	notNullRowIDs := make([]uint64, len(columnIDs))
-	return nnf.Import(notNullRowIDs, columnIDs, nil)
+	existenceRowIDs := make([]uint64, len(columnIDs))
+	return nnf.Import(existenceRowIDs, columnIDs, nil)
 }
 
 // MaxShards returns the maximum shard number for each index in a map.
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index c00e71b40..0fcd9df3a 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -509,8 +509,8 @@ func encodeCreateIndexMessage(m *pilosa.CreateIndexMessage) *internal.CreateInde
 
 func encodeIndexMeta(m *pilosa.IndexOptions) *internal.IndexMeta {
 	return &internal.IndexMeta{
-		Keys:         m.Keys,
-		TrackNotNull: m.TrackNotNull,
+		Keys:           m.Keys,
+		TrackExistence: m.TrackExistence,
 	}
 }
 
@@ -742,7 +742,7 @@ func decodeCreateIndexMessage(pb *internal.CreateIndexMessage, m *pilosa.CreateI
 
 func decodeIndexMeta(pb *internal.IndexMeta, m *pilosa.IndexOptions) {
 	m.Keys = pb.Keys
-	m.TrackNotNull = pb.TrackNotNull
+	m.TrackExistence = pb.TrackExistence
 }
 
 func decodeDeleteIndexMessage(pb *internal.DeleteIndexMessage, m *pilosa.DeleteIndexMessage) {
diff --git a/executor.go b/executor.go
index 1149f8c0e..5681644aa 100644
--- a/executor.go
+++ b/executor.go
@@ -1131,10 +1131,10 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
 		return false, ErrFieldNotFound
 	}
 
-	// Set column on not-null field.
-	if nnf := idx.unprotectedNotNullField(); nnf != nil {
+	// Set column on existence field.
+	if nnf := idx.unprotectedExistenceField(); nnf != nil {
 		if _, err := nnf.SetBit(0, colID, nil); err != nil {
-			return false, errors.Wrap(err, "setting not-null column")
+			return false, errors.Wrap(err, "setting existence column")
 		}
 	}
 
diff --git a/executor_test.go b/executor_test.go
index 602be93a5..25465411c 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -1497,13 +1497,13 @@ func TestExecutor_QueryCall(t *testing.T) {
 	})
 }
 
-// Ensure a notnull field is maintained.
-func TestExecutor_Execute_NotNull(t *testing.T) {
+// Ensure an existence field is maintained.
+func TestExecutor_Execute_Existence(t *testing.T) {
 	t.Run("Row", func(t *testing.T) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
-		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackNotNull: true})
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
 		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatal(err)
@@ -1524,26 +1524,26 @@ func TestExecutor_Execute_NotNull(t *testing.T) {
 			t.Fatalf("unexpected columns: %+v", bits)
 		}
 
-		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil {
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil {
 			t.Fatal(err)
 		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
-			t.Fatalf("unexpected notnull columns: %+v", bits)
+			t.Fatalf("unexpected existence columns: %+v", bits)
 		}
 
-		// Reopen cluster to ensure not-null is reloaded.
+		// Reopen cluster to ensure existence field is reloaded.
 		if err := c[0].Reopen(); err != nil {
 			t.Fatal(err)
 		}
 
-		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(notnull=0)`}); err != nil {
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil {
 			t.Fatal(err)
 		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) {
-			t.Fatalf("unexpected notnull columns after reopen: %+v", bits)
+			t.Fatalf("unexpected existence columns after reopen: %+v", bits)
 		}
 	})
 }
 
-func benchmarkNotNull(nn bool, b *testing.B) {
+func benchmarkExistence(nn bool, b *testing.B) {
 	c := test.MustRunCluster(b, 1)
 	defer c.Close()
 	hldr := test.Holder{Holder: c[0].Server.Holder()}
@@ -1551,7 +1551,7 @@ func benchmarkNotNull(nn bool, b *testing.B) {
 	indexName := "i"
 	fieldName := "f"
 
-	index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackNotNull: nn})
+	index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{TrackExistence: nn})
 	// Create field.
 	if _, err := index.CreateFieldIfNotExists(fieldName); err != nil {
 		b.Fatal(err)
@@ -1568,5 +1568,5 @@ func benchmarkNotNull(nn bool, b *testing.B) {
 	}
 }
 
-func BenchmarkExecutor_NotNull_True(b *testing.B)  { benchmarkNotNull(true, b) }
-func BenchmarkExecutor_NotNull_False(b *testing.B) { benchmarkNotNull(false, b) }
+func BenchmarkExecutor_Existence_True(b *testing.B)  { benchmarkExistence(true, b) }
+func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) }
diff --git a/holder.go b/holder.go
index 252fca2db..794104e53 100644
--- a/holder.go
+++ b/holder.go
@@ -39,8 +39,8 @@ const (
 	// fileLimit is the maximum open file limit (ulimit -n) to automatically set.
 	fileLimit = 262144 // (512^2)
 
-	// notNullFieldName is the name of the internal field used to store not-null values.
-	notNullFieldName = "notnull" // TODO: use a name less likely to collide?
+	// existenceFieldName is the name of the internal field used to store existence values.
+	existenceFieldName = "exists"
 )
 
 // Holder represents a container for indexes.
@@ -357,7 +357,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	}
 
 	index.keys = opt.Keys
-	index.trackNotNull = opt.TrackNotNull
+	index.trackExistence = opt.TrackExistence
 
 	if err := index.Open(); err != nil {
 		return nil, errors.Wrap(err, "opening")
diff --git a/http/client_test.go b/http/client_test.go
index 33a452f7b..f0e8cf132 100644
--- a/http/client_test.go
+++ b/http/client_test.go
@@ -655,8 +655,8 @@ func TestClient_ImportValue(t *testing.T) {
 	}
 }
 
-// Ensure client can bulk import data while tracking notnull.
-func TestClient_ImportNotNull(t *testing.T) {
+// Ensure client can bulk import data while tracking existence.
+func TestClient_ImportExistence(t *testing.T) {
 	cmd := test.MustRunCluster(t, 1)[0]
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
@@ -666,7 +666,7 @@ func TestClient_ImportNotNull(t *testing.T) {
 		idxName := "iset"
 		fldName := "fset"
 
-		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true})
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true})
 		_, err := index.CreateFieldIfNotExists(fldName)
 		if err != nil {
 			t.Fatal(err)
@@ -690,9 +690,9 @@ func TestClient_ImportNotNull(t *testing.T) {
 			t.Fatalf("unexpected columns: %+v", a)
 		}
 
-		// Verify notnull.
-		if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) {
-			t.Fatalf("unexpected notnull columns: %+v", a)
+		// Verify existence.
+		if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) {
+			t.Fatalf("unexpected existence columns: %+v", a)
 		}
 	})
 
@@ -700,7 +700,7 @@ func TestClient_ImportNotNull(t *testing.T) {
 		idxName := "iint"
 		fldName := "fint"
 
-		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackNotNull: true})
+		index := hldr.MustCreateIndexIfNotExists(idxName, pilosa.IndexOptions{TrackExistence: true})
 		field, err := index.CreateFieldIfNotExists(fldName, pilosa.OptFieldTypeInt(-100, 100))
 		if err != nil {
 			t.Fatal(err)
@@ -725,9 +725,9 @@ func TestClient_ImportNotNull(t *testing.T) {
 			t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt)
 		}
 
-		// Verify notnull.
-		if a := hldr.Row(idxName, "notnull", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
-			t.Fatalf("unexpected notnull columns: %+v", a)
+		// Verify existence.
+		if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) {
+			t.Fatalf("unexpected existence columns: %+v", a)
 		}
 	})
 }
diff --git a/index.go b/index.go
index cccf2de69..a78fd8365 100644
--- a/index.go
+++ b/index.go
@@ -37,8 +37,8 @@ type Index struct {
 	keys bool // use string keys
 
 	// Not-null tracking.
-	trackNotNull bool
-	notNullField *Field
+	trackExistence bool
+	existenceField *Field
 
 	// Fields by name.
 	fields map[string]*Field
@@ -96,8 +96,8 @@ func (i *Index) Options() IndexOptions {
 
 func (i *Index) options() IndexOptions {
 	return IndexOptions{
-		Keys:         i.keys,
-		TrackNotNull: i.trackNotNull,
+		Keys:           i.keys,
+		TrackExistence: i.trackExistence,
 	}
 }
 
@@ -117,9 +117,9 @@ func (i *Index) Open() error {
 		return errors.Wrap(err, "opening fields")
 	}
 
-	if i.trackNotNull {
-		if err := i.openNotNullField(); err != nil {
-			return errors.Wrap(err, "opening not-null field")
+	if i.trackExistence {
+		if err := i.openExistenceField(); err != nil {
+			return errors.Wrap(err, "opening existence field")
 		}
 	}
 
@@ -160,13 +160,13 @@ func (i *Index) openFields() error {
 	return nil
 }
 
-// openNotNullField gets or creates the not-null field and associates it to the index.
-func (i *Index) openNotNullField() error {
-	f, err := i.createFieldIfNotExists(notNullFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
+// openExistenceField gets or creates the existence field and associates it to the index.
+func (i *Index) openExistenceField() error {
+	f, err := i.createFieldIfNotExists(existenceFieldName, FieldOptions{CacheType: CacheTypeNone, CacheSize: 0})
 	if err != nil {
-		return errors.Wrap(err, "creating not-null field")
+		return errors.Wrap(err, "creating existence field")
 	}
-	i.notNullField = f
+	i.existenceField = f
 	return nil
 }
 
@@ -188,7 +188,7 @@ func (i *Index) loadMeta() error {
 
 	// Copy metadata fields.
 	i.keys = pb.Keys
-	i.trackNotNull = pb.TrackNotNull
+	i.trackExistence = pb.TrackExistence
 
 	return nil
 }
@@ -197,8 +197,8 @@ func (i *Index) loadMeta() error {
 func (i *Index) saveMeta() error {
 	// Marshal metadata.
 	buf, err := proto.Marshal(&internal.IndexMeta{
-		Keys:         i.keys,
-		TrackNotNull: i.trackNotNull,
+		Keys:           i.keys,
+		TrackExistence: i.trackExistence,
 	})
 	if err != nil {
 		return errors.Wrap(err, "marshalling")
@@ -275,9 +275,9 @@ func (i *Index) Fields() []*Field {
 	return a
 }
 
-// unprotectedNotNullField returns the internal field used to track not-null columns.
-func (i *Index) unprotectedNotNullField() *Field {
-	return i.notNullField
+// unprotectedExistenceField returns the internal field used to track column existence.
+func (i *Index) unprotectedExistenceField() *Field {
+	return i.existenceField
 }
 
 // recalculateCaches recalculates caches on every field in the index.
@@ -448,8 +448,8 @@ func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
 // IndexOptions represents options to set when initializing an index.
 type IndexOptions struct {
-	Keys         bool `json:"keys"`
-	TrackNotNull bool `json:"trackNotNull"`
+	Keys           bool `json:"keys"`
+	TrackExistence bool `json:"trackExistence"`
 }
 
 // hasTime returns true if a contains a non-nil time.
diff --git a/internal/private.pb.go b/internal/private.pb.go
index 2d1c0deef..c65049d51 100644
--- a/internal/private.pb.go
+++ b/internal/private.pb.go
@@ -62,8 +62,8 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type IndexMeta struct {
-	Keys         bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"`
-	TrackNotNull bool `protobuf:"varint,4,opt,name=TrackNotNull,proto3" json:"TrackNotNull,omitempty"`
+	Keys           bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"`
+	TrackExistence bool `protobuf:"varint,4,opt,name=TrackExistence,proto3" json:"TrackExistence,omitempty"`
 }
 
 func (m *IndexMeta) Reset()                    { *m = IndexMeta{} }
@@ -78,9 +78,9 @@ func (m *IndexMeta) GetKeys() bool {
 	return false
 }
 
-func (m *IndexMeta) GetTrackNotNull() bool {
+func (m *IndexMeta) GetTrackExistence() bool {
 	if m != nil {
-		return m.TrackNotNull
+		return m.TrackExistence
 	}
 	return false
 }
@@ -1059,10 +1059,10 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
-	if m.TrackNotNull {
+	if m.TrackExistence {
 		dAtA[i] = 0x20
 		i++
-		if m.TrackNotNull {
+		if m.TrackExistence {
 			dAtA[i] = 1
 		} else {
 			dAtA[i] = 0
@@ -2318,7 +2318,7 @@ func (m *IndexMeta) Size() (n int) {
 	if m.Keys {
 		n += 2
 	}
-	if m.TrackNotNull {
+	if m.TrackExistence {
 		n += 2
 	}
 	return n
@@ -2925,7 +2925,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
 			m.Keys = bool(v != 0)
 		case 4:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field TrackNotNull", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field TrackExistence", wireType)
 			}
 			var v int
 			for shift := uint(0); ; shift += 7 {
@@ -2942,7 +2942,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
-			m.TrackNotNull = bool(v != 0)
+			m.TrackExistence = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPrivate(dAtA[iNdEx:])
@@ -7184,74 +7184,74 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
 
 var fileDescriptorPrivate = []byte{
-	// 1093 bytes of a gzipped FileDescriptorProto
+	// 1095 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xdc, 0x44,
-	0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0x9b, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x95,
-	0x1a, 0x2a, 0x11, 0xaa, 0xf6, 0x86, 0xbf, 0x4a, 0x25, 0xbb, 0x01, 0x4c, 0x49, 0x28, 0xe3, 0x24,
-	0x77, 0xbd, 0x98, 0xec, 0x8e, 0x1a, 0x2b, 0x5e, 0x8f, 0xb1, 0xc7, 0x49, 0xb6, 0x17, 0xdc, 0x82,
-	0xc4, 0x0b, 0x20, 0x9e, 0x84, 0x47, 0xe0, 0x92, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x9c, 0x19, 0xff,
-	0x64, 0xb3, 0x21, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f,
-	0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x66, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c,
-	0x26, 0x03, 0xe8, 0x06, 0xc9, 0x98, 0x9f, 0xed, 0x70, 0xc9, 0x3c, 0x0f, 0xdc, 0x67, 0x7c, 0x9a,
-	0xfb, 0xce, 0xba, 0xb5, 0xd1, 0xa1, 0x78, 0xf6, 0x08, 0x2c, 0xee, 0x65, 0x6c, 0x74, 0xbc, 0x2b,
-	0xe4, 0x6e, 0x11, 0xc7, 0xbe, 0x8b, 0xba, 0x0b, 0x18, 0xf9, 0xc3, 0x82, 0xc5, 0xaf, 0x22, 0x1e,
-	0x8f, 0xbf, 0x4f, 0x65, 0x24, 0x92, 0xdc, 0x7b, 0x0f, 0xba, 0x03, 0x36, 0x3a, 0xe2, 0x7b, 0xd3,
-	0x94, 0xa3, 0xb7, 0x2e, 0xad, 0x81, 0x4a, 0x1b, 0x46, 0xaf, 0x38, 0xfa, 0xeb, 0xd3, 0x1a, 0xf0,
-	0xd6, 0xa1, 0xb7, 0x17, 0x4d, 0xf8, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf1, 0x5b, 0x78, 0xbb, 0x09,
-	0x29, 0x9a, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0xcb, 0xe0, 0xec, 0x44, 0x89, 0xdf, 0x5d, 0xb7,
-	0x36, 0x1c, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0xe7, 0xf5, 0xea, 0xe7,
-	0x11, 0x02, 0x4b, 0xc1, 0x24, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7, 0xed, 0x2c,
-	0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xde, 0x8a, 0xc5, 0xe8, 0x78, 0xc8, 0x24, 0xa3,
-	0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x37, 0x63, 0xa7, 0x05, 0x85, 0x62, 0x1e,
-	0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86, 0x47, 0x2c,
-	0x1b, 0x63, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78, 0x26, 0x01,
-	0xac, 0x34, 0xe2, 0x1b, 0x9a, 0xab, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe6, 0xbe, 0xb5, 0xee, 0x6c,
-	0xb8, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0x93, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00, 0xb9, 0x0b,
-	0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8, 0x19, 0xd2,
-	0xc8, 0xbd, 0x27, 0xd0, 0x09, 0x25, 0x4b, 0xc6, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfa, 0x60, 0xb3,
-	0x6c, 0x9a, 0xcd, 0xca, 0x6c, 0xb3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa5, 0xd5, 0x95, 0x77, 0x3f,
-	0x87, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0xa7, 0x65, 0x56, 0x8f, 0xf9, 0x54, 0xbd, 0xff, 0x84,
-	0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xd9, 0x9f, 0x58, 0xe4, 0x00, 0xbc, 0x41, 0xc6,
-	0x99, 0xe4, 0x18, 0x64, 0x87, 0xe7, 0x39, 0x7b, 0xc9, 0xaf, 0xce, 0xb8, 0xce, 0xa2, 0xdd, 0xcc,
-	0x62, 0x55, 0x07, 0xa7, 0x51, 0x07, 0xf2, 0x00, 0xbc, 0x21, 0x8f, 0xb9, 0xe4, 0xa6, 0xe3, 0xff,
-	0xc3, 0x2f, 0x09, 0x4b, 0x0e, 0xd7, 0xdb, 0x7a, 0xf7, 0xc1, 0x55, 0xe3, 0x83, 0x14, 0x7a, 0x8f,
-	0x6e, 0xd7, 0x79, 0xaa, 0x26, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0xe7, 0xda, 0x87, 0xcd,
-	0x69, 0xa5, 0x07, 0x26, 0x94, 0x83, 0xa1, 0x56, 0xeb, 0x50, 0xcd, 0xf1, 0x33, 0xd1, 0x9e, 0x96,
-	0xcf, 0xbd, 0x69, 0x34, 0xf2, 0xc2, 0xa0, 0xaa, 0x2b, 0x77, 0xd9, 0x84, 0x9b, 0x3b, 0x78, 0xae,
-	0xa8, 0xd8, 0xd7, 0x53, 0x51, 0xee, 0x55, 0x27, 0xab, 0xcd, 0xe2, 0x28, 0xf7, 0x28, 0x90, 0xc7,
-	0xd0, 0x0e, 0x47, 0x47, 0x7c, 0xc2, 0xbc, 0x0f, 0x61, 0x01, 0x79, 0xf0, 0xdc, 0x34, 0xdb, 0xad,
-	0x99, 0x24, 0xd2, 0x52, 0x4f, 0x86, 0x86, 0xff, 0x5c, 0x4e, 0xf7, 0xa1, 0x8d, 0xd1, 0x73, 0xdf,
-	0x9d, 0x75, 0x83, 0x38, 0x35, 0x6a, 0xb2, 0x0d, 0xce, 0x3e, 0x0d, 0xd4, 0x10, 0x21, 0x83, 0xd2,
-	0x8b, 0x91, 0x94, 0xef, 0x6f, 0x44, 0x2e, 0x4d, 0x36, 0xf0, 0xac, 0xb0, 0xe7, 0x22, 0x93, 0x98,
-	0xfa, 0x3e, 0xc5, 0x33, 0x79, 0x01, 0xee, 0xae, 0x18, 0x73, 0x6f, 0x09, 0xec, 0x60, 0x68, 0x7c,
-	0xd8, 0xc1, 0xd0, 0x7b, 0x1f, 0xdd, 0x9b, 0xd4, 0xf4, 0x6b, 0x12, 0xfb, 0x34, 0xa0, 0x18, 0xf8,
-	0x1e, 0xf4, 0x83, 0x7c, 0x20, 0x44, 0x36, 0x8e, 0x12, 0x26, 0x45, 0x66, 0x56, 0xee, 0x45, 0x90,
-	0x3c, 0x85, 0x65, 0xe5, 0x3e, 0x94, 0x4c, 0xf2, 0xb2, 0x7e, 0xab, 0xd0, 0x56, 0x58, 0x15, 0xce,
-	0x48, 0x38, 0x08, 0xca, 0xae, 0xac, 0x20, 0x0a, 0xe4, 0x3b, 0xed, 0x61, 0xfb, 0x84, 0x27, 0xb2,
-	0xd1, 0x01, 0x28, 0xa3, 0x83, 0x3e, 0xd5, 0x82, 0x47, 0xf4, 0x53, 0x0c, 0xe7, 0xa5, 0x9a, 0xb3,
-	0x42, 0x29, 0xea, 0xc8, 0xaf, 0x16, 0x40, 0x49, 0xa8, 0xc8, 0xab, 0x2b, 0xd6, 0xd5, 0x57, 0xbc,
-	0x8d, 0xb2, 0xc6, 0xa6, 0x65, 0x97, 0x6b, 0x2b, 0x8d, 0xd3, 0xb2, 0x07, 0x3e, 0xae, 0x7b, 0x40,
-	0x17, 0xef, 0xce, 0x4c, 0x0f, 0xe8, 0xa8, 0x75, 0x27, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0xdb, 0x0f,
-	0x1f, 0x55, 0xfd, 0x60, 0xcf, 0xba, 0x44, 0xdc, 0xb8, 0x2c, 0xbb, 0xe2, 0x19, 0xf4, 0x1a, 0xf0,
-	0x5c, 0x8f, 0x1b, 0x70, 0xeb, 0xcb, 0x13, 0x16, 0xc5, 0xec, 0x30, 0xd6, 0xeb, 0xa9, 0x5c, 0xb2,
-	0xb3, 0x30, 0x89, 0xa0, 0x3f, 0x88, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59, 0x03, 0x55,
-	0xf1, 0x6a, 0x60, 0x7e, 0xfd, 0xbc, 0x7b, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39, 0xc7, 0x5a,
-	0x49, 0x0e, 0xa0, 0xb3, 0x15, 0x06, 0x5f, 0x67, 0xa2, 0x48, 0xe7, 0x92, 0x2e, 0x3f, 0x98, 0xf6,
-	0xe5, 0x0f, 0xa6, 0x73, 0xe9, 0x83, 0xe9, 0x56, 0x1f, 0x4c, 0x12, 0xc2, 0x8a, 0xde, 0x57, 0x6a,
-	0x5e, 0x6f, 0xb2, 0xae, 0xca, 0xaf, 0x99, 0xd3, 0xf8, 0x9a, 0x85, 0xb0, 0xa2, 0xd7, 0xd2, 0x9b,
-	0x74, 0xfa, 0xbb, 0x0d, 0x2b, 0x94, 0xe7, 0xd1, 0x2b, 0x1e, 0x24, 0xb9, 0xcc, 0x8a, 0x91, 0xda,
-	0x3e, 0xea, 0xfe, 0xb7, 0xe2, 0xd0, 0x64, 0xdb, 0xa1, 0x5a, 0x78, 0x9d, 0x4e, 0xf7, 0x1e, 0x42,
-	0x6f, 0x76, 0x3a, 0x2f, 0x9b, 0x36, 0x4d, 0xbc, 0x87, 0xb0, 0x10, 0x8a, 0x22, 0x1b, 0x55, 0xed,
-	0xdb, 0xd8, 0x88, 0x9a, 0x99, 0x56, 0xd3, 0xd2, 0xac, 0x31, 0x1a, 0xad, 0x6b, 0x46, 0xe3, 0xc9,
-	0x4c, 0x2b, 0xf9, 0x6d, 0xbc, 0xf0, 0x4e, 0x7d, 0xe1, 0x82, 0x9a, 0x5e, 0xb4, 0x26, 0xbf, 0x58,
-	0xb0, 0xd8, 0xa4, 0xf0, 0x5a, 0x83, 0x5b, 0x55, 0xc4, 0x9e, 0x5b, 0x11, 0x67, 0x5e, 0x45, 0xdc,
-	0xba, 0x22, 0xf5, 0x87, 0xb9, 0xd5, 0xf8, 0x30, 0x93, 0x63, 0xb8, 0x7b, 0xa9, 0x4c, 0x03, 0x31,
-	0x49, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa, 0x05, 0xf2,
-	0x29, 0xdc, 0x09, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x75, 0x70, 0x76, 0xf9, 0xe9, 0x15, 0xcf,
-	0x57, 0x2a, 0xf2, 0x05, 0xf8, 0xfb, 0xe9, 0x98, 0x49, 0x7e, 0xa3, 0xdb, 0x5b, 0xd0, 0xd9, 0x13,
-	0xa9, 0x88, 0xc5, 0xcb, 0xe9, 0x35, 0x53, 0xef, 0xc3, 0x82, 0xde, 0xdf, 0x7a, 0x8d, 0x74, 0x69,
-	0x29, 0x92, 0xdb, 0xaa, 0xa1, 0x47, 0x2c, 0x1e, 0x15, 0xb1, 0xa2, 0xa1, 0x7e, 0xda, 0xf2, 0xad,
-	0xe5, 0x3f, 0xcf, 0xd7, 0xac, 0xbf, 0xce, 0xd7, 0xac, 0xbf, 0xcf, 0xd7, 0xac, 0xdf, 0xfe, 0x59,
-	0x7b, 0xeb, 0xb0, 0x8d, 0x3f, 0xf4, 0x8f, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x4d, 0x5d,
-	0xbe, 0xe1, 0x0b, 0x00, 0x00,
+	0x14, 0xc6, 0x3f, 0xbb, 0xd9, 0x3d, 0xdb, 0x4d, 0x13, 0x97, 0x06, 0x17, 0xa1, 0x10, 0x46, 0x15,
+	0x0d, 0x95, 0x08, 0x55, 0x7b, 0xc3, 0x5f, 0xa5, 0x92, 0x6c, 0x28, 0xa6, 0x24, 0x94, 0x71, 0x92,
+	0xbb, 0x5e, 0x4c, 0x76, 0x47, 0x8d, 0x15, 0xaf, 0xc7, 0xd8, 0xe3, 0x24, 0xdb, 0x0b, 0x6e, 0x41,
+	0xe2, 0x05, 0x10, 0x4f, 0xc2, 0x23, 0x70, 0xc9, 0x23, 0xa0, 0xf0, 0x22, 0x68, 0xce, 0x8c, 0x7f,
+	0xb2, 0xd9, 0xb0, 0x55, 0xe8, 0xdd, 0x9c, 0xef, 0x9c, 0x39, 0xe7, 0x9b, 0xf3, 0x67, 0x43, 0x3f,
+	0xcd, 0xa2, 0x13, 0x26, 0xf9, 0x46, 0x9a, 0x09, 0x29, 0xbc, 0x4e, 0x94, 0x48, 0x9e, 0x25, 0x2c,
+	0x26, 0x4f, 0xa1, 0x1b, 0x24, 0x23, 0x7e, 0xb6, 0xc3, 0x25, 0xf3, 0x3c, 0x70, 0x9f, 0xf1, 0x49,
+	0xee, 0x3b, 0x6b, 0xd6, 0x7a, 0x87, 0xe2, 0xd9, 0xfb, 0x10, 0x16, 0xf7, 0x32, 0x36, 0x3c, 0xde,
+	0x3e, 0x8b, 0x72, 0xc9, 0x93, 0x21, 0xf7, 0x5d, 0xd4, 0x4e, 0xa1, 0xe4, 0x0f, 0x0b, 0x6e, 0x7c,
+	0x1d, 0xf1, 0x78, 0xf4, 0x7d, 0x2a, 0x23, 0x91, 0xe4, 0xde, 0x7b, 0xd0, 0xdd, 0x62, 0xc3, 0x23,
+	0xbe, 0x37, 0x49, 0x39, 0x7a, 0xec, 0xd2, 0x1a, 0xa8, 0xb4, 0x61, 0xf4, 0x4a, 0x7b, 0xec, 0xd3,
+	0x1a, 0xf0, 0xd6, 0xa0, 0xb7, 0x17, 0x8d, 0xf9, 0x0f, 0x05, 0x4b, 0x64, 0x31, 0xf6, 0x5b, 0x78,
+	0xbb, 0x09, 0x29, 0xaa, 0xe8, 0xb8, 0x83, 0x2a, 0x3c, 0x7b, 0x4b, 0xe0, 0xec, 0x44, 0x89, 0xdf,
+	0x5d, 0xb3, 0xd6, 0x1d, 0xaa, 0x8e, 0x88, 0xb0, 0x33, 0x1f, 0x0c, 0xc2, 0xce, 0xaa, 0x27, 0xf6,
+	0xea, 0x27, 0x12, 0x02, 0x8b, 0xc1, 0x38, 0x15, 0x99, 0xa4, 0x3c, 0x4f, 0x45, 0x92, 0xa3, 0xa7,
+	0xed, 0x2c, 0xf3, 0x2d, 0x74, 0xae, 0x8e, 0xe4, 0x27, 0x58, 0xda, 0x8c, 0xc5, 0xf0, 0x78, 0xc0,
+	0x24, 0xa3, 0xfc, 0xc7, 0x82, 0xe7, 0xd2, 0x7b, 0x1b, 0x5a, 0x98, 0x3b, 0x63, 0xa7, 0x05, 0x85,
+	0x62, 0x1e, 0x7c, 0x5b, 0xa3, 0x28, 0x28, 0x14, 0xef, 0x63, 0x26, 0x5c, 0xaa, 0x05, 0x85, 0x86,
+	0x47, 0x2c, 0x1b, 0x61, 0x06, 0x5c, 0xaa, 0x05, 0xc5, 0xf1, 0x20, 0xe2, 0xa7, 0xe6, 0xd9, 0x78,
+	0x26, 0x01, 0x2c, 0x37, 0xe2, 0x1b, 0x9a, 0x2b, 0xd0, 0xa6, 0xe2, 0x34, 0x18, 0xe4, 0xbe, 0xb5,
+	0xe6, 0xac, 0xbb, 0xd4, 0x48, 0x98, 0x5c, 0x11, 0x17, 0xe3, 0x44, 0xa9, 0x6c, 0x54, 0xd5, 0x00,
+	0xb9, 0x03, 0x2d, 0xcc, 0xb4, 0x7a, 0x65, 0x7d, 0x57, 0x1d, 0xc9, 0xcf, 0x16, 0x74, 0x77, 0xd8,
+	0x19, 0xd2, 0xc8, 0xbd, 0xc7, 0xd0, 0x09, 0x25, 0x4b, 0x46, 0x8a, 0xa0, 0x32, 0xea, 0x3d, 0xfc,
+	0x60, 0xa3, 0x6c, 0x9c, 0x8d, 0xca, 0x6c, 0xa3, 0xb4, 0xd9, 0x4e, 0x64, 0x36, 0xa1, 0xd5, 0x95,
+	0x77, 0xbf, 0x80, 0xfe, 0x05, 0x95, 0x8a, 0x77, 0xcc, 0x27, 0x65, 0x56, 0x8f, 0xf9, 0x44, 0xbd,
+	0xff, 0x84, 0xc5, 0x05, 0xc7, 0x5c, 0xb9, 0x54, 0x0b, 0x9f, 0xdb, 0x9f, 0x5a, 0xe4, 0x00, 0xbc,
+	0xad, 0x8c, 0x33, 0xc9, 0x31, 0xc8, 0x0e, 0xcf, 0x73, 0xf6, 0x92, 0x5f, 0x9d, 0x71, 0x9d, 0x45,
+	0xbb, 0x99, 0xc5, 0xaa, 0x0e, 0x4e, 0xa3, 0x0e, 0xe4, 0x3e, 0x78, 0x03, 0x1e, 0x73, 0xc9, 0x4d,
+	0xd7, 0xff, 0x87, 0x5f, 0x12, 0x96, 0x1c, 0xe6, 0xdb, 0x7a, 0xf7, 0xc0, 0x55, 0x23, 0x84, 0x14,
+	0x7a, 0x0f, 0x6f, 0xd5, 0x79, 0xaa, 0xa6, 0x8b, 0xa2, 0x01, 0x89, 0x4b, 0xa7, 0xc8, 0x67, 0xee,
+	0xc3, 0x66, 0xb4, 0xd2, 0x7d, 0x13, 0xca, 0xc1, 0x50, 0x2b, 0x75, 0xa8, 0xe6, 0xf8, 0x99, 0x68,
+	0x4f, 0xca, 0xe7, 0x5e, 0x37, 0x1a, 0x79, 0x61, 0x50, 0xd5, 0x95, 0xbb, 0x6c, 0xcc, 0xcd, 0x1d,
+	0x3c, 0x57, 0x54, 0xec, 0xf9, 0x54, 0x94, 0x7b, 0xd5, 0xc9, 0x6a, 0xbb, 0x38, 0xca, 0x3d, 0x0a,
+	0xe4, 0x11, 0xb4, 0xc3, 0xe1, 0x11, 0x1f, 0x33, 0xef, 0x23, 0x58, 0x40, 0x1e, 0x3c, 0x37, 0xcd,
+	0x76, 0x73, 0x2a, 0x89, 0xb4, 0xd4, 0x93, 0x81, 0xe1, 0x3f, 0x93, 0xd3, 0x3d, 0x68, 0x63, 0xf4,
+	0xdc, 0x77, 0xa7, 0xdd, 0x20, 0x4e, 0x8d, 0x9a, 0x6c, 0x83, 0xb3, 0x4f, 0x03, 0x35, 0x44, 0xc8,
+	0xa0, 0xf4, 0x62, 0x24, 0xe5, 0xfb, 0x1b, 0x91, 0x4b, 0x93, 0x0d, 0x3c, 0x2b, 0xec, 0xb9, 0xc8,
+	0x24, 0xa6, 0xbe, 0x4f, 0xf1, 0x4c, 0x5e, 0x80, 0xbb, 0x2b, 0x46, 0xdc, 0x5b, 0x04, 0x3b, 0x18,
+	0x18, 0x1f, 0x76, 0x30, 0xf0, 0xde, 0x47, 0xf7, 0x26, 0x35, 0xfd, 0x9a, 0xc4, 0x3e, 0x0d, 0x28,
+	0x06, 0xbe, 0x0b, 0xfd, 0x20, 0xdf, 0x12, 0x22, 0x1b, 0x45, 0x09, 0x93, 0x22, 0x33, 0x6b, 0xf7,
+	0x22, 0x48, 0x9e, 0xc0, 0x92, 0x72, 0x1f, 0x4a, 0x26, 0x79, 0x59, 0xbf, 0x15, 0x68, 0x2b, 0xac,
+	0x0a, 0x67, 0x24, 0x1c, 0x04, 0x65, 0x57, 0x56, 0x10, 0x05, 0xf2, 0x9d, 0xf6, 0xb0, 0x7d, 0xc2,
+	0x13, 0xd9, 0xe8, 0x00, 0x94, 0xd1, 0x41, 0x9f, 0x6a, 0xc1, 0x23, 0xfa, 0x29, 0x86, 0xf3, 0x62,
+	0xcd, 0x59, 0xa1, 0x14, 0x75, 0xe4, 0x57, 0x0b, 0xa0, 0x24, 0x54, 0xe4, 0xd5, 0x15, 0xeb, 0xea,
+	0x2b, 0xde, 0x7a, 0x59, 0x63, 0xd3, 0xb2, 0x4b, 0xb5, 0x95, 0xc6, 0x69, 0xd9, 0x03, 0x9f, 0xd4,
+	0x3d, 0xa0, 0x8b, 0x77, 0x7b, 0xaa, 0x07, 0x74, 0xd4, 0xba, 0x13, 0x9e, 0x43, 0xaf, 0x81, 0xcf,
+	0xec, 0x87, 0x8f, 0xab, 0x7e, 0xb0, 0xa7, 0x5d, 0x22, 0x6e, 0x5c, 0x96, 0x5d, 0xf1, 0x0c, 0x7a,
+	0x0d, 0x78, 0xa6, 0xc7, 0x75, 0xb8, 0xf9, 0xd5, 0x09, 0x8b, 0x62, 0x76, 0x18, 0xeb, 0xf5, 0x54,
+	0x2e, 0xd9, 0x69, 0x98, 0x44, 0xd0, 0xdf, 0x8a, 0x8b, 0x5c, 0xf2, 0xcc, 0xb8, 0x53, 0x9b, 0x59,
+	0x03, 0x55, 0xf1, 0x6a, 0x60, 0x76, 0xfd, 0xbc, 0xbb, 0xd0, 0x52, 0x69, 0xd4, 0x83, 0x73, 0x39,
+	0xc7, 0x5a, 0x49, 0x0e, 0xa0, 0xb3, 0x19, 0x06, 0x4f, 0x33, 0x51, 0xa4, 0x33, 0x49, 0x97, 0x1f,
+	0x4c, 0xfb, 0xf2, 0x07, 0xd3, 0xb9, 0xf4, 0xc1, 0x74, 0xab, 0x0f, 0x26, 0x09, 0x61, 0x59, 0xef,
+	0x2b, 0x35, 0xaf, 0xd7, 0x59, 0x57, 0xe5, 0xd7, 0xcc, 0x69, 0x7c, 0xcd, 0x42, 0x58, 0xd6, 0x6b,
+	0xe9, 0x4d, 0x3a, 0xfd, 0xdd, 0x86, 0x65, 0xca, 0xf3, 0xe8, 0x15, 0x0f, 0x92, 0x5c, 0x66, 0xc5,
+	0x50, 0x6d, 0x1f, 0x75, 0xff, 0x5b, 0x71, 0x68, 0xb2, 0xed, 0x50, 0x2d, 0xbc, 0x4e, 0xa7, 0x7b,
+	0x0f, 0xa0, 0x37, 0x3d, 0x9d, 0x97, 0x4d, 0x9b, 0x26, 0xde, 0x03, 0x58, 0x08, 0x45, 0x91, 0x0d,
+	0xab, 0xf6, 0x6d, 0x6c, 0x44, 0xcd, 0x4c, 0xab, 0x69, 0x69, 0xd6, 0x18, 0x8d, 0xd6, 0x9c, 0xd1,
+	0x78, 0x3c, 0xd5, 0x4a, 0x7e, 0x1b, 0x2f, 0xbc, 0x53, 0x5f, 0xb8, 0xa0, 0xa6, 0x17, 0xad, 0xc9,
+	0x2f, 0x16, 0xdc, 0x68, 0x52, 0x78, 0xad, 0xc1, 0xad, 0x2a, 0x62, 0xcf, 0xac, 0x88, 0x33, 0xab,
+	0x22, 0x6e, 0x5d, 0x91, 0xfa, 0xc3, 0xdc, 0x6a, 0x7c, 0x98, 0xc9, 0x31, 0xdc, 0xb9, 0x54, 0xa6,
+	0x2d, 0x31, 0x4e, 0x55, 0x3f, 0xfc, 0x8f, 0x72, 0xa9, 0x95, 0x96, 0x65, 0xa6, 0x50, 0x5d, 0xaa,
+	0x05, 0xf2, 0x19, 0xdc, 0x0e, 0xb9, 0x6c, 0x14, 0xa9, 0xec, 0xb6, 0x35, 0x70, 0x76, 0xf9, 0xe9,
+	0x15, 0xcf, 0x57, 0x2a, 0xf2, 0x25, 0xf8, 0xfb, 0xe9, 0x88, 0x49, 0x7e, 0xad, 0xdb, 0x9b, 0xd0,
+	0xd9, 0x13, 0xa9, 0x88, 0xc5, 0xcb, 0xc9, 0x9c, 0xa9, 0xf7, 0x61, 0x41, 0xef, 0x6f, 0xbd, 0x46,
+	0xba, 0xb4, 0x14, 0xc9, 0x2d, 0xd5, 0xd0, 0x43, 0x16, 0x0f, 0x8b, 0x58, 0xd1, 0x50, 0x3f, 0x6d,
+	0xf9, 0xe6, 0xd2, 0x9f, 0xe7, 0xab, 0xd6, 0x5f, 0xe7, 0xab, 0xd6, 0xdf, 0xe7, 0xab, 0xd6, 0x6f,
+	0xff, 0xac, 0xbe, 0x75, 0xd8, 0xc6, 0x9f, 0xfa, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x31,
+	0x07, 0xf3, 0xac, 0xe5, 0x0b, 0x00, 0x00,
 }
diff --git a/internal/private.proto b/internal/private.proto
index 1e97e29bc..88d16363f 100644
--- a/internal/private.proto
+++ b/internal/private.proto
@@ -4,7 +4,7 @@ package internal;
 
 message IndexMeta {
 	bool Keys = 3;
-	bool TrackNotNull = 4;
+	bool TrackExistence = 4;
 }
 
 message FieldOptions {
diff --git a/pilosa.go b/pilosa.go
index 45d77a9f3..7378eb628 100644
--- a/pilosa.go
+++ b/pilosa.go
@@ -49,6 +49,8 @@ var (
 	ErrName  = errors.New("invalid index or field name, must match [a-z0-9_-]")
 	ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
 
+	ErrReservedName = errors.New("reserved index or field name")
+
 	// ErrFragmentNotFound is returned when a fragment does not exist.
 	ErrFragmentNotFound = errors.New("fragment not found")
 	ErrQueryRequired    = errors.New("query required")
@@ -125,6 +127,9 @@ const TimeFormat = "2006-01-02T15:04"
 
 // validateName ensures that the name is a valid format.
 func validateName(name string) error {
+	if name == existenceFieldName {
+		return ErrReservedName
+	}
 	if !nameRegexp.Match([]byte(name)) {
 		return ErrName
 	}
diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go
index 139d5c2b7..96dae1433 100644
--- a/pilosa_internal_test.go
+++ b/pilosa_internal_test.go
@@ -34,6 +34,7 @@ func TestValidateNameInvalid(t *testing.T) {
 	names := []string{
 		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
 		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+		"exists",
 	}
 	for _, name := range names {
 		if validateName(name) == nil {
diff --git a/test/holder.go b/test/holder.go
index 80778fe2a..8d080e493 100644
--- a/test/holder.go
+++ b/test/holder.go
@@ -91,6 +91,20 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
 	return row
 }
 
+// ReadRow returns a Row for a given field. If the field does not exist,
+// it panics rather than creating the field.
+func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row {
+	f := h.Holder.Field(index, field)
+	if f == nil {
+		panic(pilosa.ErrFieldNotFound)
+	}
+	row, err := f.Row(rowID)
+	if err != nil {
+		panic(err)
+	}
+	return row
+}
+
 func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
 	idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
 	f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeDefault())

From f8c745340ff467983f840e682d7ff4601b39be3a Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Mon, 10 Sep 2018 16:56:33 -0500
Subject: [PATCH 08/11] stop tracking existence if the existence field is
 deleted

---
 fragment_internal_test.go |  4 ++--
 index.go                  | 12 +++++++++++
 index_internal_test.go    | 44 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index c85b7e88c..a0d1354ba 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -746,7 +746,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
 	cacheSize := uint32(3)
 
 	// Create Index.
-	index := mustOpenIndex()
+	index := mustOpenIndex(IndexOptions{})
 	defer index.Close()
 
 	// Create field.
@@ -912,7 +912,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
 
 // Ensure a fragment's cache can be persisted between restarts.
 func TestFragment_RankCache_Persistence(t *testing.T) {
-	index := mustOpenIndex()
+	index := mustOpenIndex(IndexOptions{})
 	defer index.Close()
 
 	// Create field.
diff --git a/index.go b/index.go
index a78fd8365..34ab01b28 100644
--- a/index.go
+++ b/index.go
@@ -421,6 +421,18 @@ func (i *Index) DeleteField(name string) error {
 		return errors.Wrap(err, "removing directory")
 	}
 
+	// If the field being deleted is the existence field,
+	// turn off existence tracking on the index.
+	if name == existenceFieldName {
+		i.trackExistence = false
+		i.existenceField = nil
+
+		// Update meta data on disk.
+		if err := i.saveMeta(); err != nil {
+			return errors.Wrap(err, "saving existence meta data")
+		}
+	}
+
 	// Remove reference.
 	delete(i.fields, name)
 
diff --git a/index_internal_test.go b/index_internal_test.go
index 1e6d592ab..de4368d2a 100644
--- a/index_internal_test.go
+++ b/index_internal_test.go
@@ -16,10 +16,11 @@ package pilosa
 
 import (
 	"io/ioutil"
+	"testing"
 )
 
 // mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func mustOpenIndex() *Index {
+func mustOpenIndex(opt IndexOptions) *Index {
 	path, err := ioutil.TempDir("", "pilosa-index-")
 	if err != nil {
 		panic(err)
@@ -28,6 +29,10 @@ func mustOpenIndex() *Index {
 	if err != nil {
 		panic(err)
 	}
+
+	index.keys = opt.Keys
+	index.trackExistence = opt.TrackExistence
+
 	if err := index.Open(); err != nil {
 		panic(err)
 	}
@@ -44,3 +49,40 @@ func (i *Index) reopen() error {
 	}
 	return nil
 }
+
+// Ensure that deleting the existence field is handled properly.
+func TestIndex_Existence_Delete(t *testing.T) {
+	// Create Index (with existence tracking).
+	index := mustOpenIndex(IndexOptions{TrackExistence: true})
+	defer index.Close()
+
+	// Ensure existence field has been created.
+	ef := index.Field(existenceFieldName)
+	if ef == nil {
+		t.Fatalf("expected field to have been created: %s", existenceFieldName)
+	} else if !index.trackExistence {
+		t.Fatalf("expected index.trackExistence to be true")
+	} else if index.existenceField == nil {
+		t.Fatalf("expected index.existenceField to be non-nil")
+	}
+
+	// Delete existence field.
+	if err := index.DeleteField(existenceFieldName); err != nil {
+		t.Fatal(err)
+	}
+
+	// Re-open index.
+	if err := index.reopen(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure existence field no longer exists.
+	ef = index.Field(existenceFieldName)
+	if ef != nil {
+		t.Fatalf("expected field to have been deleted: %s", existenceFieldName)
+	} else if index.trackExistence {
+		t.Fatalf("expected index.trackExistence to be false")
+	} else if index.existenceField != nil {
+		t.Fatalf("expected index.existenceField to be nil")
+	}
+}

From f7abf6062741d8fb129171a21991064592bb2667 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Tue, 11 Sep 2018 10:24:49 -0500
Subject: [PATCH 09/11] add lock around existencFld

---
 api.go                 |  6 +++---
 executor.go            |  4 ++--
 index.go               | 17 ++++++++++-------
 index_internal_test.go |  4 ++--
 4 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/api.go b/api.go
index beb2741d0..c8aa2b5cc 100644
--- a/api.go
+++ b/api.go
@@ -741,13 +741,13 @@ func (api *API) ImportValue(_ context.Context, req *ImportValueRequest) error {
 }
 
 func importExistenceColumns(index *Index, columnIDs []uint64) error {
-	nnf := index.unprotectedExistenceField()
-	if nnf == nil {
+	ef := index.existenceField()
+	if ef == nil {
 		return nil
 	}
 
 	existenceRowIDs := make([]uint64, len(columnIDs))
-	return nnf.Import(existenceRowIDs, columnIDs, nil)
+	return ef.Import(existenceRowIDs, columnIDs, nil)
 }
 
 // MaxShards returns the maximum shard number for each index in a map.
diff --git a/executor.go b/executor.go
index 5681644aa..389cb573b 100644
--- a/executor.go
+++ b/executor.go
@@ -1132,8 +1132,8 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op
 	}
 
 	// Set column on existence field.
-	if nnf := idx.unprotectedExistenceField(); nnf != nil {
-		if _, err := nnf.SetBit(0, colID, nil); err != nil {
+	if ef := idx.existenceField(); ef != nil {
+		if _, err := ef.SetBit(0, colID, nil); err != nil {
 			return false, errors.Wrap(err, "setting existence column")
 		}
 	}
diff --git a/index.go b/index.go
index 34ab01b28..0992ab213 100644
--- a/index.go
+++ b/index.go
@@ -36,9 +36,9 @@ type Index struct {
 	name string
 	keys bool // use string keys
 
-	// Not-null tracking.
+	// Existence tracking.
 	trackExistence bool
-	existenceField *Field
+	existenceFld   *Field
 
 	// Fields by name.
 	fields map[string]*Field
@@ -166,7 +166,7 @@ func (i *Index) openExistenceField() error {
 	if err != nil {
 		return errors.Wrap(err, "creating existence field")
 	}
-	i.existenceField = f
+	i.existenceFld = f
 	return nil
 }
 
@@ -275,9 +275,12 @@ func (i *Index) Fields() []*Field {
 	return a
 }
 
-// unprotectedExistenceField returns the internal field used to track column existence.
-func (i *Index) unprotectedExistenceField() *Field {
-	return i.existenceField
+// existenceField returns the internal field used to track column existence.
+func (i *Index) existenceField() *Field {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+
+	return i.existenceFld
 }
 
 // recalculateCaches recalculates caches on every field in the index.
@@ -425,7 +428,7 @@ func (i *Index) DeleteField(name string) error {
 	// turn off existence tracking on the index.
 	if name == existenceFieldName {
 		i.trackExistence = false
-		i.existenceField = nil
+		i.existenceFld = nil
 
 		// Update meta data on disk.
 		if err := i.saveMeta(); err != nil {
diff --git a/index_internal_test.go b/index_internal_test.go
index de4368d2a..fd83fd775 100644
--- a/index_internal_test.go
+++ b/index_internal_test.go
@@ -62,7 +62,7 @@ func TestIndex_Existence_Delete(t *testing.T) {
 		t.Fatalf("expected field to have been created: %s", existenceFieldName)
 	} else if !index.trackExistence {
 		t.Fatalf("expected index.trackExistence to be true")
-	} else if index.existenceField == nil {
+	} else if index.existenceFld == nil {
 		t.Fatalf("expected index.existenceField to be non-nil")
 	}
 
@@ -82,7 +82,7 @@ func TestIndex_Existence_Delete(t *testing.T) {
 		t.Fatalf("expected field to have been deleted: %s", existenceFieldName)
 	} else if index.trackExistence {
 		t.Fatalf("expected index.trackExistence to be false")
-	} else if index.existenceField != nil {
+	} else if index.existenceFld != nil {
 		t.Fatalf("expected index.existenceField to be nil")
 	}
 }

From a89e1c521becfe0a7291cc4fcb25e79925e18f83 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Tue, 11 Sep 2018 14:52:03 -0500
Subject: [PATCH 10/11] benchmark import instead of set on existence field

---
 executor_test.go | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/executor_test.go b/executor_test.go
index 25465411c..5704e607b 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -1557,12 +1557,22 @@ func benchmarkExistence(nn bool, b *testing.B) {
 		b.Fatal(err)
 	}
 
+	bitCount := 10000
+	req := &pilosa.ImportRequest{
+		Index:     indexName,
+		Field:     fieldName,
+		Shard:     0,
+		RowIDs:    make([]uint64, bitCount),
+		ColumnIDs: make([]uint64, bitCount),
+	}
+	for i := 0; i < bitCount; i++ {
+		req.RowIDs[i] = uint64(rand.Intn(100000))
+		req.ColumnIDs[i] = uint64(rand.Intn(1 << 20))
+	}
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		colID := uint64(rand.Intn(1 << 20))
-		rowID := uint64(rand.Intn(100000))
-		qry := fmt.Sprintf(`Set(%d, %s=%d)`, colID, fieldName, rowID)
-		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: qry}); err != nil {
+		if err := c[0].API.Import(context.Background(), req); err != nil {
 			b.Fatal(err)
 		}
 	}

From 4ea48e1b40b9161e49c73aadc39a11d74ea82a12 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 21 Aug 2018 12:53:53 -0500
Subject: [PATCH 11/11] remove unused log buffers from test cluster, fixes race

the buffers were unused internally and external users had no access to them.
Those wishing to read the logs of the cluster in tests may replace stdout/stderr
with buffers on the Command struct.

The race occurred when a node was stopped and then started again. some
memberlist goroutines might not be completely cleaned up by the time the node
restarted, and then two loggers were using the same output buffer.
---
 test/pilosa.go | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/test/pilosa.go b/test/pilosa.go
index e7111ad92..9dbd78057 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
 	"io/ioutil"
 	gohttp "net/http"
 	"os"
@@ -40,10 +39,6 @@ type Command struct {
 	*server.Command
 
 	commandOptions []server.CommandOption
-
-	stdin  bytes.Buffer
-	stdout bytes.Buffer
-	stderr bytes.Buffer
 }
 
 func OptAllowedOrigins(origins []string) server.CommandOption {
@@ -65,17 +60,15 @@ func newCommand(opts ...server.CommandOption) *Command {
 	// beginning of the option slice so that it can be overridden by user-passed
 	// options.
 	opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...)
-	m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
+	m := &Command{commandOptions: opts}
+	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
 	m.Config.DataDir = path
 	m.Config.Bind = "http://localhost:0"
 	m.Config.Cluster.Disabled = true
-	m.Command.Stdin = &m.stdin
-	m.Command.Stdout = &m.stdout
-	m.Command.Stderr = &m.stderr
 
 	if testing.Verbose() {
-		m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
-		m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
+		m.Command.Stdout = os.Stdout
+		m.Command.Stderr = os.Stderr
 	}
 
 	return m
@@ -120,7 +113,7 @@ func (m *Command) Reopen() error {
 
 	// Create new main with the same config.
 	config := m.Command.Config
-	m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr, m.commandOptions...)
+	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, m.commandOptions...)
 	m.Command.Config = config
 
 	// Run new program.