Merge branch 'master' into n-to-int32

api.go

@@ -801,6 +801,8 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error {
 }
 
 // MaxShards returns the maximum shard number for each index in a map.
+// TODO (2.0): This method has been deprecated. Instead, use
+// AvailableShardsByIndex.
 func (api *API) MaxShards(_ context.Context) map[string]uint64 {
 	m := make(map[string]uint64)
 	for k, v := range api.holder.availableShardsByIndex() {

cache.go

@@ -167,8 +167,10 @@ func NewRankCache(maxEntries uint32) *rankCache {
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	// Ignore if the column count is below the threshold.
-	if n < c.thresholdValue {
+	// Ignore if the column count is below the threshold,
+	// unless the count is 0, which is effectively used
+	// to clear the cache value.
+	if n < c.thresholdValue && n > 0 {
 		return
 	}
 

docs/query-language.md

@@ -236,6 +236,36 @@ Clear(10, stargazer=1)
 
 This represents removing the relationship between the user with id=1 and the repository with id=10.
 
+#### ClearRow
+
+**Spec:**
+
+```
+ClearRow(<FIELD>=<ROW>)
+```
+
+**Description:**
+
+`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns.
+
+**Result Type:** boolean
+
+A return value of `true` indicates that at least one column was toggled from 1 to 0.
+
+A return value of `false` indicates that all bits in the row were already 0 and nothing changed.
+
+**Examples:**
+
+Clear all bit in row 1 in the stargazer field:
+```request
+ClearRow(stargazer=1)
+```
+```response
+{"results":[true]}
+```
+
+This represents removing the relationship between the user with id=1 and all repositories.
+
 ### Read Operations
 
 #### Row

encoding/proto/proto.go

@@ -1,3 +1,17 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package proto
 
 import (
@@ -803,30 +817,32 @@ func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent)
 
 func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) {
 	m.Node = &pilosa.Node{}
-	decodeIndexStatuses(pb.Indexes, m.Indexes)
+	m.Indexes = decodeIndexStatuses(pb.Indexes)
 	m.Schema = &pilosa.Schema{}
 	decodeSchema(pb.Schema, m.Schema)
 }
 
-func decodeIndexStatuses(a []*internal.IndexStatus, m []*pilosa.IndexStatus) {
-	m = m[:0]
+func decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.IndexStatus {
+	m := make([]*pilosa.IndexStatus, 0)
 	for i := range a {
 		m = append(m, &pilosa.IndexStatus{})
 		decodeIndexStatus(a[i], m[i])
 	}
+	return m
 }
 
 func decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) {
 	m.Name = pb.Name
-	decodeFieldStatuses(pb.Fields, m.Fields)
+	m.Fields = decodeFieldStatuses(pb.Fields)
 }
 
-func decodeFieldStatuses(a []*internal.FieldStatus, m []*pilosa.FieldStatus) {
-	m = m[:0]
+func decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.FieldStatus {
+	m := make([]*pilosa.FieldStatus, 0)
 	for i := range a {
 		m = append(m, &pilosa.FieldStatus{})
 		decodeFieldStatus(a[i], m[i])
 	}
+	return m
 }
 
 func decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) {

executor.go

@@ -182,6 +182,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s
 		return e.executeMax(ctx, index, c, shards, opt)
 	case "Clear":
 		return e.executeClearBit(ctx, index, c, opt)
+	case "ClearRow":
+		return e.executeClearRow(ctx, index, c, shards, opt)
 	case "Count":
 		e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
 		return e.executeCount(ctx, index, c, shards, opt)
@@ -1107,7 +1109,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
 	return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt)
 }
 
-// executeClearBitField executes a Clear() call for a single view.
+// executeClearBitField executes a Clear() call for a field.
 func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) {
 	shard := colID / ShardWidth
 	ret := false
@@ -1137,6 +1139,80 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq
 	return ret, nil
 }
 
+// executeClearRow executes a ClearRow() call.
+func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) {
+	// Ensure the field type supports ClearRow().
+	fieldName, err := c.FieldArg()
+	if err != nil {
+		return false, errors.New("ClearRow() argument required: field")
+	}
+	field := e.Holder.Field(index, fieldName)
+	if field == nil {
+		return false, ErrFieldNotFound
+	}
+
+	switch field.Type() {
+	case FieldTypeSet, FieldTypeTime, FieldTypeMutex, FieldTypeBool:
+		// These field types support ClearRow().
+	default:
+		return false, fmt.Errorf("ClearRow() is not supported on %s field types", field.Type())
+	}
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(shard uint64) (interface{}, error) {
+		return e.executeClearRowShard(ctx, index, c, shard)
+	}
+
+	// Merge returned results at coordinating node.
+	reduceFn := func(prev, v interface{}) interface{} {
+		val := v.(bool)
+		if prev == nil {
+			return val
+		}
+		return val || prev.(bool)
+	}
+
+	result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	return result.(bool), err
+}
+
+// executeClearRowShard executes a ClearRow() call for a single shard.
+func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) {
+	fieldName, err := c.FieldArg()
+	if err != nil {
+		return false, errors.New("ClearRow() argument required: field")
+	}
+
+	// Read fields using labels.
+	rowID, ok, err := c.UintArg(fieldName)
+	if err != nil {
+		return false, fmt.Errorf("reading ClearRow() row: %v", err)
+	} else if !ok {
+		return false, fmt.Errorf("ClearRow() row argument '%v' required", rowLabel)
+	}
+
+	field := e.Holder.Field(index, fieldName)
+	if field == nil {
+		return false, ErrFieldNotFound
+	}
+
+	// Remove the row from all views.
+	changed := false
+	for _, view := range field.views() {
+		fragment := e.Holder.fragment(index, fieldName, view.name, shard)
+		if fragment == nil {
+			continue
+		}
+		cleared, err := fragment.clearRow(rowID)
+		if err != nil {
+			return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard)
+		}
+		changed = changed || cleared
+	}
+
+	return changed, nil
+}
+
 // executeSet executes a Set() call.
 func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) {
 	// Read colID.

executor_test.go

@@ -1657,6 +1657,262 @@ func TestExecutor_Execute_Not(t *testing.T) {
 	}
 }
 
+// Ensure a row can be cleared.
+func TestExecutor_Execute_ClearRow(t *testing.T) {
+	t.Run("Set", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set bits.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
+			fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+
+		// Clear the row and ensure we get a `true` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); !res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Clear the row again and ensure we get a `false` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Ensure the row is empty.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+
+		// Ensure other rows were not affected.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+	})
+	t.Run("Mutex", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set bits.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` +
+			fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) +
+			fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) +
+			fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20),
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+
+		// Clear the row and ensure we get a `true` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); !res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Clear the row again and ensure we get a `false` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Ensure the row is empty.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+
+		// Ensure other rows were not affected.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil {
+			t.Fatal(err)
+		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) {
+			t.Fatalf("unexpected columns: %+v", bits)
+		}
+	})
+	t.Run("Time", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD")))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Set columns.
+		cc := `
+			Set(2, f=1, 1999-12-31T00:00)
+			Set(3, f=1, 2000-01-01T00:00)
+			Set(4, f=1, 2000-01-02T00:00)
+			Set(5, f=1, 2000-02-01T00:00)
+			Set(6, f=1, 2001-01-01T00:00)
+			Set(7, f=1, 2002-01-01T02:00)
+
+			Set(2, f=1, 1999-12-30T00:00)
+			Set(2, f=1, 2002-02-01T00:00)
+			Set(2, f=10, 2001-01-01T00:00)
+		`
+		rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`
+		rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`
+
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
+			t.Fatal(err)
+		}
+
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+
+		// Clear the row and ensure we get a `true` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); !res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Ensure the row is empty.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+
+		// Ensure other rows were not affected.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil {
+			t.Fatal(err)
+		} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) {
+			t.Fatalf("unexpected columns: %+v", columns)
+		}
+	})
+	t.Run("Int", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure that clearing a row raises an error.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil {
+			t.Fatal("expected clear row to return an error")
+		}
+	})
+	t.Run("TopN", func(t *testing.T) {
+		c := test.MustRunCluster(t, 1)
+		defer c.Close()
+		hldr := test.Holder{Holder: c[0].Server.Holder()}
+		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cc := `
+			Set(2, f=1)
+			Set(3, f=1)
+			Set(4, f=1)
+			Set(5, f=1)
+			Set(6, f=1)
+			Set(7, f=1)
+			Set(8, f=1)
+
+			Set(2, f=2)
+			Set(3, f=2)
+			Set(4, f=2)
+			Set(5, f=2)
+			Set(6, f=2)
+			Set(7, f=2)
+
+			Set(2, f=3)
+			Set(3, f=3)
+			Set(4, f=3)
+			Set(5, f=3)
+			Set(6, f=3)
+		`
+
+		// Set bits.
+		if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := c[0].RecalculateCaches(); err != nil {
+			t.Fatalf("recalculating caches: %v", err)
+		}
+
+		// Check the TopN results.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
+			{ID: 1, Count: 7},
+			{ID: 2, Count: 6},
+			{ID: 3, Count: 5},
+		}}) {
+			t.Fatalf("topn wrong results: %v", res.Results)
+		}
+
+		// Clear the row and ensure we get a `true` response.
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil {
+			t.Fatal(err)
+		} else if res := res.Results[0].(bool); !res {
+			t.Fatalf("unexpected clear row result: %+v", res)
+		}
+
+		// Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache).
+		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{
+			{ID: 1, Count: 7},
+			{ID: 3, Count: 5},
+		}}) {
+			t.Fatalf("topn wrong results: %v", res.Results)
+		}
+	})
+}
+
 func benchmarkExistence(nn bool, b *testing.B) {
 	c := test.MustRunCluster(b, 1)
 	defer c.Close()

field.go

@@ -230,8 +230,17 @@ func (f *Field) AvailableShards() *roaring.Bitmap {
 	return b
 }
 
-// addRemoteAvailableShards merges the set of available shards into the current known set.
-func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) {
+// addRemoteAvailableShards merges the set of available shards into the current known set
+// and saves the set to a file.
+func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error {
+	f.mergeRemoteAvailableShards(b)
+
+	// Save the updated bitmap to the data store.
+	return f.saveAvailableShards()
+}
+
+// mergeRemoteAvailableShards merges the set of available shards into the current known set.
+func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.remoteAvailableShards = f.remoteAvailableShards.Union(b)
@@ -291,6 +300,10 @@ func (f *Field) Open() error {
 			return errors.Wrap(err, "loading meta")
 		}
 
+		if err := f.loadAvailableShards(); err != nil {
+			return errors.Wrap(err, "loading available shards")
+		}
+
 		// Apply the field options loaded from meta.
 		if err := f.applyOptions(f.options); err != nil {
 			return errors.Wrap(err, "applying options")
@@ -467,6 +480,47 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 	return nil
 }
 
+// loadAvailableShards reads remoteAvailableShards data for the field, if any.
+func (f *Field) loadAvailableShards() error {
+	bm := roaring.NewBitmap()
+
+	// Read data from meta file.
+	buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards"))
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return errors.Wrap(err, "reading available shards")
+	} else {
+		if err := bm.UnmarshalBinary(buf); err != nil {
+			return errors.Wrap(err, "unmarshaling")
+		}
+	}
+
+	// Merge bitmap from file into field.
+	f.mergeRemoteAvailableShards(bm)
+
+	return nil
+}
+
+// saveAvailableShards writes remoteAvailableShards data for the field.
+func (f *Field) saveAvailableShards() error {
+	// Open or create file.
+	file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return errors.Wrap(err, "opening available shards file")
+	}
+
+	f.mu.RLock()
+	defer f.mu.RUnlock()
+
+	// Write available shards to file.
+	if _, err := f.remoteAvailableShards.WriteTo(file); err != nil {
+		return errors.Wrap(err, "writing bitmap to buffer")
+	}
+
+	return nil
+}
+
 // Close closes the field and its views.
 func (f *Field) Close() error {
 	f.mu.Lock()

field_internal_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/pql"
+	"github.com/pilosa/pilosa/roaring"
 )
 
 // Ensure a bsiGroup can adjust to its baseValue.
@@ -341,3 +342,22 @@ func TestField_RowTime(t *testing.T) {
 	}
 
 }
+
+func TestField_PersistAvailableShards(t *testing.T) {
+	f := MustOpenField(OptFieldTypeDefault())
+
+	// bm represents remote available shards.
+	bm := roaring.NewBitmap(1, 2, 3)
+
+	if err := f.addRemoteAvailableShards(bm); err != nil {
+		t.Fatal(err)
+	}
+
+	// Reload field and verify that shard data is persisted.
+	if err := f.Reopen(); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) {
+		t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice())
+	}
+
+}

fragment.go

@@ -391,12 +391,13 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
 // handleMutex will clear an existing row and store the new row
 // in the vector.
 func (f *fragment) handleMutex(rowID, columnID uint64) error {
-	if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID {
+	if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
+		return errors.Wrap(err, "getting mutex vector data")
+	} else if found && existingRowID != rowID {
 		if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil {
 			return errors.Wrap(err, "clearing mutex value")
 		}
 	}
-	f.mutexVector.Set(columnID, rowID)
 	return nil
 }
 
@@ -490,6 +491,45 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er
 	return changed, nil
 }
 
+// ClearRow clears a row for a given rowID within the fragment.
+// This updates both the on-disk storage and the in-cache bitmap.
+func (f *fragment) clearRow(rowID uint64) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.unprotectedClearRow(rowID)
+}
+
+func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) {
+	changed = false
+
+	// First container of the row in storage.
+	headContainerKey := rowID << shardVsContainerExponent
+
+	// Remove every container in the row.
+	for i := uint64(0); i < (1 << shardVsContainerExponent); i++ {
+		k := headContainerKey + i
+		// Technically we could bypass the Get() call and only
+		// call Remove(), but the Get() gives us the ability
+		// to return true if any existing data was removed.
+		if cont := f.storage.Containers.Get(k); cont != nil {
+			f.storage.Containers.Remove(k)
+			changed = true
+		}
+	}
+
+	// Clear the row in cache.
+	f.cache.Add(rowID, 0)
+
+	// Snapshot storage.
+	if err := f.snapshot(); err != nil {
+		return false, errors.Wrap(err, "snapshotting")
+	}
+
+	f.stats.Count("clearRow", 1, 1.0)
+
+	return changed, nil
+}
+
 func (f *fragment) bit(rowID, columnID uint64) (bool, error) {
 	pos, err := f.pos(rowID, columnID)
 	if err != nil {
@@ -1430,7 +1470,9 @@ func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error {
 			rowID, columnID := rowIDs[i], columnIDs[i]
 
 			// Handle mutex vector (i.e. clear an existing row).
-			if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID {
+			if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil {
+				return errors.Wrap(err, "getting mutex vector data")
+			} else if found && existingRowID != rowID {
 				// Determine the position of the bit in the storage.
 				pos, err := f.pos(existingRowID, columnID)
 				if err != nil {
@@ -2180,8 +2222,7 @@ func pos(rowID, columnID uint64) uint64 {
 // vector stores the mapping of colID to rowID.
 // It's used for a mutex field type.
 type vector interface {
-	Get(colID uint64) (uint64, bool)
-	Set(colID, rowID uint64)
+	Get(colID uint64) (uint64, bool, error)
 }
 
 // rowsVector implements the vector interface by looking
@@ -2200,17 +2241,16 @@ func newRowsVector(f *fragment) *rowsVector {
 // Get returns the rowID associated to the given colID.
 // Additionally, it returns true if a value was found,
 // otherwise it returns false.
-func (v *rowsVector) Get(colID uint64) (uint64, bool) {
+func (v *rowsVector) Get(colID uint64) (uint64, bool, error) {
 	rows := v.f.rowsForColumn(colID)
-	if len(rows) == 1 {
-		return rows[0], true
+	if len(rows) > 1 {
+		return 0, false, errors.New("found multiple row values for column")
+	} else if len(rows) == 1 {
+		return rows[0], true, nil
 	}
-	return 0, false
+	return 0, false, nil
 }
 
-// Set is not used for rowsVector.
-func (v *rowsVector) Set(colID, rowID uint64) {}
-
 // boolVector implements the vector interface by looking
 // at data in rows 0 and 1.
 type boolVector struct {
@@ -2227,16 +2267,17 @@ func newBoolVector(f *fragment) *boolVector {
 // Get returns the rowID associated to the given colID.
 // Additionally, it returns true if a value was found,
 // otherwise it returns false.
-func (v *boolVector) Get(colID uint64) (uint64, bool) {
+func (v *boolVector) Get(colID uint64) (uint64, bool, error) {
 	rows := v.f.rowsForColumn(colID)
-	if len(rows) == 1 {
+	if len(rows) > 1 {
+		return 0, false, errors.New("found multiple row values for column")
+	} else if len(rows) == 1 {
 		switch rows[0] {
 		case falseRowID, trueRowID:
-			return rows[0], true
+			return rows[0], true, nil
+		default:
+			return 0, false, errors.New("found non-boolean value")
 		}
 	}
-	return 0, false
+	return 0, false, nil
 }
-
-// Set is not used for boolVector.
-func (v *boolVector) Set(colID, rowID uint64) {}

fragment_internal_test.go

@@ -95,6 +95,33 @@ func TestFragment_ClearBit(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can clear a row.
+func TestFragment_ClearRow(t *testing.T) {
+	f := mustOpenFragment("i", "f", viewStandard, 0, "")
+	defer f.Close()
+
+	// Set and then clear bits on the fragment.
+	if _, err := f.setBit(1000, 1); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.setBit(1000, 65536); err != nil {
+		t.Fatal(err)
+	} else if _, err := f.unprotectedClearRow(1000); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify count on row.
+	if n := f.row(1000).Count(); n != 0 {
+		t.Fatalf("unexpected count: %d", n)
+	}
+
+	// Close and reopen the fragment & verify the data.
+	if err := f.reopen(); err != nil {
+		t.Fatal(err)
+	} else if n := f.row(1000).Count(); n != 0 {
+		t.Fatalf("unexpected count (reopen): %d", n)
+	}
+}
+
 // Ensure a fragment can set & read a value.
 func TestFragment_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {

pql/pql.peg

@@ -10,6 +10,7 @@ Call <-  'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close
        / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()}
        / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()}
        / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()}
+       / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()}
        / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()}
        / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()}
        / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() }

server.go

@@ -481,7 +481,9 @@ func (s *Server) receiveMessage(m Message) error {
 		if f == nil {
 			return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field)
 		}
-		f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard))
+		if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil {
+			return errors.Wrap(err, "adding remote available shards")
+		}
 	case *CreateIndexMessage:
 		opt := obj.Meta
 		_, err := s.holder.CreateIndex(obj.Index, *opt)
@@ -640,13 +642,15 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error {
 		for _, fs := range is.Fields {
 			f := s.holder.Field(is.Name, fs.Name)
 
-			// if we don't know about an field locally, log a error because
+			// if we don't know about a field locally, log an error because
 			// fields should be created and synced prior to shard creation
 			if f == nil {
 				s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name)
 				continue
 			}
-			f.addRemoteAvailableShards(fs.AvailableShards)
+			if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil {
+				return errors.Wrap(err, "adding remote available shards")
+			}
 		}
 	}
 

server/server_test.go

@@ -621,4 +621,75 @@ func TestMain_ImportTimestamp(t *testing.T) {
 	}
 }
 
+func TestClusterQueriesAfterRestart(t *testing.T) {
+	cluster := test.MustRunCluster(t, 3)
+	defer cluster.Close()
+	cmd1 := cluster[1]
+
+	cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{})
+	cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
+
+	// build a query to set the first bit in 100 shards
+	query := strings.Builder{}
+	for i := 0; i < 100; i++ {
+		query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth))
+	}
+	_, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "testidx",
+		Query: query.String(),
+	})
+	if err != nil {
+		t.Fatalf("setting 100 bits in 100 shards: %v", err)
+	}
+
+	results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "testidx",
+		Query: "Count(Row(testfield=0))",
+	})
+	if err != nil {
+		t.Fatalf("counting row: %v", err)
+	}
+	if results.Results[0].(uint64) != 100 {
+		t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
+	}
+
+	err = cmd1.Command.Close()
+	if err != nil {
+		t.Fatalf("closing node0: %v", err)
+	}
+
+	// confirm that cluster stops accepting queries after one node closes
+	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
+	}
+
+	// Create new main with the same config.
+	config := cmd1.Command.Config
+	config.Bind = cmd1.API.Node().URI.HostPort()
+
+	// this isn't necessary, but makes the test run way faster
+	config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port))
+	cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr)
+	cmd1.Command.Config = config
+	err = cmd1.Start()
+	if err != nil {
+		t.Fatalf("reopening node 0: %v", err)
+	}
+
+	for cmd1.API.State() != pilosa.ClusterStateNormal {
+		time.Sleep(time.Millisecond)
+	}
+
+	results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{
+		Index: "testidx",
+		Query: "Count(Row(testfield=0))",
+	})
+	if err != nil {
+		t.Fatalf("counting row: %v", err)
+	}
+	if results.Results[0].(uint64) != 100 {
+		t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0])
+	}
+}
+
 // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.