diff --git a/field.go b/field.go index dfc098237..341695816 100644 --- a/field.go +++ b/field.go @@ -316,18 +316,30 @@ func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) { // loadAvailableShards reads remoteAvailableShards data for the field, if any. func (f *Field) loadAvailableShards() error { - bm := roaring.NewBitmap() // Read data from meta file. path := filepath.Join(f.path, ".available.shards") buf, err := ioutil.ReadFile(path) + // doesn't exist: this is fine if os.IsNotExist(err) { return nil - } else if err != nil { - return errors.Wrap(err, "reading available shards") - } else { - if err := bm.UnmarshalBinary(buf); err != nil { - return errors.Wrap(err, "unmarshaling") + } + // some other problem: + if err != nil { + f.logger.Printf("available shards file present but unreadable, discarding: %v", err) + err = os.Remove(path) + if err != nil { + return errors.Wrap(err, "deleting corrupt available shards list") } + return nil + } + bm := roaring.NewBitmap() + if err = bm.UnmarshalBinary(buf); err != nil { + f.logger.Printf("available shards file corrupt, discarding: %v", err) + err = os.Remove(path) + if err != nil { + return errors.Wrap(err, "deleting corrupt available shards list") + } + return nil } // Merge bitmap from file into field. f.mergeRemoteAvailableShards(bm) diff --git a/field_internal_test.go b/field_internal_test.go index 3fe3dc83f..41cb5832b 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "math" "os" + "path/filepath" "reflect" "testing" "time" @@ -366,6 +367,62 @@ func TestField_PersistAvailableShards(t *testing.T) { } +func TestField_CorruptAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.AddRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + path := filepath.Join(f.path, ".available.shards") + + avail, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + n, err := avail.Write([]byte{23}) + if err != nil || n != 1 { + t.Fatal(err) + } + avail.Close() + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) { + t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice()) + } +} + +func TestField_TruncatedAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.AddRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + path := filepath.Join(f.path, ".available.shards") + + avail, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + avail.Close() + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), []uint64(nil)) { + t.Fatalf("unexpected available shards (reopen). expected: %#v, but got: %#v", []uint64{}, f.remoteAvailableShards.Slice()) + } +} + // Ensure that persisting available shards having a smaller footprint (for example, // when going from a bitmap to a smaller, RLE representation) succeeds. func TestField_PersistAvailableShardsFootprint(t *testing.T) {