scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-12 23:51:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

alter tests to allow for shardwidth22

Browse source 
also, reset BitDepth on field and bsiGroup during
importRoaringOverwrite
This commit is contained in:
Travis 2020-03-31 23:05:59 -05:00
parent da503776e2
commit 182c1d3c42
5 changed files with 101 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
boltdb/translate_test.go
View file

@ -32,23 +32,22 @@ func TestTranslateStore_TranslateKey(t *testing.T) {
defer MustCloseTranslateStore(s)
// Ensure initial key translates to first ID for shard
if id, err := s.TranslateKey("foo"); err != nil {
id1, err := s.TranslateKey("foo")
if err != nil {
t.Fatal(err)
} else if got, want := id, uint64(247463937); got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
}
// Ensure next key autoincrements.
if id, err := s.TranslateKey("bar"); err != nil {
t.Fatal(err)
} else if got, want := id, uint64(247463938); got != want {
} else if got, want := id, id1+1; got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
}
// Ensure retranslating existing key returns original ID.
if id, err := s.TranslateKey("foo"); err != nil {
t.Fatal(err)
} else if got, want := id, uint64(247463937); got != want {
} else if got, want := id, id1; got != want {
t.Fatalf("TranslateKey()=%d, want %d", got, want)
}
}
@ -58,31 +57,30 @@ func TestTranslateStore_TranslateKeys(t *testing.T) {
defer MustCloseTranslateStore(s)
// Ensure initial keys translate to incrementing IDs.
if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
ids1, err := s.TranslateKeys([]string{"foo", "bar"})
if err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(247463937); got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(247463938); got != want {
} else if got, want := ids1[1], ids1[0]+1; got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
}
// Ensure retranslation returns original IDs.
if ids, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(247463937); got != want {
} else if got, want := ids[0], ids1[0]; got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(247463938); got != want {
} else if got, want := ids[1], ids1[1]; got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
}
// Ensure retranslating with existing and non-existing keys returns correctly.
if ids, err := s.TranslateKeys([]string{"foo", "baz", "bar"}); err != nil {
t.Fatal(err)
} else if got, want := ids[0], uint64(247463937); got != want {
} else if got, want := ids[0], ids1[0]; got != want {
t.Fatalf("TranslateKeys()[0]=%d, want %d", got, want)
} else if got, want := ids[1], uint64(247463939); got != want {
} else if got, want := ids[1], ids1[0]+2; got != want {
t.Fatalf("TranslateKeys()[1]=%d, want %d", got, want)
} else if got, want := ids[2], uint64(247463938); got != want {
} else if got, want := ids[2], ids1[1]; got != want {
t.Fatalf("TranslateKeys()[2]=%d, want %d", got, want)
}
}
@ -154,7 +152,8 @@ func TestTranslateStore_EntryReader(t *testing.T) {
defer MustCloseTranslateStore(s)
// Create multiple new keys.
if _, err := s.TranslateKeys([]string{"foo", "bar"}); err != nil {
ids1, err := s.TranslateKeys([]string{"foo", "bar"})
if err != nil {
t.Fatal(err)
}
@ -169,7 +168,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Read first entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(247463937); got != want {
} else if got, want := entry.ID, ids1[0]; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "foo"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -178,21 +177,22 @@ func TestTranslateStore_EntryReader(t *testing.T) {
// Read next entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(247463938); got != want {
} else if got, want := entry.ID, ids1[1]; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "bar"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
}
// Insert next key while reader is open.
if _, err := s.TranslateKey("baz"); err != nil {
id2, err := s.TranslateKey("baz")
if err != nil {
t.Fatal(err)
}
// Read newly created entry.
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(247463939); got != want {
} else if got, want := entry.ID, id2; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "baz"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -216,20 +216,25 @@ func TestTranslateStore_EntryReader(t *testing.T) {
}
defer r.Close()
// cache holds the translated key id so we can check it later
cache := make(chan uint64)
// Insert key in separate goroutine.
// Sleep momentarily to reader hangs.
translateErr := make(chan error)
go func() {
time.Sleep(100 * time.Millisecond)
if _, err := s.TranslateKey("foo"); err != nil {
id, err := s.TranslateKey("foo")
if err != nil {
translateErr <- err
}
cache <- id
}()
var entry pilosa.TranslateEntry
if err := r.ReadEntry(&entry); err != nil {
t.Fatal(err)
} else if got, want := entry.ID, uint64(247463937); got != want {
} else if got, want := entry.ID, <-cache; got != want {
t.Fatalf("ReadEntry() ID=%d, want %d", got, want)
} else if got, want := entry.Key, "foo"; got != want {
t.Fatalf("ReadEntry() Key=%s, want %s", got, want)
@ -340,7 +345,8 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
}
// Populate the store with the keys in batch0.
if _, err := s.TranslateKeys(batch0); err != nil {
batch0IDs, err := s.TranslateKeys(batch0)
if err != nil {
t.Fatal(err)
}
@ -356,11 +362,12 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
}
// Populate the store with the keys in batch1.
if _, err := s.TranslateKeys(batch1); err != nil {
batch1IDs, err := s.TranslateKeys(batch1)
if err != nil {
t.Fatal(err)
}
expIDs := []uint64{247463987, 247464087}
expIDs := []uint64{batch0IDs[50], batch1IDs[50]}
// Check the IDs for a key from each batch.
if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
@ -378,13 +385,12 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
// This time, we expect the second key to be different because
// we overwrote the store, and then just set that key.
expIDs = []uint64{247463987, 247464037}
// Check the IDs for a key from each batch.
if ids, err := s.TranslateKeys([]string{"key50", "key150"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(expIDs, ids) {
t.Fatalf("last expected ids: %v, but got: %v", expIDs, ids)
} else if ids[0] != expIDs[0] {
t.Fatalf("last expected ids[0]: %d, but got: %d", expIDs[0], ids[0])
} else if ids[1] == expIDs[1] {
t.Fatalf("last expected different ids[1]: %d, but got: %d", expIDs[1], ids[1])
}
})
}

24
field.go
View file

@ -1839,6 +1839,30 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, data []byte, shard u
return err
}
// If field is int or decimal, then we need to update field.options.BitDepth
// and bsiGroup.BitDepth based on the imported data.
switch f.Options().Type {
case FieldTypeInt, FieldTypeDecimal:
frag.calculateMaxRowID()
maxRowID, _ := frag.maxRow(nil)
var bitDepth uint
if maxRowID+1 > bsiOffsetBit {
bitDepth = uint(maxRowID + 1 - bsiOffsetBit)
}
bsig := f.bsiGroup(f.name)
f.mu.Lock()
defer f.mu.Unlock()
if bitDepth > f.options.BitDepth {
f.options.BitDepth = bitDepth
}
if bsig != nil {
bsig.BitDepth = bitDepth
}
}
return nil
}

6
fragment.go
View file

@ -1142,6 +1142,12 @@ func (f *fragment) maxRow(filter *Row) (uint64, uint64) {
return 0, 0
}
// calculateMaxRowID determines the field's maxRowID value based
// on the contents of its storage, and sets the struct argument.
func (f *fragment) calculateMaxRowID() {
f.maxRowID = f.storage.Max() / ShardWidth
}
// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *fragment) rangeOp(op pql.Token, bitDepth uint, predicate int64) (*Row, error) {
switch op {

16
fragment_internal_test.go
View file

@ -3641,7 +3641,21 @@ func TestRemapCache(t *testing.T) {
func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
f := mustOpenFragment("i", "f", viewStandard, 0, "")
b := []byte{60, 48, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
// byShardWidth is a map of the same roaring (fragment) data generated
// with different shard widths.
// TODO: a better approach may be to generate this in the test based
// on shard width.
byShardWidth := make(map[uint64][]byte)
// row/col: 1/1
byShardWidth[1<<20] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
byShardWidth[1<<22] = []byte{60, 48, 0, 0, 1, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 24, 0, 0, 0, 1, 0}
var b []byte
if data, ok := byShardWidth[ShardWidth]; ok {
b = data
}
defer f.Clean(t)
err := f.importRoaringT(b, false)
if err != nil {

24
server/handler_test.go
View file

@ -240,7 +240,20 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
}
w := httptest.NewRecorder()
roaringData, _ := hex.DecodeString("3C30000002000000000000000000000001000000200000000000000001000000280000002A00000001000100")
// byShardWidth is a map of the same roaring (fragment) data generated
// with different shard widths.
// TODO: a better approach may be to generate this in the test based
// on shard width.
byShardWidth := make(map[uint64][]byte)
// col/val: 3/3, 8/8
byShardWidth[1<<20] = []byte{60, 48, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 72, 0, 0, 0, 76, 0, 0, 0, 78, 0, 0, 0, 80, 0, 0, 0, 3, 0, 8, 0, 3, 0, 3, 0, 8, 0}
byShardWidth[1<<22] = []byte{60, 48, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 128, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 72, 0, 0, 0, 76, 0, 0, 0, 78, 0, 0, 0, 80, 0, 0, 0, 3, 0, 8, 0, 3, 0, 3, 0, 8, 0}
var roaringData []byte
if data, ok := byShardWidth[pilosa.ShardWidth]; ok {
roaringData = data
}
msg := pilosa.ImportRoaringRequest{
Action: pilosa.RequestActionOverwrite,
@ -263,7 +276,7 @@ func TestHandler_Endpoints(t *testing.T) {
if err != nil {
t.Fatalf("querying: %v", err)
}
if row := resp.Results[0].(*pilosa.Row); !reflect.DeepEqual(row.Columns(), []uint64{1}) {
if row := resp.Results[0].(*pilosa.Row); !reflect.DeepEqual(row.Columns(), []uint64{3, 8}) {
t.Fatalf("Unexpected result %v", row.Columns())
}
})
@ -1068,7 +1081,12 @@ func TestHandler_Endpoints(t *testing.T) {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
target := []uint64{162529281, 159383553, 160432129}
var target []uint64
if pilosa.ShardWidth == 1<<22 {
target = []uint64{650117121, 637534209, 641728513}
} else {
target = []uint64{162529281, 159383553, 160432129}
}
resp := pilosa.TranslateKeysResponse{}
err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp)
if err != nil {