From a26ebe3e23cecf92c531e57363a770dc1cd2bc0c Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Jul 2018 14:30:03 -0500 Subject: [PATCH 1/3] frag.rows() and frag.rowsForColumn() from #1496 --- fragment.go | 57 ++++++++++++++++++++++++++++++++++++++- fragment_internal_test.go | 28 +++++++++++++++++++ roaring/roaring.go | 6 ++--- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/fragment.go b/fragment.go index 835c1abe8..697f11f92 100644 --- a/fragment.go +++ b/fragment.go @@ -153,7 +153,6 @@ func (f *fragment) Open() error { pos := f.storage.Max() f.maxRowID = pos / ShardWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) - return nil }(); err != nil { f.close() @@ -1680,6 +1679,62 @@ func (f *fragment) readCacheFromArchive(r io.Reader) error { return nil } +func (f *fragment) rows() []uint64 { + i, _ := f.storage.Containers.Iterator(0) + rows := make([]uint64, 0) + + var lastRow uint64 + lastRow = math.MaxUint64 + + // Loop over the existing containers. + for i.Next() { + key, _ := i.Value() + + // virtual row for the current container + vRow := key >> 4 + + // skip dups + if vRow == lastRow { + continue + } + + rows = append(rows, vRow) + lastRow = vRow + } + return rows + +} + +func (f *fragment) rowsForColumn(columnID uint64) []uint64 { + colID := columnID % ShardWidth + i, _ := f.storage.Containers.Iterator(0) + + colKey := uint64(0) + colVal := uint16(colID & 0xFFFF) + + rows := make([]uint64, 0) + + // Loop over the existing containers. + for i.Next() { + key, c := i.Value() + + // virtual row for the current container + vRow := key >> 4 + + // column container key for virtual row + colKey = ((vRow * ShardWidth) + colID) >> 16 + + if colKey != key { + continue + } + + if c.Contains(colVal) { + rows = append(rows, vRow) + } + } + return rows +} + // FragmentBlock represents info about a subsection of the rows in a block. // This is used for comparing data in remote blocks for active anti-entropy. type FragmentBlock struct { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e665a10ba..f487ba40c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1278,3 +1278,31 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { } } } + +// Test Various methods of retrieving RowIDs +func TestFragment_RowsIteration(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + expected1 := make([]uint64, 0) + expected2 := make([]uint64, 0) + for i := uint64(100); i < uint64(200); i++ { + if _, err := f.setBit(i, i%2); err != nil { + t.Fatal(err) + } + expected1 = append(expected1, i) + if i%2 == 1 { + expected2 = append(expected2, i) + } + } + + ids := f.rows() + if !reflect.DeepEqual(expected1, ids) { + t.Fatalf("Do not match %v %v", expected1, ids) + + } + + ids = f.rowsForColumn(1) + if !reflect.DeepEqual(expected2, ids) { + t.Fatalf("Do not match %v %v", expected2, ids) + } +} diff --git a/roaring/roaring.go b/roaring/roaring.go index 3756d1db7..3a0418aa3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -177,7 +177,7 @@ func (b *Bitmap) Contains(v uint64) bool { if c == nil { return false } - return c.contains(lowbits(v)) + return c.Contains(lowbits(v)) } // Remove removes values from the bitmap. @@ -1272,8 +1272,8 @@ func (c *Container) runAdd(v uint16) bool { return true } -// contains returns true if v is in the container. -func (c *Container) contains(v uint16) bool { +// Contains returns true if v is in the container. +func (c *Container) Contains(v uint16) bool { if c.isArray() { return c.arrayContains(v) } else if c.isRun() { From 07abb505cc622cbdf40a628998343767820d234a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Jul 2018 15:06:51 -0500 Subject: [PATCH 2/3] add more testing to row iteration --- fragment_internal_test.go | 86 +++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index f487ba40c..0d4e9fa49 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1281,28 +1281,78 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { // Test Various methods of retrieving RowIDs func TestFragment_RowsIteration(t *testing.T) { - f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() - expected1 := make([]uint64, 0) - expected2 := make([]uint64, 0) - for i := uint64(100); i < uint64(200); i++ { - if _, err := f.setBit(i, i%2); err != nil { + t.Run("firstContainer", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + expectedAll := make([]uint64, 0) + expectedOdd := make([]uint64, 0) + for i := uint64(100); i < uint64(200); i++ { + if _, err := f.setBit(i, i%2); err != nil { + t.Fatal(err) + } + expectedAll = append(expectedAll, i) + if i%2 == 1 { + expectedOdd = append(expectedOdd, i) + } + } + + ids := f.rows() + if !reflect.DeepEqual(expectedAll, ids) { + t.Fatalf("Do not match %v %v", expectedAll, ids) + } + + ids = f.rowsForColumn(1) + if !reflect.DeepEqual(expectedOdd, ids) { + t.Fatalf("Do not match %v %v", expectedOdd, ids) + } + }) + + t.Run("secondRow", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + expected := []uint64{1, 2} + if _, err := f.setBit(1, 66000); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(2, 66000); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(2, 166000); err != nil { t.Fatal(err) } - expected1 = append(expected1, i) - if i%2 == 1 { - expected2 = append(expected2, i) + + ids := f.rows() + if !reflect.DeepEqual(expected, ids) { + t.Fatalf("Do not match %v %v", expected, ids) } - } - ids := f.rows() - if !reflect.DeepEqual(expected1, ids) { - t.Fatalf("Do not match %v %v", expected1, ids) + ids = f.rowsForColumn(66000) + if !reflect.DeepEqual(expected, ids) { + t.Fatalf("Do not match %v %v", expected, ids) + } + }) - } + t.Run("combinations", func(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() - ids = f.rowsForColumn(1) - if !reflect.DeepEqual(expected2, ids) { - t.Fatalf("Do not match %v %v", expected2, ids) - } + expectedRows := make([]uint64, 0) + for r := uint64(1); r < uint64(10000); r += 100 { + expectedRows = append(expectedRows, r) + for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 { + if _, err := f.setBit(r, c); err != nil { + t.Fatal(err) + } + + ids := f.rows() + if !reflect.DeepEqual(expectedRows, ids) { + t.Fatalf("Do not match %v %v", expectedRows, ids) + } + ids = f.rowsForColumn(c) + if !reflect.DeepEqual(expectedRows, ids) { + t.Fatalf("Do not match %v %v", expectedRows, ids) + } + } + } + }) } From dc5d7a66ef433f446105d0b802e7a6aad6b738aa Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Jul 2018 15:56:48 -0500 Subject: [PATCH 3/3] ensure that changing ShardWidth is supported in rowsForColumn() --- fragment.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index 697f11f92..371b4ef46 100644 --- a/fragment.go +++ b/fragment.go @@ -47,6 +47,13 @@ const ( // ShardWidth is the number of column IDs in a shard. ShardWidth = 1048576 + // containersPerRowSegment is dependent upon ShardWidth, + // and it represents the number of containers per shard row + // (or rowSegment). Since containers are set in roaring + // to be 2^16, then this const should be ShardWidth / 2^16. + // It is represented as the exponent n of 2^n. + containersPerRowSegment = 4 + // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" @@ -1691,7 +1698,7 @@ func (f *fragment) rows() []uint64 { key, _ := i.Value() // virtual row for the current container - vRow := key >> 4 + vRow := key >> containersPerRowSegment // skip dups if vRow == lastRow { @@ -1719,7 +1726,7 @@ func (f *fragment) rowsForColumn(columnID uint64) []uint64 { key, c := i.Value() // virtual row for the current container - vRow := key >> 4 + vRow := key >> containersPerRowSegment // column container key for virtual row colKey = ((vRow * ShardWidth) + colID) >> 16