From bf7a1bc08a84640e452c795ae22069784244259e Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 12 Jun 2017 11:51:24 -0500 Subject: [PATCH] WIP bug fix, cleanup, additional tests --- fragment_test.go | 72 +++++++++++++++++++++ roaring/roaring.go | 105 ++++++++++++++++++++++++------- roaring/roaring_internal_test.go | 36 ++--------- 3 files changed, 159 insertions(+), 54 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index f6b36f11e..9ed53f24b 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -17,6 +17,7 @@ package pilosa_test import ( "bytes" "flag" + "fmt" "io/ioutil" "math" "os" @@ -248,6 +249,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { + //fmt.Printf("testfragment: setting %d, %d\n", i, j) f.MustSetBits(i, j) } } @@ -272,6 +274,45 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } } +// temporary test for catching op log bug +func TestFragment_TopN_Intersect_Moderate(t *testing.T) { + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + defer f.Close() + + // Create an intersecting input row. + src := pilosa.NewBitmap( + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + ) + + // Set bits on rows 0 - 99. Higher rows have higher bit counts. + for i := uint64(0); i < 100; i++ { + for j := uint64(0); j < i; j++ { + //fmt.Printf("testfragment: setting %d, %d\n", i, j) + f.MustSetBits(i, j) + } + } + f.RecalculateCache() + + // Retrieve top rows. + if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + {ID: 99, Count: 19}, + {ID: 98, Count: 18}, + {ID: 97, Count: 17}, + {ID: 96, Count: 16}, + {ID: 95, Count: 15}, + {ID: 94, Count: 14}, + {ID: 93, Count: 13}, + {ID: 92, Count: 12}, + {ID: 91, Count: 11}, + {ID: 90, Count: 10}, + }) { + t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) + } +} + // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) @@ -789,3 +830,34 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { t.Fatalf("unexpected pair(1): %v", pairs[2]) } } + +func TestFragment_Snapshot_Run(t *testing.T) { + fmt.Printf("mustopenfragment\n") + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + fmt.Printf("\n---\nset bits\n") + // Set bits on the fragment. + for i := uint64(1); i < 3; i++ { + if _, err := f.SetBit(1000, i); err != nil { + t.Fatal(err) + } + } + + fmt.Printf("\n---\nsnapshot\n") + // Snapshot bitmap and verify data. + if err := f.Snapshot(); err != nil { + t.Fatal(err) + } else if n := f.Row(1000).Count(); n != 2 { + t.Fatalf("unexpected count: %d", n) + } + + fmt.Printf("\n---\nreopen\n") + // Close and reopen the fragment & verify the data. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if n := f.Row(1000).Count(); n != 2 { + t.Fatalf("unexpected count (reopen): %d", n) + } +} + diff --git a/roaring/roaring.go b/roaring/roaring.go index 25c8c6d53..5cf82a348 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -26,9 +26,17 @@ import ( ) const ( - // cookie is the first four bytes in a roaring bitmap file. - cookieNoRuns = uint32(12346) - cookie = uint32(12347) + // magicNumber is an identifier, in bytes 0-1 of the file. + magicNumberNoRuns = uint32(12346) + magicNumber = uint32(12347) + + // storageVersion indicates the storage version, in bytes 2-3. + storageVersion = uint32(0) + + // cookie is the first four bytes in a roaring bitmap file, + // formed by joining magicNumber and storageVersion + cookieNoRuns = magicNumberNoRuns<<16 + storageVersion + cookie = magicNumber<<16 + storageVersion // headerBaseSize is the size of the cookie and key count at the beginning of a file. // Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8. @@ -661,7 +669,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Map byte slice directly to the container data. c := b.containers[i] if c.n <= ArrayMaxSize { - if containsRuns && (runFlagBitset[uint8(i)/8]&(1<= len(c.runs)-1 { + r := c.runs[itr.j] + runLength := int(r.last - r.start) + + if itr.k >= runLength { + itr.i++ + itr.j = -1 + continue + } else { + itr.k++ + return itr.peek(), false + } + } + + r := c.runs[itr.j] + runLength := int(r.last - r.start) + + if itr.k >= runLength { + itr.k = 0 + itr.j++ + } else { + itr.k++ + } + + return itr.peek(), false + } + // Move to the next possible index in the bitmap container. itr.j++ @@ -904,6 +950,9 @@ func (itr *Iterator) peek() uint64 { if c.isArray() { return uint64(key)<<16 | uint64(c.array[itr.j]) } + if c.isRun() { + return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint32(itr.k)) + } return uint64(key)<<16 | uint64(itr.j) } @@ -1002,10 +1051,12 @@ func (c *container) isArray() bool { return c.bitmap == nil && c.runs == nil } +// isBitmap returns true if the container is a bitmap container func (c *container) isBitmap() bool { return c.array == nil && c.runs == nil } +// isRun returns true if the container is a run-length-encoded container func (c *container) isRun() bool { return c.array == nil && c.bitmap == nil } @@ -1582,7 +1633,7 @@ func (c *container) size() int { if c.isArray() { return len(c.array) * 4 } else if c.isRun() { - return len(c.runs) * 8 + return len(c.runs)*8 + 2 } else { return len(c.bitmap) * 8 } @@ -1597,7 +1648,7 @@ func (c *container) info() ContainerInfo { info.Alloc = len(c.array) * 4 } else if c.isRun() { info.Type = "run" - info.Alloc = len(c.runs) * 8 + info.Alloc = len(c.runs)*8 + 2 } else { info.Type = "bitmap" info.Alloc = len(c.bitmap) * 8 @@ -2323,26 +2374,28 @@ func differenceArrayArray(a, b *container) *container { func differenceArrayRun(a, b *container) *container { // func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - if b.n == 0 { + if a.n == 0 || b.n == 0 { return a.clone() } output := &container{array: make([]uint32, 0, a.n)} // cardinality upper bound: card(A) - i := 0 // array index - j := 0 // run index + i := 0 // array index + j := 0 // run index // keep all array elements before beginning of runs - for ; i < b.runs[j].start; i++ { + for ; i < int(b.runs[j].start); i++ { output.array = append(output.array, a.array[i]) } // handle overlap for ; i < a.n; i++ { + // if array element in run, keep if !(a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last) { output.array = append(output.array, a.array[i]) } + // update current run if i >= int(b.runs[j].last) { j++ if j == len(b.runs) { @@ -2354,7 +2407,7 @@ func differenceArrayRun(a, b *container) *container { // keep all array elements after end of runs output.array = append(output.array, a.array[i:]...) - + return output } @@ -2372,27 +2425,33 @@ func differenceBitmapRun(a, b *container) *container { func differenceRunArray(a, b *container) *container { // TODO + if a.n == 0 || b.n == 0 { + return a.clone() + } + output := &container{runs: make([]interval32, 0, a.n)} return output } func differenceRunBitmap(a, b *container) *container { + // TODO + if a.n == 0 || b.n == 0 { + return a.clone() + } + output := &container{runs: make([]interval32, 0, a.n)} itr := newBufIterator(newBitmapIterator(b.bitmap)) - fmt.Printf("\ndifferenceRunBitmap\n") for i := 0; ; { vb, eof := itr.next() - if eof || { + if eof { break } - - fmt.Println(i, vb, eof) + fmt.Println(i, vb) i++ } - return output } @@ -2403,8 +2462,8 @@ func differenceRunRun(a, b *container) *container { return a.clone() } - apos := 0 // current a-run index - bpos := 0 // current b-run index + apos := 0 // current a-run index + bpos := 0 // current b-run index astart := a.runs[apos].start alast := a.runs[apos].last bstart := b.runs[bpos].start @@ -2412,7 +2471,7 @@ func differenceRunRun(a, b *container) *container { alen := len(a.runs) blen := len(b.runs) - output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else + output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index deb92f34e..526d4f194 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -996,36 +996,6 @@ func TestUnionBitmapRun(t *testing.T) { } } -func TestRunToArray(t *testing.T) { - a := &container{} - tests := []struct { - runs []interval32 - exp []uint32 - }{ - { - runs: []interval32{{start: 0, last: 0}}, - exp: []uint32{0}, - }, - { - runs: []interval32{{start: 0, last: 4}}, - exp: []uint32{0, 1, 2, 3, 4}, - }, - { - runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}}, - exp: []uint32{2, 5, 6, 7, 13, 14, 17}, - }, - } - - for i, test := range tests { - a.runs = test.runs - a.n = len(test.exp) - a.runToArray() - if !reflect.DeepEqual(a.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) - } - } -} - func TestBitmapCountRuns(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN)} tests := []struct { @@ -1133,6 +1103,7 @@ func TestDifferenceArrayRun(t *testing.T) { } for i, test := range tests { a.array = test.array + a.n = len(a.array) b.runs = test.runs b.n = b.runCountRange(0, 100) ret := differenceArrayRun(a, b) @@ -1160,6 +1131,7 @@ func TestDifferenceRunArray(t *testing.T) { a.runs = test.runs a.n = a.runCountRange(0, 100) b.array = test.array + b.n = len(b.array) ret := differenceRunArray(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -1187,6 +1159,7 @@ func TestDifferenceRunBitmap(t *testing.T) { for i, v := range test.bitmap { b.bitmap[i] = v } + b.n = b.bitmapCountRange(0, 100) ret := differenceRunBitmap(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -1212,11 +1185,12 @@ func TestDifferenceBitmapRun(t *testing.T) { for i, v := range test.bitmap { a.bitmap[i] = v } + a.n = a.bitmapCountRange(0, 100) b.runs = test.runs b.n = b.runCountRange(0, 100) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap[:len(test.exp)]) + t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) } } }