From 2b0cd2f234aa345a1b64bbe9f85538abd909f21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Sat, 6 Jun 2020 01:02:23 +0200 Subject: [PATCH 1/4] Add unionRunRunInPlace --- roaring/roaring.go | 142 ++++++++++++++++- roaring/roaring_internal_test.go | 262 +++++++++++++++++++++++++++++++ 2 files changed, 402 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index cbb7dc583..dd13a99de 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3134,8 +3134,7 @@ func (c *Container) unionInPlace(other *Container) *Container { c = c.runToBitmap() return unionBitmapArrayInPlace(c, other) case containerRun: - c = c.runToBitmap() - return unionBitmapRunInPlace(c, other) + return unionRunRunInPlace(c, other) } } if roaringParanoia { @@ -4715,6 +4714,145 @@ func unionBitmapBitmapInPlace(a, b *Container) *Container { return a } +// unions run b into run a, mutating a in place. +func unionRunRunInPlace(a, b *Container) *Container { + statsHit("unionInPlace/RunRun") + + a = a.Thaw() + runs, n := unionInterval16InPlace(a.runs(), b.runs()) + + a.setN(n) + a.setRuns(runs) + return a +} + +func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { + n := int32(0) + + an, bn := len(a), len(b) + i, j, k := 0, 0, 0 + for { + // select the next interval - v + var ( + v *interval16 + local *interval16 + visa bool // v is from a + ) + if i < an && j < bn { + if a[i].start <= b[j].start { + v, visa = &a[i], true + i++ + } else if j < bn { + v = &b[j] + j++ + } + } else { + if i < an { + v, visa = &a[i], true + i++ + } else if j < bn { + v = &b[j] + j++ + } + } + // no more intervals - break + if v == nil { + if len(a) > 0 { + // count the last interval + n += int32(a[k].last-a[k].start) + 1 + } + break + } + + next: + if k == an { + // append what's left + a = append(a, *v) + an++ + continue + } + + // current, locally unioned interval + if local == nil { + local = &a[k] + } + + if v.last < local.start { + // [---- v ----][---- local ----] + // insert v into a + a = append(a, interval16{}) + copy(a[i+1:], a[i:]) + a[i] = *v + an++ + i++ + continue + } + + if v.start > local.last { + // [---- local ----][---- v ----] + // set already unioned interval and go next + // if int32(v.start-local.last) <= 1 { + // local.last = v.last + // continue + // } + + a[k] = *local + n += int32(local.last-local.start) + 1 + k++ + + // if we set unioned interval we can go to the next interval + if visa { + // v is from a, so we can skip "digested intervals" + // and jump already to v + local = v + } else { + // ...otherwise, move to the next one, + // so let assign local in next iteration + local = nil + } + + goto next + } + + if v.start < local.start { + if v.last >= local.start && v.last <= local.last { + // [---- local ----] + // [---- v ----] + local.start = v.start + } else if v.last > local.last { + // [- local -] + // [---- v ----] + *local = *v + } + continue + } + + if v.start >= local.start && v.start <= local.last { + if v.last <= local.last { + // [---- local ----] + // [-- v --] + + // this assignment looks silly, but if we extended local + // and "digested" many nested intervals, we can squash a, e.g.: + // [a1] [a2] [a3] [a4] + // [------ b ------] + // will give us: + // [ ----- a1 -----] [a4] + a[k] = *local + } else if v.last > local.last { + // [---- local ----] + // [---- v ----] + local.last = v.last + } + } + } + + if len(a) > 0 { + a = a[:k+1] + } + return a, n +} + func difference(a, b *Container) *Container { if a.N() == 0 || b.N() == maxContainerVal+1 { return nil diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 3a0749cc8..665ac9857 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -721,6 +721,268 @@ func TestUnionMixed(t *testing.T) { }) } +func TestUnionInterval16InPlace(t *testing.T) { + t.Run("single overlap", func(t *testing.T) { + a := []interval16{interval16{1, 10}, interval16{21, 28}} + b := []interval16{interval16{8, 12}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 20 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 12}, interval16{21, 28}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("nested intervals", func(t *testing.T) { + a := []interval16{interval16{3, 13}, interval16{17, 20}} + b := []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 20 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 13}, interval16{14, 20}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("no overlap", func(t *testing.T) { + a := []interval16{interval16{3, 4}, interval16{7, 8}} + b := []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 10 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 6}, interval16{7, 8}, interval16{9, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("b in a", func(t *testing.T) { + a := []interval16{interval16{1, 10}} + b := []interval16{interval16{5, 7}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 10 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("a eq b", func(t *testing.T) { + a := []interval16{interval16{1, 10}} + b := []interval16{interval16{1, 10}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 10 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("a in b", func(t *testing.T) { + a := []interval16{interval16{5, 7}} + b := []interval16{interval16{1, 10}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 10 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("a ahead b", func(t *testing.T) { + a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + b := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 13 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{ + interval16{1, 2}, + interval16{3, 4}, + interval16{5, 7}, + interval16{10, 11}, + interval16{12, 13}, + interval16{14, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("b ahead a", func(t *testing.T) { + a := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} + b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 13 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{ + interval16{1, 2}, + interval16{3, 4}, + interval16{5, 7}, + interval16{10, 11}, + interval16{12, 13}, + interval16{14, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("empty a and b", func(t *testing.T) { + a := []interval16{} + b := []interval16{} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 0 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("empty a", func(t *testing.T) { + a := []interval16{} + b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 7 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("empty b", func(t *testing.T) { + a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + b := []interval16{} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 7 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("single a", func(t *testing.T) { + a := []interval16{interval16{1, 2}} + b := []interval16{} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 2 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("single b", func(t *testing.T) { + a := []interval16{} + b := []interval16{interval16{1, 2}} + + runs, n := unionInterval16InPlace(a, b) + + const expectedN = 2 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + }) + + t.Run("oddBitsSet lastBitUnset", func(t *testing.T) { + a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} + b := []interval16{interval16{0, 4}} + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{0, 4}, interval16{5, 5}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 6 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + }) +} + func TestIntersectMixed(t *testing.T) { a := NewContainerRun([]interval16{{start: 5, last: 10}}) b := NewContainerArray([]uint16{1, 4, 5, 7, 10, 11, 12}) From eddbb7d0b00f0eb2a48f2171a98fa53789ca0230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 10 Jun 2020 17:36:30 +0200 Subject: [PATCH 2/4] Optimize run intervals by combnining neighbours --- roaring/roaring.go | 81 +++----- roaring/roaring_internal_test.go | 330 +++++++++++++++++++++++++------ 2 files changed, 298 insertions(+), 113 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index dd13a99de..4cc1f9751 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4721,8 +4721,8 @@ func unionRunRunInPlace(a, b *Container) *Container { a = a.Thaw() runs, n := unionInterval16InPlace(a.runs(), b.runs()) - a.setN(n) a.setRuns(runs) + a.setN(n) return a } @@ -4732,57 +4732,50 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { an, bn := len(a), len(b) i, j, k := 0, 0, 0 for { - // select the next interval - v var ( - v *interval16 + v interval16 local *interval16 - visa bool // v is from a ) + if i < an && j < bn { if a[i].start <= b[j].start { - v, visa = &a[i], true + v = a[i] i++ - } else if j < bn { - v = &b[j] + } else { + v = b[j] j++ } } else { if i < an { - v, visa = &a[i], true + v = a[i] i++ } else if j < bn { - v = &b[j] + v = b[j] j++ + } else { + break } } - // no more intervals - break - if v == nil { - if len(a) > 0 { - // count the last interval - n += int32(a[k].last-a[k].start) + 1 - } - break - } - next: + reeval: if k == an { - // append what's left - a = append(a, *v) + // append what's left and try to optimize it later + a = append(a, v) an++ continue } - // current, locally unioned interval + // locally unioned interval if local == nil { local = &a[k] } if v.last < local.start { // [---- v ----][---- local ----] - // insert v into a + // prepend v into a a = append(a, interval16{}) copy(a[i+1:], a[i:]) - a[i] = *v + a[i] = v an++ i++ continue @@ -4790,28 +4783,19 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { if v.start > local.last { // [---- local ----][---- v ----] - // set already unioned interval and go next - // if int32(v.start-local.last) <= 1 { - // local.last = v.last - // continue - // } - a[k] = *local - n += int32(local.last-local.start) + 1 - k++ - - // if we set unioned interval we can go to the next interval - if visa { - // v is from a, so we can skip "digested intervals" - // and jump already to v - local = v - } else { - // ...otherwise, move to the next one, - // so let assign local in next iteration - local = nil + // check if we can optimize run and combine neighbours + if int32(v.start-local.last) <= 1 { + local.last = v.last + continue } - goto next + // set already unioned interval and reevaluate v + a[k] = *local + k++ + n += int32(local.last-local.start) + 1 + local = &v + goto reeval } if v.start < local.start { @@ -4822,18 +4806,15 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { } else if v.last > local.last { // [- local -] // [---- v ----] - *local = *v + *local = v } - continue - } - - if v.start >= local.start && v.start <= local.last { + } else if v.start <= local.last { if v.last <= local.last { // [---- local ----] // [-- v --] - // this assignment looks silly, but if we extended local - // and "digested" many nested intervals, we can squash a, e.g.: + // this assignment overwrites the first available position to set - a[k] + // it's done to "swallow" many nested intervals, e.g.: // [a1] [a2] [a3] [a4] // [------ b ------] // will give us: @@ -4848,6 +4829,8 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { } if len(a) > 0 { + // count the latest interval + n += int32(a[k].last-a[k].start) + 1 a = a[:k+1] } return a, n diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 665ac9857..1f5fe572b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -725,16 +725,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single overlap", func(t *testing.T) { a := []interval16{interval16{1, 10}, interval16{21, 28}} b := []interval16{interval16{8, 12}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 12}, interval16{21, 28}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 20 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 12}, interval16{21, 28}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -742,16 +752,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("nested intervals", func(t *testing.T) { a := []interval16{interval16{3, 13}, interval16{17, 20}} b := []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 20}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 20 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 13}, interval16{14, 20}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -759,16 +779,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("no overlap", func(t *testing.T) { a := []interval16{interval16{3, 4}, interval16{7, 8}} b := []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 6}, interval16{7, 8}, interval16{9, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -776,16 +806,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("b in a", func(t *testing.T) { a := []interval16{interval16{1, 10}} b := []interval16{interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -793,16 +833,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a eq b", func(t *testing.T) { a := []interval16{interval16{1, 10}} b := []interval16{interval16{1, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -810,16 +860,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a in b", func(t *testing.T) { a := []interval16{interval16{5, 7}} b := []interval16{interval16{1, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -827,23 +887,29 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a ahead b", func(t *testing.T) { a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} b := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{ + interval16{1, 7}, + interval16{10, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 13 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{ - interval16{1, 2}, - interval16{3, 4}, - interval16{5, 7}, - interval16{10, 11}, - interval16{12, 13}, - interval16{14, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -851,24 +917,29 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("b ahead a", func(t *testing.T) { a := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{ + interval16{1, 7}, + interval16{10, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 13 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{ - interval16{1, 2}, - interval16{3, 4}, - interval16{5, 7}, - interval16{10, 11}, - interval16{12, 13}, - interval16{14, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -876,17 +947,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty a and b", func(t *testing.T) { a := []interval16{} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 0 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -894,17 +974,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty a", func(t *testing.T) { a := []interval16{} b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 7 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -912,17 +1001,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty b", func(t *testing.T) { a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 7 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -930,17 +1028,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single a", func(t *testing.T) { a := []interval16{interval16{1, 2}} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 2 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -948,28 +1055,66 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single b", func(t *testing.T) { a := []interval16{} b := []interval16{interval16{1, 2}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 2 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}} + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("single a single b", func(t *testing.T) { + a := []interval16{interval16{3, 4}} + b := []interval16{interval16{1, 2}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{1, 4}} for i, v := range expected { if runs[i] != v { t.Fatalf("runs expected: %+v, got: %+v", expected, runs) } } + + const expectedN = 4 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } }) t.Run("oddBitsSet lastBitUnset", func(t *testing.T) { a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} b := []interval16{interval16{0, 4}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) - expected := []interval16{interval16{0, 4}, interval16{5, 5}} + expected := []interval16{interval16{0, 5}} for i, v := range expected { if runs[i] != v { t.Fatalf("runs expected: %+v, got: %+v", expected, runs) @@ -980,6 +1125,63 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Fatalf("N expected: %d, got: %d", expectedN, n) } + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("all bits", func(t *testing.T) { + a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} + b := []interval16{interval16{0, 0}, interval16{2, 2}, interval16{4, 4}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{0, 5}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 6 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("short a long b", func(t *testing.T) { + a := []interval16{interval16{5, 5}, interval16{7, 7}, interval16{9, 10}, interval16{12, 12}, interval16{15, 17}, interval16{19, 20}} + b := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 18}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 20}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 18 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } }) } From 443f2d8f7c0699ac4475a393cae9a208199ccf94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Thu, 18 Jun 2020 15:27:31 +0200 Subject: [PATCH 3/4] The third attempt to implement unionRunRunInPlace --- roaring/roaring.go | 250 +++++++----- roaring/roaring_internal_test.go | 628 +++++++++---------------------- 2 files changed, 333 insertions(+), 545 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 4cc1f9751..a8de21d6c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4726,116 +4726,186 @@ func unionRunRunInPlace(a, b *Container) *Container { return a } +// unionInterval16InPlace merges two slice of intervals in place (in a). +// The main concept is to go value by value (instead of interval by interval) +// and count `.start` and `.last` points. +// If we get the `state == 0` it means we just built a new interval (`val`), +// and we can set it in `a` at the possition `off` func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { n := int32(0) - an, bn := len(a), len(b) - i, j, k := 0, 0, 0 + + var ( + // ai - index of a, aii - subindex (0: a[ai].start, 1: a[ai].last). + ai, aii int = 0, 0 + + // bi - index of b, bii - subindex (0: b[bi].start, 1: b[bi].last). + bi, bii int = 0, 0 + + // Offset of a - next available index to set. + off int = 0 + // Value to set/append to a at off + val interval16 + + // Current state - state equals 0 means we are clear (out of intervals) + // When we start a new interval we add +1 when we get out of interval we add -1. + state int + + // subindex (ii) to state mapping + // .start: [0] -> 1 + // .last: [1] -> -1 + iiMap = [2]int{1, -1} + + // If fromB is equal 2 it means that both val.start and val.last come from b, + // so we need to extend a, first + fromB int8 + + // eval functions evaluates global state and value + eval = func(arr [2]uint16, ii int, onlyB bool) { + if state == 0 && ii == 0 { + // we are clear and start a new interval + val.start = arr[ii] + if onlyB { + fromB++ + } + } + + state += iiMap[ii] + + if state == 0 { + // we just got out of interval + // ii == 1 + val.last = arr[ii] + if onlyB { + fromB++ + } + } + } + // eval2 function is a special variant for eval function + // it's only used when two interval endings are equal, e.g.: + // a: ------------------| + // b: -----------| + // the most important part is to change the global for both endings + // before we check if we're getting out of interval and start the new one. + eval2 = func(arr [2]uint16, i1, i2 int) { + if state == 0 && (i1 == 0 || i2 == 0) { + // we are clear and start a new interval + val.start = arr[i1] + + } + + state += iiMap[i1] + state += iiMap[i2] + + if state == 0 { + // (i1 == 1 || i2 == 1) + // we just got out of interval + val.last = arr[i1] + } + } + ) + for { - var ( - v interval16 - local *interval16 - ) + // av, bv reflects a[ai] and b[bi] intervals as an array, + // so we can internally iterate over values (points). + var av, bv [2]uint16 - if i < an && j < bn { - if a[i].start <= b[j].start { - v = a[i] - i++ - } else { - v = b[j] - j++ + if ai < an && bi < bn { + av[0], av[1] = a[ai].start, a[ai].last + bv[0], bv[1] = b[bi].start, b[bi].last + + if av[aii] < bv[bii] { + // a: |------------------- + // b: |------------------- + + eval(av, aii, false) + aii++ + } else if av[aii] == bv[bii] { + // a: |------------------- + // b: |------------------- + // or + // a: ------------------| + // b: |------------------- + // or + // a: ------------------| + // b: |------------| + // ... + + eval2(av, aii, bii) + aii++ + bii++ + } else { // bv[bii] < av[aii] + // a: |------------------- + // b: |------------------- + + eval(bv, bii, true) + bii++ } + } else if ai < an { // only a left + av[0], av[1] = a[ai].start, a[ai].last + eval(av, aii, false) + aii++ + } else if bi < bn { // only b left + bv[0], bv[1] = b[bi].start, b[bi].last + eval(bv, bii, false) + bii++ } else { - if i < an { - v = a[i] - i++ - } else if j < bn { - v = b[j] - j++ - } else { - break + break + } + + if state == 0 { + if fromB == 2 { + // val.start and val.last come from b, so we need to extend a, first + a = append(a, interval16{}) + copy(a[off+1:], a[off:]) + ai++ + an++ } + fromB = 0 + a, off = appendInterval16At(a, val, off) + n += int32(val.last-val.start) + 1 } - reeval: - if k == an { - // append what's left and try to optimize it later - a = append(a, v) - an++ - continue + if aii == 2 { + // move to the next a's interval + aii = 0 + ai++ } - // locally unioned interval - if local == nil { - local = &a[k] - } - - if v.last < local.start { - // [---- v ----][---- local ----] - // prepend v into a - a = append(a, interval16{}) - copy(a[i+1:], a[i:]) - a[i] = v - an++ - i++ - continue - } - - if v.start > local.last { - // [---- local ----][---- v ----] - - // check if we can optimize run and combine neighbours - if int32(v.start-local.last) <= 1 { - local.last = v.last - continue - } - - // set already unioned interval and reevaluate v - a[k] = *local - k++ - n += int32(local.last-local.start) + 1 - local = &v - goto reeval - } - - if v.start < local.start { - if v.last >= local.start && v.last <= local.last { - // [---- local ----] - // [---- v ----] - local.start = v.start - } else if v.last > local.last { - // [- local -] - // [---- v ----] - *local = v - } - } else if v.start <= local.last { - if v.last <= local.last { - // [---- local ----] - // [-- v --] - - // this assignment overwrites the first available position to set - a[k] - // it's done to "swallow" many nested intervals, e.g.: - // [a1] [a2] [a3] [a4] - // [------ b ------] - // will give us: - // [ ----- a1 -----] [a4] - a[k] = *local - } else if v.last > local.last { - // [---- local ----] - // [---- v ----] - local.last = v.last - } + if bii == 2 { + // move to the next b's interval + bii = 0 + bi++ } } if len(a) > 0 { - // count the latest interval - n += int32(a[k].last-a[k].start) + 1 - a = a[:k+1] + a = a[:off] } return a, n } +// appendInterval16At appends or sets val in a at off position +// The function returns modified a ([]interval16) and new offset (off) +func appendInterval16At(a []interval16, val interval16, off int) ([]interval16, int) { + + if off > 0 && int32(val.start)-int32(a[off-1].last) <= 1 { + a[off-1].last = val.last + return a, off + } + + if off == len(a) { + a = append(a, val) + off++ + return a, off + } + + a[off] = val + off++ + + return a, off +} + func difference(a, b *Container) *Container { if a.N() == 0 || b.N() == maxContainerVal+1 { return nil diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 1f5fe572b..4e0375f76 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -722,467 +722,185 @@ func TestUnionMixed(t *testing.T) { } func TestUnionInterval16InPlace(t *testing.T) { - t.Run("single overlap", func(t *testing.T) { - a := []interval16{interval16{1, 10}, interval16{21, 28}} - b := []interval16{interval16{8, 12}} - bb := make([]interval16, len(b)) - copy(bb, b) + tests := []struct { + name string + a []interval16 + b []interval16 + expected []interval16 + expectedN int32 + }{ + { + name: "firstBitUnset lastBitSet", + a: []interval16{interval16{1, 10}}, + b: []interval16{interval16{10, 10}}, + expected: []interval16{interval16{1, 10}}, + expectedN: 10, + }, + { + name: "single overlap", + a: []interval16{interval16{1, 10}, interval16{21, 28}}, + b: []interval16{interval16{8, 12}}, + expected: []interval16{interval16{1, 12}, interval16{21, 28}}, + expectedN: 20, + }, + { + name: "nested intervals", + a: []interval16{interval16{3, 13}, interval16{17, 20}}, + b: []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}}, + expected: []interval16{interval16{1, 20}}, + expectedN: 20, + }, + { + name: "no overlap", + a: []interval16{interval16{3, 4}, interval16{7, 8}}, + b: []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}}, + expected: []interval16{interval16{1, 10}}, + expectedN: 10, + }, + { + name: "b in a", + a: []interval16{interval16{1, 10}}, + b: []interval16{interval16{5, 7}}, + expected: []interval16{interval16{1, 10}}, + expectedN: 10, + }, + { + name: "a eq b", + a: []interval16{interval16{1, 10}}, + b: []interval16{interval16{1, 10}}, + expected: []interval16{interval16{1, 10}}, + expectedN: 10, + }, + { + name: "a in b", + a: []interval16{interval16{5, 7}}, + b: []interval16{interval16{1, 10}}, + expected: []interval16{interval16{1, 10}}, + expectedN: 10, + }, + { + name: "a ahead b", + a: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, + b: []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}}, + expected: []interval16{interval16{1, 7}, interval16{10, 15}}, + expectedN: 13, + }, + { + name: "b ahead a", + a: []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}}, + b: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, + expected: []interval16{interval16{1, 7}, interval16{10, 15}}, + expectedN: 13, + }, + { + name: "empty a and b", + a: []interval16{}, + b: []interval16{}, + expected: []interval16{}, + expectedN: 0, + }, + { + name: "empty a", + a: []interval16{}, + b: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, + expected: []interval16{interval16{1, 7}}, + expectedN: 7, + }, + { + name: "empty b", + a: []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}}, + b: []interval16{}, + expected: []interval16{interval16{1, 7}}, + expectedN: 7, + }, + { + name: "single a", + a: []interval16{interval16{1, 2}}, + b: []interval16{}, + expected: []interval16{interval16{1, 2}}, + expectedN: 2, + }, + { + name: "single b", + a: []interval16{}, + b: []interval16{interval16{1, 2}}, + expected: []interval16{interval16{1, 2}}, + expectedN: 2, + }, + { + name: "single a single b", + a: []interval16{interval16{3, 4}}, + b: []interval16{interval16{1, 2}}, + expected: []interval16{interval16{1, 4}}, + expectedN: 4, + }, + { + name: "oddBitsSet lastBitUnset", + a: []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}}, + b: []interval16{interval16{0, 4}}, + expected: []interval16{interval16{0, 5}}, + expectedN: 6, + }, + { + name: "all bits", + a: []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}}, + b: []interval16{interval16{0, 0}, interval16{2, 2}, interval16{4, 4}}, + expected: []interval16{interval16{0, 5}}, + expectedN: 6, + }, + { + name: "short a long b", + a: []interval16{interval16{5, 5}, interval16{7, 7}, interval16{9, 10}, interval16{12, 12}, interval16{15, 17}, interval16{19, 20}}, + b: []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 18}}, + expected: []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 20}}, + expectedN: 18, + }, + { + name: "common endings", + a: []interval16{interval16{1, 5}, interval16{15, 20}, interval16{25, 35}}, + b: []interval16{interval16{1, 10}, interval16{15, 20}, interval16{30, 35}}, + expected: []interval16{interval16{1, 10}, interval16{15, 20}, interval16{25, 35}}, + expectedN: 27, + }, + { + name: "common endings and overlap", + a: []interval16{interval16{1, 5}, interval16{10, 15}}, + b: []interval16{interval16{5, 10}, interval16{12, 17}}, + expected: []interval16{interval16{1, 17}}, + expectedN: 17, + }, + { + name: "no common endings and overlap", + a: []interval16{interval16{5, 10}, interval16{12, 17}}, + b: []interval16{interval16{0, 11}, interval16{15, 20}}, + expected: []interval16{interval16{0, 20}}, + expectedN: 21, + }, + } - runs, n := unionInterval16InPlace(a, b) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + bb := make([]interval16, len(tc.b)) + copy(bb, tc.b) - expected := []interval16{interval16{1, 12}, interval16{21, 28}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + runs, n := unionInterval16InPlace(tc.a, tc.b) + + for i, v := range tc.expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", tc.expected, runs) + } } - } - - const expectedN = 20 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + if n != tc.expectedN { + t.Fatalf("N expected: %d, got: %d", tc.expectedN, n) } - } - }) - t.Run("nested intervals", func(t *testing.T) { - a := []interval16{interval16{3, 13}, interval16{17, 20}} - b := []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 20}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + for i, v := range bb { + if tc.b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, tc.b) + } } - } - - const expectedN = 20 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("no overlap", func(t *testing.T) { - a := []interval16{interval16{3, 4}, interval16{7, 8}} - b := []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 10 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("b in a", func(t *testing.T) { - a := []interval16{interval16{1, 10}} - b := []interval16{interval16{5, 7}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 10 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("a eq b", func(t *testing.T) { - a := []interval16{interval16{1, 10}} - b := []interval16{interval16{1, 10}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 10 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("a in b", func(t *testing.T) { - a := []interval16{interval16{5, 7}} - b := []interval16{interval16{1, 10}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 10 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("a ahead b", func(t *testing.T) { - a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - b := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{ - interval16{1, 7}, - interval16{10, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 13 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("b ahead a", func(t *testing.T) { - a := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} - b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{ - interval16{1, 7}, - interval16{10, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 13 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("empty a and b", func(t *testing.T) { - a := []interval16{} - b := []interval16{} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 0 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("empty a", func(t *testing.T) { - a := []interval16{} - b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 7 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("empty b", func(t *testing.T) { - a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - b := []interval16{} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 7 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("single a", func(t *testing.T) { - a := []interval16{interval16{1, 2}} - b := []interval16{} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 2}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 2 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("single b", func(t *testing.T) { - a := []interval16{} - b := []interval16{interval16{1, 2}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 2}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 2 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("single a single b", func(t *testing.T) { - a := []interval16{interval16{3, 4}} - b := []interval16{interval16{1, 2}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 4}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - - const expectedN = 4 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("oddBitsSet lastBitUnset", func(t *testing.T) { - a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} - b := []interval16{interval16{0, 4}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{0, 5}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - const expectedN = 6 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("all bits", func(t *testing.T) { - a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} - b := []interval16{interval16{0, 0}, interval16{2, 2}, interval16{4, 4}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{0, 5}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - const expectedN = 6 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) - - t.Run("short a long b", func(t *testing.T) { - a := []interval16{interval16{5, 5}, interval16{7, 7}, interval16{9, 10}, interval16{12, 12}, interval16{15, 17}, interval16{19, 20}} - b := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 18}} - bb := make([]interval16, len(b)) - copy(bb, b) - - runs, n := unionInterval16InPlace(a, b) - - expected := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 20}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) - } - } - const expectedN = 18 - if n != expectedN { - t.Fatalf("N expected: %d, got: %d", expectedN, n) - } - - for i, v := range bb { - if b[i] != v { - t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) - } - } - }) + }) + } } func TestIntersectMixed(t *testing.T) { From 905cda7f08601ca2f8f6de60ebeb91d71411bba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Mon, 22 Jun 2020 15:05:59 +0200 Subject: [PATCH 4/4] Add benchmarks --- roaring/roaring.go | 2 +- roaring/roaring_internal_test.go | 74 ++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index a8de21d6c..f407b8f69 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4863,7 +4863,7 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { } fromB = 0 a, off = appendInterval16At(a, val, off) - n += int32(val.last-val.start) + 1 + n += int32(val.last) - int32(val.start) + 1 } if aii == 2 { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 4e0375f76..4bb014b73 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -4329,3 +4329,77 @@ func TestDifferenceInPlace_N(t *testing.T) { t.Error("expected difference of containers to have n=0") } } + +func BenchmarkUnionRunRunInPlace(bm *testing.B) { + bm.Skip("Skipping long running BenchmarkUnionRunRunInPlace") + + runs := []struct { + name string + fn func() []interval16 + }{ + {"FirstBitSet", runFirstBitSet}, + {"LastBitSet", runLastBitSet}, + {"FirstBitUnset", runFirstBitUnset}, + {"LastBitUnset", runLastBitUnset}, + {"InnerBitsSet", runInnerBitsSet}, + {"OuterBitsSet", runOuterBitsSet}, + {"OddBitsSet", runOddBitsSet}, + {"EvenBitsSet", runEvenBitsSet}, + } + + for _, ar := range runs { + for _, br := range runs { + bm.Run("RunToBitmapRun-"+ar.name+"_"+br.name, func(bm *testing.B) { + for i := 0; i < bm.N; i++ { + arun := doContainer(containerRun, ar.fn()) + brun := doContainer(containerRun, br.fn()) + + abmp := arun.runToBitmap() + unionBitmapRunInPlace(abmp, brun) + } + }) + + bm.Run("RunRun-"+ar.name+"_"+br.name, func(bm *testing.B) { + for i := 0; i < bm.N; i++ { + arun := doContainer(containerRun, ar.fn()) + brun := doContainer(containerRun, br.fn()) + + unionRunRunInPlace(arun, brun) + } + }) + } + } +} + +func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) { + runs := []struct { + name string + run []interval16 + }{ + {name: "FirstBitSet", run: runFirstBitSet()}, + {name: "LastBitSet", run: runLastBitSet()}, + {name: "FirstBitUnset", run: runFirstBitUnset()}, + {name: "LastBitUnset", run: runLastBitUnset()}, + {name: "InnerBitsSet", run: runInnerBitsSet()}, + {name: "OuterBitsSet", run: runOuterBitsSet()}, + {name: "OddBitsSet", run: runOddBitsSet()}, + {name: "EvenBitsSet", run: runEvenBitsSet()}, + } + + for _, a := range runs { + for _, b := range runs { + t.Run(a.name+"-"+b.name, func(t *testing.T) { + arun := doContainer(containerRun, a.run) + brun := doContainer(containerRun, b.run) + + out1 := unionBitmapRunInPlace(arun.runToBitmap(), brun) + out2 := unionRunRunInPlace(arun, brun) + + err := out1.BitwiseCompare(out2.runToBitmap()) + if err != nil { + t.Fatal(err) + } + }) + } + } +}