From eddbb7d0b00f0eb2a48f2171a98fa53789ca0230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Wed, 10 Jun 2020 17:36:30 +0200 Subject: [PATCH] Optimize run intervals by combnining neighbours --- roaring/roaring.go | 81 +++----- roaring/roaring_internal_test.go | 330 +++++++++++++++++++++++++------ 2 files changed, 298 insertions(+), 113 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index dd13a99de..4cc1f9751 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4721,8 +4721,8 @@ func unionRunRunInPlace(a, b *Container) *Container { a = a.Thaw() runs, n := unionInterval16InPlace(a.runs(), b.runs()) - a.setN(n) a.setRuns(runs) + a.setN(n) return a } @@ -4732,57 +4732,50 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { an, bn := len(a), len(b) i, j, k := 0, 0, 0 for { - // select the next interval - v var ( - v *interval16 + v interval16 local *interval16 - visa bool // v is from a ) + if i < an && j < bn { if a[i].start <= b[j].start { - v, visa = &a[i], true + v = a[i] i++ - } else if j < bn { - v = &b[j] + } else { + v = b[j] j++ } } else { if i < an { - v, visa = &a[i], true + v = a[i] i++ } else if j < bn { - v = &b[j] + v = b[j] j++ + } else { + break } } - // no more intervals - break - if v == nil { - if len(a) > 0 { - // count the last interval - n += int32(a[k].last-a[k].start) + 1 - } - break - } - next: + reeval: if k == an { - // append what's left - a = append(a, *v) + // append what's left and try to optimize it later + a = append(a, v) an++ continue } - // current, locally unioned interval + // locally unioned interval if local == nil { local = &a[k] } if v.last < local.start { // [---- v ----][---- local ----] - // insert v into a + // prepend v into a a = append(a, interval16{}) copy(a[i+1:], a[i:]) - a[i] = *v + a[i] = v an++ i++ continue @@ -4790,28 +4783,19 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { if v.start > local.last { // [---- local ----][---- v ----] - // set already unioned interval and go next - // if int32(v.start-local.last) <= 1 { - // local.last = v.last - // continue - // } - a[k] = *local - n += int32(local.last-local.start) + 1 - k++ - - // if we set unioned interval we can go to the next interval - if visa { - // v is from a, so we can skip "digested intervals" - // and jump already to v - local = v - } else { - // ...otherwise, move to the next one, - // so let assign local in next iteration - local = nil + // check if we can optimize run and combine neighbours + if int32(v.start-local.last) <= 1 { + local.last = v.last + continue } - goto next + // set already unioned interval and reevaluate v + a[k] = *local + k++ + n += int32(local.last-local.start) + 1 + local = &v + goto reeval } if v.start < local.start { @@ -4822,18 +4806,15 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { } else if v.last > local.last { // [- local -] // [---- v ----] - *local = *v + *local = v } - continue - } - - if v.start >= local.start && v.start <= local.last { + } else if v.start <= local.last { if v.last <= local.last { // [---- local ----] // [-- v --] - // this assignment looks silly, but if we extended local - // and "digested" many nested intervals, we can squash a, e.g.: + // this assignment overwrites the first available position to set - a[k] + // it's done to "swallow" many nested intervals, e.g.: // [a1] [a2] [a3] [a4] // [------ b ------] // will give us: @@ -4848,6 +4829,8 @@ func unionInterval16InPlace(a, b []interval16) ([]interval16, int32) { } if len(a) > 0 { + // count the latest interval + n += int32(a[k].last-a[k].start) + 1 a = a[:k+1] } return a, n diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 665ac9857..1f5fe572b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -725,16 +725,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single overlap", func(t *testing.T) { a := []interval16{interval16{1, 10}, interval16{21, 28}} b := []interval16{interval16{8, 12}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 12}, interval16{21, 28}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 20 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 12}, interval16{21, 28}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -742,16 +752,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("nested intervals", func(t *testing.T) { a := []interval16{interval16{3, 13}, interval16{17, 20}} b := []interval16{interval16{1, 4}, interval16{6, 7}, interval16{8, 9}, interval16{10, 11}, interval16{14, 17}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 20}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 20 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 13}, interval16{14, 20}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -759,16 +779,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("no overlap", func(t *testing.T) { a := []interval16{interval16{3, 4}, interval16{7, 8}} b := []interval16{interval16{1, 2}, interval16{5, 6}, interval16{9, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 6}, interval16{7, 8}, interval16{9, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -776,16 +806,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("b in a", func(t *testing.T) { a := []interval16{interval16{1, 10}} b := []interval16{interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -793,16 +833,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a eq b", func(t *testing.T) { a := []interval16{interval16{1, 10}} b := []interval16{interval16{1, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -810,16 +860,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a in b", func(t *testing.T) { a := []interval16{interval16{5, 7}} b := []interval16{interval16{1, 10}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 10}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 10 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 10}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -827,23 +887,29 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("a ahead b", func(t *testing.T) { a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} b := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} + bb := make([]interval16, len(b)) + copy(bb, b) + runs, n := unionInterval16InPlace(a, b) + expected := []interval16{ + interval16{1, 7}, + interval16{10, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 13 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{ - interval16{1, 2}, - interval16{3, 4}, - interval16{5, 7}, - interval16{10, 11}, - interval16{12, 13}, - interval16{14, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -851,24 +917,29 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("b ahead a", func(t *testing.T) { a := []interval16{interval16{10, 11}, interval16{12, 13}, interval16{14, 15}} b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{ + interval16{1, 7}, + interval16{10, 15}, + } + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 13 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{ - interval16{1, 2}, - interval16{3, 4}, - interval16{5, 7}, - interval16{10, 11}, - interval16{12, 13}, - interval16{14, 15}, - } - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -876,17 +947,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty a and b", func(t *testing.T) { a := []interval16{} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 0 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -894,17 +974,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty a", func(t *testing.T) { a := []interval16{} b := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 7 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -912,17 +1001,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("empty b", func(t *testing.T) { a := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 7}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 7 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}, interval16{3, 4}, interval16{5, 7}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -930,17 +1028,26 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single a", func(t *testing.T) { a := []interval16{interval16{1, 2}} b := []interval16{} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 2 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}} - for i, v := range expected { - if runs[i] != v { - t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) } } }) @@ -948,28 +1055,66 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Run("single b", func(t *testing.T) { a := []interval16{} b := []interval16{interval16{1, 2}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) + expected := []interval16{interval16{1, 2}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 2 if n != expectedN { t.Fatalf("N expected: %d, got: %d", expectedN, n) } - expected := []interval16{interval16{1, 2}} + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("single a single b", func(t *testing.T) { + a := []interval16{interval16{3, 4}} + b := []interval16{interval16{1, 2}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{1, 4}} for i, v := range expected { if runs[i] != v { t.Fatalf("runs expected: %+v, got: %+v", expected, runs) } } + + const expectedN = 4 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } }) t.Run("oddBitsSet lastBitUnset", func(t *testing.T) { a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} b := []interval16{interval16{0, 4}} + bb := make([]interval16, len(b)) + copy(bb, b) runs, n := unionInterval16InPlace(a, b) - expected := []interval16{interval16{0, 4}, interval16{5, 5}} + expected := []interval16{interval16{0, 5}} for i, v := range expected { if runs[i] != v { t.Fatalf("runs expected: %+v, got: %+v", expected, runs) @@ -980,6 +1125,63 @@ func TestUnionInterval16InPlace(t *testing.T) { t.Fatalf("N expected: %d, got: %d", expectedN, n) } + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("all bits", func(t *testing.T) { + a := []interval16{interval16{1, 1}, interval16{3, 3}, interval16{5, 5}} + b := []interval16{interval16{0, 0}, interval16{2, 2}, interval16{4, 4}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{0, 5}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 6 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } + }) + + t.Run("short a long b", func(t *testing.T) { + a := []interval16{interval16{5, 5}, interval16{7, 7}, interval16{9, 10}, interval16{12, 12}, interval16{15, 17}, interval16{19, 20}} + b := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 18}} + bb := make([]interval16, len(b)) + copy(bb, b) + + runs, n := unionInterval16InPlace(a, b) + + expected := []interval16{interval16{1, 10}, interval16{12, 12}, interval16{14, 20}} + for i, v := range expected { + if runs[i] != v { + t.Fatalf("runs expected: %+v, got: %+v", expected, runs) + } + } + const expectedN = 18 + if n != expectedN { + t.Fatalf("N expected: %d, got: %d", expectedN, n) + } + + for i, v := range bb { + if b[i] != v { + t.Fatalf("b changed - runs expected: %+v, got: %+v", bb, b) + } + } }) }