From da5f8f11bb96a8d80f109dca34a15ba26f5c9a0b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 11:58:20 -0600 Subject: [PATCH 01/19] benchmark for snapshotting --- fragment_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fragment_test.go b/fragment_test.go index 2ccf538f7..14d36a56c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1075,3 +1075,25 @@ func TestFragment_Snapshot_Run(t *testing.T) { t.Fatalf("unexpected count (reopen): %d", n) } } + +func BenchmarkFragment_Snapshot(b *testing.B) { + if *FragmentPath == "" { + b.Skip("no fragment specified") + } + + // Open the fragment specified by the path. + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + if err := f.Open(); err != nil { + b.Fatal(err) + } + defer f.Close() + + // Reset timer and execute benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := f.Snapshot() + if err != nil { + b.Fatalf("unexpected count (reopen): %s", err) + } + } +} From e1735a60ec31486a3f43ab8aeb14eb539bfb522a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Jan 2018 12:15:15 -0600 Subject: [PATCH 02/19] added alloc logging --- fragment_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/fragment_test.go b/fragment_test.go index 14d36a56c..8a54d3b07 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -1090,6 +1090,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { // Reset timer and execute benchmark. b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { err := f.Snapshot() if err != nil { From 668689e11909f598a1165bd159bdfdcc39ad650b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 5 Feb 2018 16:35:28 -0600 Subject: [PATCH 03/19] ensure that the license is prepended to all source files --- config_test.go | 14 ++++++++++++++ ctl/common.go | 14 ++++++++++++++ diagnostics/diagnostics.go | 14 ++++++++++++++ diagnostics/diagnostics_test.go | 14 ++++++++++++++ pilosa_test.go | 14 ++++++++++++++ server/server.go | 1 + stats_test.go | 14 ++++++++++++++ test/attr.go | 14 ++++++++++++++ test/client.go | 14 ++++++++++++++ test/cluster.go | 16 +++++++++++++++- test/executor.go | 14 ++++++++++++++ test/fragment.go | 14 ++++++++++++++ test/frame.go | 14 ++++++++++++++ test/handler.go | 14 ++++++++++++++ test/holder.go | 14 ++++++++++++++ test/index.go | 14 ++++++++++++++ test/pilosa.go | 14 ++++++++++++++ test/pilosa_test.go | 14 ++++++++++++++ test/test.go | 14 ++++++++++++++ version.go | 14 ++++++++++++++ 20 files changed, 268 insertions(+), 1 deletion(-) diff --git a/config_test.go b/config_test.go index c409fd80f..dbeba89a0 100644 --- a/config_test.go +++ b/config_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/ctl/common.go b/ctl/common.go index 826462b8c..9bba68657 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ctl import ( diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index eb761e6c9..60248d6bb 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package diagnostics import ( diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 7ac4906d4..8f85a57db 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package diagnostics_test import ( diff --git a/pilosa_test.go b/pilosa_test.go index b4d9cc2e2..d82e27fa3 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/server/server.go b/server/server.go index 115f77263..d706d83f1 100644 --- a/server/server.go +++ b/server/server.go @@ -16,6 +16,7 @@ // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the // objects that Pilosa needs. + package server import ( diff --git a/stats_test.go b/stats_test.go index 07254a702..93d7d9bb1 100644 --- a/stats_test.go +++ b/stats_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( diff --git a/test/attr.go b/test/attr.go index 620848084..9363011b1 100644 --- a/test/attr.go +++ b/test/attr.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/client.go b/test/client.go index 5eac8cb79..9e391e88b 100644 --- a/test/client.go +++ b/test/client.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/cluster.go b/test/cluster.go index c45b1c989..adf82fc0a 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( @@ -15,7 +29,7 @@ func NewCluster(n int) *pilosa.Cluster { for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ Scheme: "http", - Host: fmt.Sprintf("host%d", i), + Host: fmt.Sprintf("host%d", i), }) } diff --git a/test/executor.go b/test/executor.go index f00692361..9b4233035 100644 --- a/test/executor.go +++ b/test/executor.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/fragment.go b/test/fragment.go index 36ebba5bf..b208e7a3a 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/frame.go b/test/frame.go index 6234c802e..e107b7d85 100644 --- a/test/frame.go +++ b/test/frame.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/handler.go b/test/handler.go index aa5a56acb..4743ea30c 100644 --- a/test/handler.go +++ b/test/handler.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/holder.go b/test/holder.go index 4cd0642e4..c9d21d2f2 100644 --- a/test/holder.go +++ b/test/holder.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/index.go b/test/index.go index a751b3af6..5354c85f9 100644 --- a/test/index.go +++ b/test/index.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/pilosa.go b/test/pilosa.go index 267e2fb16..e14b2d032 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 20ae1df0d..ca2104866 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test_test import ( diff --git a/test/test.go b/test/test.go index cf98c76b8..c83b657c0 100644 --- a/test/test.go +++ b/test/test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import "flag" diff --git a/version.go b/version.go index 3c7d19ca3..3e148ea7b 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa var Version = "v0.0.0" From eca01e4b4fa8e8418dc58a75a175e5a4cb92f91d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 6 Feb 2018 08:51:38 -0600 Subject: [PATCH 04/19] replace sha1 hash with faster xxhash --- Gopkg.lock | 16 ++++++++++++++-- attr.go | 6 ++++-- fragment.go | 7 ++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 7837cadff..76fcccbd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -37,6 +37,12 @@ revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" version = "v1.3.1" +[[projects]] + name = "github.com/cespare/xxhash" + packages = ["."] + revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" + version = "v1.0.0" + [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] @@ -189,10 +195,16 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["host","internal/common","mem","process"] + packages = ["cpu","host","internal/common","mem","net","process"] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -268,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" + inputs-digest = "53ae0cdcbfa8419b233a0544d7ebb80614af3dd04bebdc0c8f2d3620a87efa91" solver-name = "gps-cdcl" solver-version = 1 diff --git a/attr.go b/attr.go index ab29fc9c0..437cab433 100644 --- a/attr.go +++ b/attr.go @@ -16,13 +16,15 @@ package pilosa import ( "bytes" - "crypto/sha1" + "encoding/binary" "fmt" "sort" "sync" "time" + "github.com/cespare/xxhash" + "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -242,7 +244,7 @@ func (s *AttrStore) Blocks() ([]AttrBlock, error) { block := AttrBlock{ID: cur.blockID()} // Compute checksum of every key/value in block. - h := sha1.New() + h := xxhash.New() for k, v := cur.next(); k != nil; k, v = cur.next() { h.Write(k) h.Write(v) diff --git a/fragment.go b/fragment.go index 930c8e02b..95bcb7f2d 100644 --- a/fragment.go +++ b/fragment.go @@ -20,7 +20,6 @@ import ( "bytes" "container/heap" "context" - "crypto/sha1" "encoding/binary" "errors" "fmt" @@ -36,6 +35,8 @@ import ( "time" "unsafe" + "github.com/cespare/xxhash" + "math" "github.com/gogo/protobuf/proto" @@ -1020,7 +1021,7 @@ type TopOptions struct { // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { - h := sha1.New() + h := xxhash.New() for _, block := range f.Blocks() { h.Write(block.Checksum) } @@ -1660,7 +1661,7 @@ type blockHasher struct { func newBlockHasher() blockHasher { return blockHasher{ blockID: -1, - hash: sha1.New(), + hash: xxhash.New(), } } func (h *blockHasher) Reset() { From cc8733eedb6f13cfcc8497b757091266d0c2ef92 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 6 Feb 2018 10:02:53 -0600 Subject: [PATCH 05/19] fix bug where a count query and bitmap query could return different numbers There was a case where the Bitmap iterator logic could skip over a bit in a run container if 1. the run container was not the first container in the bitmap, and 2. The first run in the run container had only one bit. The bug was due to how the iterator was initialized with iterator.Seek(0) which sets up the initial values of itr.i,j,k based on the type of the first container. It was failing to set itr.k to -1 unless the first container was an RLE container. itr.k is only used by RLE containers in the iterator, and must be set to -1 when an RLE container is encountered. When Iterator.Next() encountered the run container and itr.k was set to 0, it checked to see if itr.k <= run.last - run.first, and if so it assumes that it was finished with the run and moved to the next one. run.last - run.first is 0 in the case of a single bit run, so that bit was skipped. After this, itr.k is set to -1 and all further iteration proceeds as expected. --- bitmap_test.go | 33 +++++++-- roaring/roaring.go | 4 + roaring/roaring_internal_test.go | 122 +++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 6 deletions(-) diff --git a/bitmap_test.go b/bitmap_test.go index 4fcd1b115..7eed299fc 100644 --- a/bitmap_test.go +++ b/bitmap_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "fmt" "reflect" "testing" @@ -23,14 +24,34 @@ import ( // Ensure a bitmap can be merged func TestBitmap_Merge(t *testing.T) { - bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth) - bm2 := pilosa.NewBitmap(3, 4, 5) - bm1.Merge(bm2) - - if bm1.Count() != 7 { - t.Fatalf("Count after merge %d != 7\n", bm1.Count()) + tests := []struct { + bm1 *pilosa.Bitmap + bm2 *pilosa.Bitmap + exp uint64 + }{ + { + bm1: pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth), + bm2: pilosa.NewBitmap(3, 4, 5), + exp: 7, + }, + { + bm1: pilosa.NewBitmap(), + bm2: pilosa.NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + exp: 7, + }, } + for i, test := range tests { + t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) { + test.bm1.Merge(test.bm2) + if cnt := test.bm1.Count(); cnt != test.exp { + t.Fatalf("merged count %d is not %d", cnt, test.exp) + } + if length := len(test.bm1.Bits()); uint64(length) != test.exp { + t.Fatalf("merged length %d is not %d", length, test.exp) + } + }) + } } // Ensure a bitmap can Xor'ed diff --git a/roaring/roaring.go b/roaring/roaring.go index 055c81e36..a685e479c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -840,6 +840,10 @@ func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) } // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { + // k should always be -1 unless we're seeking into a run container. Then the + // "if c.isRun" section will take care of it. + itr.k = -1 + // Move to the correct container. itr.i = search64(itr.bitmap.keys, highbits(seek)) if itr.i < 0 { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 4b2813908..a2fbb7d64 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2244,6 +2244,44 @@ func TestIteratorRuns(t *testing.T) { } } +func TestIteratorVarious(t *testing.T) { + tests := []struct { + bm *Bitmap + exp uint64 + }{ + { + bm: NewBitmap(3, 4, 5), + exp: 3, + }, + { + bm: bitmapVariousContainers(), + exp: 61221, + }, + { + bm: NewBitmap(2, 66000, 70000, 70001, 70002, 70003, 70004), + exp: 7, + }, + } + + for i, test := range tests { + test.bm.Optimize() + t.Run(fmt.Sprintf("#%d:", i), func(t *testing.T) { + if cnt := test.bm.Count(); cnt != test.exp { + t.Fatalf("merged count %d is not %d", cnt, test.exp) + } + iter := test.bm.Iterator() + bits := make([]uint64, 0, test.bm.Count()) + for v, eof := iter.Next(); !eof; v, eof = iter.Next() { + bits = append(bits, v) + } + if length := len(bits); uint64(length) != test.exp { + t.Fatalf("length %d is not %d", length, test.exp) + } + }) + } + +} + func TestRunBinSearchContains(t *testing.T) { tests := []struct { runs []interval16 @@ -2547,3 +2585,87 @@ func bitmapEvens() []uint64 { } return bitmap } + +var containerWidth uint64 = 65536 + +// rleCont returns a slice of numbers all in the range starting from +// container_width*num, and ending at container_width*(num+1)-1. If left is +// true, then the first 100 bits will be set, if mid is true, 100 bits in the +// middle will be set, if right is true, the last 100 bits will be set. +// sets 100 bits per true +func rleCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + for i := uint64(0); i < 100; i++ { + ret = append(ret, base+i) + } + } + if mid { + for i := containerWidth / 2; i < containerWidth/2+100; i++ { + ret = append(ret, base+i) + } + } + if right { + for i := containerWidth - 100; i < containerWidth; i++ { + ret = append(ret, base+i) + } + } + return ret +} + +// sets 2 bits per true. +func arrCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + ret = append(ret, base+0, base+2) + } + if mid { + half := containerWidth / 2 + ret = append(ret, base+half, base+half+2) + } + if right { + ret = append(ret, base+containerWidth-3, base+containerWidth-1) + } + return ret +} + +// sets 6667 bits per true. +func bitCont(num int, left, mid, right bool) []uint64 { + ret := make([]uint64, 0) + base := containerWidth * uint64(num) + if left { + for i := uint64(0); i < 20001; i += 3 { + ret = append(ret, base+i) + } + } + if mid { + for i := uint64(21000); i < 41001; i += 3 { + ret = append(ret, base+i) + } + } + if right { + for i := uint64(45537); i <= 65535; i += 3 { + ret = append(ret, base+i) + } + } + return ret +} + +func bitmapVariousContainers() *Bitmap { + bits := make([]uint64, 0) + bits = append(bits, rleCont(0, true, true, true)...) + bits = append(bits, rleCont(1, true, true, true)...) + bits = append(bits, arrCont(2, true, true, true)...) + bits = append(bits, arrCont(3, true, true, true)...) + bits = append(bits, bitCont(4, true, true, true)...) + bits = append(bits, bitCont(5, true, true, true)...) + bits = append(bits, rleCont(6, true, true, true)...) + bits = append(bits, bitCont(7, true, true, true)...) + bits = append(bits, arrCont(8, true, true, true)...) + bits = append(bits, rleCont(9, true, true, true)...) + bm := NewBitmap(bits...) + bm.Optimize() + return bm +} From a01a64aec5d28a71174537ec8b5c34a209388c82 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 6 Feb 2018 10:51:03 -0600 Subject: [PATCH 06/19] remove unneeded dep --- Gopkg.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 76fcccbd4..ed0469fc0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -199,12 +199,6 @@ revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" -[[projects]] - branch = "master" - name = "github.com/shirou/w32" - packages = ["."] - revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" - [[projects]] name = "github.com/sony/gobreaker" packages = ["."] From e9bd1c7f0dd4f65c06ac683dabac632c4f0a4eb2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 6 Feb 2018 14:15:37 -0600 Subject: [PATCH 07/19] Add license notice to two files --- server/default.go | 14 ++++++++++++++ server/release.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/server/default.go b/server/default.go index 6e9726afe..a7131c82a 100644 --- a/server/default.go +++ b/server/default.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // +build !release // // This file sets defaults to be overridden by release.go diff --git a/server/release.go b/server/release.go index de5b4a4d0..54d00b244 100644 --- a/server/release.go +++ b/server/release.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // +build release // // This file sets release-specific variables. From 5529040514fa0aa5499add048d69c774e58e33b6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 7 Feb 2018 11:29:58 -0600 Subject: [PATCH 08/19] clarify the options in configuration for metrics.service --- docs/administration.md | 6 +++--- docs/configuration.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 795d6f1c4..b77d05262 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -150,9 +150,9 @@ You can opt-out of the Pilosa diagnostics reporting by setting either the comman Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. The metrics configuration options are: - - Host to receive events - - Polling interval for runtime metrics - - Metric type (StatsD, Expvar). + - [Host](../configuration#metrics-host): specify host that receives metric events + - [Poll Interval](../configuration#metrics-poll-interval): specify polling interval for runtime metrics + - [Service](../configuration#metrics-service): declare type StatsD or Expvar ##### Tags StatsD Tags adhere to the DataDog format (key:value), and we tag the following: diff --git a/docs/configuration.md b/docs/configuration.md index 43b70c8f1..2385f2494 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -220,7 +220,7 @@ Any flag that has a value that is a comma separated list on the command line bec cpu-time = "30s" ``` ##### Metric Service -* Description: Which stats service to use (StatsD or ExpVar). +* Description: Which stats service to use. Choose from [statsd, expvar]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: From f799d98564198424a76db1ca221427583b9516c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 9 Feb 2018 17:33:07 +0300 Subject: [PATCH 09/19] Fixes #1092 --- Gopkg.lock | 8 +++++++- server.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index ed0469fc0..a8071c915 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -199,6 +199,12 @@ revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -274,6 +280,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "53ae0cdcbfa8419b233a0544d7ebb80614af3dd04bebdc0c8f2d3620a87efa91" + inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/server.go b/server.go index c8e536c50..83d417abd 100644 --- a/server.go +++ b/server.go @@ -15,6 +15,7 @@ package pilosa import ( + "context" "crypto/tls" "errors" "fmt" @@ -35,7 +36,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/diagnostics" "github.com/pilosa/pilosa/internal" - "golang.org/x/net/context" "golang.org/x/sync/errgroup" ) From a2580915ba110238f9dcfe310a870d0791e8e857 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 9 Feb 2018 17:34:18 +0300 Subject: [PATCH 10/19] remove golang/org context from the lock --- Gopkg.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a8071c915..f33a9ec4f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -247,12 +247,6 @@ revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" version = "v1.0.0" -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = ["context"] - revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" - [[projects]] branch = "master" name = "golang.org/x/sync" From ac4010bce0951c60ffe1f648d830a9dfa0d0a6b9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 9 Feb 2018 11:35:13 -0600 Subject: [PATCH 11/19] avoid overflow bug in differenceRunArray which was appending a full run to the container --- roaring/roaring.go | 8 ++++++++ roaring/roaring_internal_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index a685e479c..efdce69a0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2504,6 +2504,8 @@ func differenceRunArray(a, b *container) *container { bidx := 0 vb := b.array[bidx] + +RUNLOOP: for _, run := range a.runs { start := run.start for vb < run.start { @@ -2515,6 +2517,9 @@ func differenceRunArray(a, b *container) *container { } for vb >= run.start && vb <= run.last { if vb == start { + if vb == 65535 { // overflow + break RUNLOOP + } start++ bidx++ if bidx >= len(b.array) { @@ -2525,6 +2530,9 @@ func differenceRunArray(a, b *container) *container { } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) output.n += int(vb - start) + if vb == 65535 { // overflow + break RUNLOOP + } start = vb + 1 bidx++ if bidx >= len(b.array) { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a2fbb7d64..f79f53053 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1479,6 +1479,16 @@ func TestDifferenceRunArray(t *testing.T) { array: []uint16{0, 9, 10, 11, 12, 13, 14, 17, 19, 25, 27}, exp: []interval16{{start: 1, last: 8}, {start: 15, last: 16}, {start: 20, last: 24}, {start: 26, last: 26}, {start: 28, last: 28}}, }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65533, last: 65535}}, + array: []uint16{65533, 65534, 65535}, + exp: []interval16{{start: 0, last: 20}}, + }, + { + runs: []interval16{{start: 0, last: 20}, {start: 65530, last: 65535}}, + array: []uint16{37, 65535}, + exp: []interval16{{start: 0, last: 20}, {start: 65530, last: 65534}}, + }, } for i, test := range tests { a.runs = test.runs From db7e88f6d55af32374e5b03d9cab44b0d6edfa7f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 9 Feb 2018 15:14:26 -0600 Subject: [PATCH 12/19] add request/response tags to the docs --- docs/api-reference.md | 116 +++++++++++++--------------------------- docs/getting-started.md | 112 ++++++++++++++++++++++++++++++++------ docs/query-language.md | 5 +- docs/tutorials.md | 113 +++++++++++++++++++++++--------------- 4 files changed, 205 insertions(+), 141 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 206c30f55..52ea2c369 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -13,13 +13,10 @@ nav = [] Returns the schema of all indexes in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index ``` - -Response: -``` +``` response {"indexes":[{"name":"user","frames":[{"name":"collab"}]}]} ``` @@ -29,13 +26,10 @@ Response: Returns the schema of the specified index in JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user ``` - -Response: -``` +``` response {"index":{"name":"user"}, "frames":[{"name":"collab"}]}]} ``` @@ -45,13 +39,10 @@ Response: Creates an index with the given name. -Request: -``` +``` request curl -XPOST localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -61,13 +52,10 @@ Response: Removes the given index. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user ``` - -Response: -``` +``` response {} ``` @@ -77,15 +65,12 @@ Response: Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default. -Request: -``` +``` request curl localhost:10101/index/user/query \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` - -Response: -``` +``` response {"results":[{"attrs":{},"bits":[100]}]} ``` @@ -95,14 +80,12 @@ The response doesn't include column attributes by default. To return them, set t The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices. -Request: -``` +``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ -d 'Bitmap(frame="language", rowID=5)' ``` -Response: -``` +``` response { "results":[{"attrs":{},"bits":[100]}], "columnAttrs":[{"id":100,"attrs":{"name":"Klingon"}}] @@ -134,21 +117,21 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. -Request: -``` +``` request curl localhost:10101/index/user/frame/language \ -X POST \ -d '{"options": {"inverseEnabled": true}}' ``` - +``` response +{} ``` + +``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` - -Response: -``` +``` response {} ``` @@ -158,13 +141,10 @@ Response: Removes the given frame. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/frame/language ``` - -Response: -``` +``` response {} ``` @@ -188,15 +168,12 @@ The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Va * MDH: month, day and hour * YMDH: year, month, day and hour -Request: -``` +``` request curl localhost:10101/index/user/frame/language/time-quantum \ -X POST \ -d '{"timeQuantum": "YM"}' ``` - -Response: -``` +``` response {} ``` @@ -211,15 +188,12 @@ The request payload is JSON, and it must contain the fields `type`, `min`, `max` * `min` (int): Minimum value allowed for this field. * `max` (int): Maximum value allowed for this field. -Request: -``` +``` request curl localhost:10101/index/repository/frame/stats/field/pullrequests \ -X POST \ -d '{"type": "int", "min": 0, "max": 1000000}' ``` - -Response: -``` +``` response {} ``` @@ -251,8 +225,7 @@ The `action` describes how the field value will be processed. Each `action` may - `mapping`: Map the value to a RowID in the `valueMap`. * `valueMap` (object): string and integer pairs used to map field values to RowID's. -Request: -``` +``` request curl localhost:10101/index/user/input-definition/stargazer-input \ -X POST \ -d '{ @@ -284,9 +257,7 @@ curl localhost:10101/index/user/input-definition/stargazer-input \ ] }' ``` - -Response: -``` +``` response {} ``` @@ -296,13 +267,10 @@ Response: Returns the given input definition as JSON. -Request: -``` +``` request curl -XGET localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} ``` @@ -312,13 +280,10 @@ Response: Removes the given input definition. -Request: -``` +``` request curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input ``` - -Response: -``` +``` response {} ``` @@ -330,15 +295,12 @@ Processes the JSON payload using the given input definition. The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column label, and additional fields that will be handled by corresponding actions in the input definition. -Request: -``` +``` request curl localhost:10101/index/user/input/stargazer-input \ -X POST \ -d '[{"language_id": "Go", "repo_id": 92274475}]' ``` - -Response: -``` +``` response {} ``` @@ -348,13 +310,10 @@ Response: Returns the hosts in the cluster. -Request: -``` +``` request curl -XGET localhost:10101/hosts ``` - -Response: -``` +``` response [{"host":":10101"}] ``` @@ -364,13 +323,10 @@ Response: Returns the version of the Pilosa server. -Request: -``` +``` request curl -XGET localhost:10101/version ``` - -Response: -``` +``` response {"version":"v0.6.0"} ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 35042d59d..d4802e8ee 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -31,11 +31,12 @@ docker run -it --rm --name pilosa -p 10101:10101 pilosa/pilosa:latest ``` Let's make sure Pilosa is running: -``` +``` request curl localhost:10101/status ``` - -Which should output: `{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}}` +``` response +{"status":{"Nodes":[{"Host":":10101","State":"UP"}]}} +``` ### Sample Project @@ -45,30 +46,41 @@ Although Pilosa doesn't keep the data in a tabular format, we still use the term #### Create the Schema -Note: The queries in this section which are used to set up the indexes in Pilosa just the empty object on success: `{}` - if you would like to verify that a query worked as you expected, you can request the schema as follows: -``` +``` request curl localhost:10101/schema +``` +``` response {"indexes":null} ``` Before we can import data or run queries, we need to create our indexes and the frames within them. Let's create the repository index first: -``` +``` request curl localhost:10101/index/repository -X POST ``` +``` response +{} +``` Let's create the `stargazer` frame which has user IDs of stargazers as its rows: -``` +``` request curl localhost:10101/index/repository/frame/stargazer \ -X POST \ -d '{"options": {"timeQuantum": "YMD"}}' ``` +``` response +{} +``` Since our data contains time stamps for the time users starred repos, we set the *time quantum* for the `stargazer` frame in the options as well. Time quantum is the resolution of the time we want to use, and we set it to `YMD` (year, month, day) for `stargazer`. Next up is the `language` frame, which will contain IDs for programming languages: +``` request +curl localhost:10101/index/repository/frame/language \ + -X POST ``` -curl localhost:10101/index/repository/frame/language -X POST +``` response +{} ``` #### Import Data From CSV Files @@ -107,46 +119,112 @@ Alternatively Pilosa can import JSON data using an [Input Definition](../input-d Which repositories did user 14 star: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'Bitmap(frame="stargazer", rowID=14)' ``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,362,368,391,396,409,416,430,436,450,454,460,461,464,466,469,470,483,484,486,490,491,503,504,514] + } + ] +} +``` What are the top 5 languages in the sample data: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'TopN(frame="language", n=5)' ``` +``` response +{ + "results":[ + [ + {"id":5,"count":119}, + {"id":1,"count":50}, + {"id":4,"count":48}, + {"id":9,"count":31}, + {"id":13,"count":25} + ] + ] +} +``` Which repositories were starred by user 14 and 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,3,362,396,416,461,464,466,470,486] + } + ] +} ``` Which repositories were starred by user 14 or 19: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Union(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19))' + -d 'Union( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[1,2,3,361,362,368,376,377,378,382,386,388,391,396,398,400,409,411,412,416,426,428,430,435,436,450,452,453,454,456,460,461,464,465,466,469,470,483,484,486,487,489,490,491,500,503,504,505,512,514] + } + ] +} ``` Which repositories were starred by user 14 and 19 and also were written in language 1: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Intersect(Bitmap(frame="stargazer", rowID=14), Bitmap(frame="stargazer", rowID=19), Bitmap(frame="language", rowID=1))' + -d 'Intersect( + Bitmap(frame="stargazer", rowID=14), + Bitmap(frame="stargazer", rowID=19), + Bitmap(frame="language", rowID=1) + )' +``` +``` response +{ + "results":[ + { + "attrs":{}, + "bits":[2,362,416,461] + } + ] +} ``` Set user 99999 as a stargazer for repository 77777: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' ``` +``` response +{"results":[true]} +``` ### What's Next? diff --git a/docs/query-language.md b/docs/query-language.md index f28257fe6..0e02351ab 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -34,11 +34,14 @@ The default row label is `rowID`, and the default column label is `columnID`. Ch Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started) section to set up an index, frames, and populate them with some data. The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: -``` +``` request curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' ``` +``` response +{"results":[true]} +``` #### Arguments and Types diff --git a/docs/tutorials.md b/docs/tutorials.md index 9c19f3a67..4d787577c 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -178,38 +178,53 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. All nodes should be in the `UP` state: -``` +``` response {"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} ``` #### Running Queries Having confirmed that our cluster is running OK, let's run a few queries. But before that, we need to create an index and a frame: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index -d '' ``` +``` response +{} +``` This will create index `sample-index` with default options. Let's create the frame now: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/frame/sample-frame -d '' ``` +``` response +{} +``` We just created frame `sample-frame` with default options. Let's run a `SetBit` query: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' ``` +``` response +{"results":[true]} +``` Confirm that the bit was indeed set: -``` +``` request curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` The same response should be returned when querying other nodes in the cluster: -``` +``` request curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' ``` +``` response +{"results":[{"attrs":{},"bits":[100]}]} +``` #### What's Next? @@ -223,20 +238,16 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: -``` +``` request curl localhost:10101/index/patients \ -X POST ``` - -Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient. -``` -curl localhost:10101/index/patients/frame/measurements \ - -X POST \ - -d '{"options":{"rangeEnabled": true}}' +``` response +{} ``` In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. -``` +``` request curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ @@ -248,17 +259,23 @@ curl localhost:10101/index/patients/frame/measurements \ ] }}' ``` +``` response +{} +``` If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' ``` +``` response +{"results":[null]} +``` In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. @@ -282,39 +299,38 @@ pilosa import -i patients -f measurements --field age ages.csv Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Range(frame="measurements", age > 40)' ``` -You should get the following results: -``` +``` response {"results":[{"attrs":{},"bits":[2,6,9]}]} ``` You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(frame="measurements", field="age")' ``` -The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -``` +``` response {"results":[{"sum":377,"count":9}]} ``` +The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. -``` +``` request curl localhost:10101/index/patients/query \ -X POST \ -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' ``` -Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. -``` +``` response {"results":[{"sum":191,"count":3}]} ``` +Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. ### Storing Row and Column Attributes @@ -323,20 +339,26 @@ Notice in this case that the count is only `3` because of the `age > 40` filter Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data. First, create an index called `books` to use for this tutorial: -``` +``` request curl localhost:10101/index/books \ -X POST ``` +``` response +{} +``` Next, create a frame in the `books` index called `members` which will represent library members who have read books. -``` +``` request curl localhost:10101/index/books/frame/members \ -X POST \ -d '{}' ``` +``` response +{} +``` Now, let's add some books to our index. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) @@ -345,9 +367,12 @@ curl localhost:10101/index/books/query \ SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' ``` +``` response +{"results":[null,null,null,null,null]} +``` And add some members. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") @@ -356,20 +381,22 @@ curl localhost:10101/index/books/query \ SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' ``` +``` response +{"results":[null,null,null,null,null]} +``` At this point we can query one of the `member` records by querying that row. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -You should get the following result set: -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} ``` Now let's add some data to the matrix such that each pair represents a member who has read that book. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'SetBit(frame="members", rowID=10001, columnID=3) @@ -390,28 +417,28 @@ curl localhost:10101/index/books/query \ SetBit(frame="members", rowID=10005, columnID=4) SetBit(frame="members", rowID=10005, columnID=5)' ``` +``` response +{"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} +``` Now pull the record for `Sue Perkins` again. -``` +``` request curl localhost:10101/index/books/query \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` -Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. -``` +``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} ``` +Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Bitmap(frame="members", rowID=10002)' ``` - -Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute. - -``` +``` response { "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], "columnAttrs":[ @@ -421,15 +448,15 @@ Here, the `book` attributes will be included in the result set at the `columnAtt ] } ``` +The `book` attributes are included in the result set at the `columnAttrs` attribute. Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members: -``` +``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' ``` - -``` +``` response { "results":[{"attrs":{},"bits":[4]}], "columnAttrs":[ From e2c2bd11e45408cea366dface8d0489473b6a205 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 12 Feb 2018 17:16:38 +0300 Subject: [PATCH 13/19] put back golang.org/x/net dep, required by golang.org/x/sync/errgroup --- Gopkg.lock | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Gopkg.lock b/Gopkg.lock index f33a9ec4f..a8071c915 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -247,6 +247,12 @@ revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" version = "v1.0.0" +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2" + [[projects]] branch = "master" name = "golang.org/x/sync" From 657ff05e88167b4be628a93d91883e780f470cb4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 12 Feb 2018 17:28:49 +0300 Subject: [PATCH 14/19] fix format strings which fail on go/master --- handler_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/handler_test.go b/handler_test.go index 04903c979..585230b15 100644 --- a/handler_test.go +++ b/handler_test.go @@ -357,7 +357,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } @@ -445,7 +445,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { @@ -506,7 +506,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeBitmap { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -579,7 +579,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { - t.Fatalf("unexpected response type: %s", resp.Results[0].Type) + t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) } From 944a0b5b3d511923d16615453784e30a37aa7d47 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sun, 11 Feb 2018 21:21:48 -0600 Subject: [PATCH 15/19] fixes a shift logic bug in bitmapZeroRange that was causing an overflow-like condition --- roaring/roaring.go | 2 +- roaring/roaring_internal_test.go | 86 ++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index efdce69a0..b7cdec1c9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2337,7 +2337,7 @@ func (c *container) bitmapZeroRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { c.n -= int(popcnt(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index f79f53053..28b398c96 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1593,14 +1593,69 @@ func TestDifferenceBitmapRun(t *testing.T) { runs: []interval16{{start: 4, last: 7}, {start: 32, last: 47}}, exp: []uint64{0xFFFF0000FFFFFF0F}, }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}, {start: 7, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFBF}, + runs: []interval16{{start: 0, last: 5}}, + exp: []uint64{0xFFFFFFFFFFFFFF80}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 63}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF}, + }, + { + bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF}, + runs: []interval16{{start: 60, last: 65}, {start: 67, last: 72}, {start: 126, last: 130}}, + exp: []uint64{0x0FFFFFFFFFFFFFFF, 0x3FFFFFFFFFFFFE04, 0xFFFFFFFFFFFFFFF8}, + }, + { + bitmap: []uint64{0x0000000000000001}, + runs: []interval16{{start: 0, last: 0}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0x8000000000000000}, + runs: []interval16{{start: 63, last: 63}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: []uint64{0xC000000000000000, 0x0000000000000003}, + runs: []interval16{{start: 63, last: 64}}, + exp: []uint64{0x4000000000000000, 0x0000000000000002}, + }, + { + bitmap: []uint64{0x0000000000000000}, + runs: []interval16{{start: 5, last: 7}}, + exp: []uint64{0x0000000000000000}, + }, + { + bitmap: bitmapLast(), + runs: []interval16{{start: 65535, last: 65535}}, + exp: bitmapEmpty(), + }, + { + bitmap: bitmapFull(), + runs: []interval16{{start: 0, last: 65535}}, + exp: bitmapEmpty(), + }, } for i, test := range tests { for i, v := range test.bitmap { a.bitmap[i] = v } - a.n = a.bitmapCountRange(0, 100) + a.n = a.bitmapCountRange(0, 65536) b.runs = test.runs - b.n = b.runCountRange(0, 100) + b.n = b.runCountRange(0, 65536) ret := differenceBitmapRun(a, b) if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) { t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)]) @@ -2438,7 +2493,7 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { } } -func TestSearc64(t *testing.T) { +func TestSearch64(t *testing.T) { tests := []struct { a []uint64 value uint64 @@ -2596,6 +2651,31 @@ func bitmapEvens() []uint64 { return bitmap } +func bitmapLast() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN-1; i++ { + bitmap[i] = 0 + } + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapEmpty() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0 + } + return bitmap +} + var containerWidth uint64 = 65536 // rleCont returns a slice of numbers all in the range starting from From 6529866b5759e294de1634f8963eb1d252d7decc Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 13:30:07 -0600 Subject: [PATCH 16/19] templates for testing various container operations --- roaring/roaring_helpers_test.go | 256 ++++++++++++++++++++ roaring/roaring_internal_test.go | 390 +++++++++++++++++++++++++++++-- 2 files changed, 628 insertions(+), 18 deletions(-) create mode 100644 roaring/roaring_helpers_test.go diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go new file mode 100644 index 000000000..de3dd002c --- /dev/null +++ b/roaring/roaring_helpers_test.go @@ -0,0 +1,256 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +/////////////////////////////////////////////////////////////////////////// + +var containerWidth uint64 = 65536 + +////////////////// array +func arrayEmpty() []uint16 { + return make([]uint16, 0) +} + +func arrayFull() []uint16 { + array := make([]uint16, containerWidth) + for i := 0; i < int(containerWidth); i++ { + array[i] = uint16(i) + } + return array +} + +func arrayFirstBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + return array +} + +func arrayLastBitSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(65535)) + return array +} + +func arrayFirstBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 1; i < int(containerWidth); i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayLastBitUnset() []uint16 { + array := make([]uint16, containerWidth-1) + for i := 0; i < int(containerWidth)-1; i++ { + array[i] = uint16(i) + } + return array +} + +func arrayInnerBitsSet() []uint16 { + array := make([]uint16, containerWidth-2) + for i := 1; i < int(containerWidth)-1; i++ { + array[i-1] = uint16(i) + } + return array +} + +func arrayOuterBitsSet() []uint16 { + array := make([]uint16, 0) + array = append(array, uint16(0)) + array = append(array, uint16(65535)) + return array +} + +////////////////// bitmap +func bitmapEmpty() []uint64 { + return make([]uint64, bitmapN) +} + +func bitmapFull() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapFirstBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + return bitmap +} + +func bitmapLastBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitUnset() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0xFFFFFFFFFFFFFFFE + for i := 1; i < bitmapN; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + return bitmap +} + +func bitmapLastBitUnset() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN-1; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapInnerBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0xFFFFFFFFFFFFFFFE + for i := 1; i < bitmapN-1; i++ { + bitmap[i] = 0xFFFFFFFFFFFFFFFF + } + bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF + return bitmap +} + +func bitmapOuterBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000001 + for i := 1; i < bitmapN-1; i++ { + bitmap[i] = 0x0000000000000000 + } + bitmap[bitmapN-1] = 0x8000000000000000 + return bitmap +} + +////////////////// run +func runEmpty() []interval16 { + return make([]interval16, 0) +} + +func runFull() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65535}) + return run +} + +func runFirstBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + return run +} + +func runLastBitSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +func runFirstBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65535}) + return run +} + +func runLastBitUnset() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 65534}) + return run +} + +func runInnerBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 1, last: 65534}) + return run +} + +func runOuterBitsSet() []interval16 { + run := make([]interval16, 0) + run = append(run, interval16{start: 0, last: 0}) + run = append(run, interval16{start: 65535, last: 65535}) + return run +} + +/////////////////////////////////////////////////////////////////////////// + +type testOp struct { + f func(a, b *container) *container + x string + y string + exp string +} + +func doContainer(containerType byte, data interface{}) *container { + c := &container{ + containerType: containerType, + } + + switch containerType { + case ContainerArray: + c.array = data.([]uint16) + case ContainerBitmap: + c.bitmap = data.([]uint64) + case ContainerRun: + c.runs = data.([]interval16) + } + c.n = c.count() + + return c +} + +func setupContainerTests() map[byte]map[string]*container { + + cts := make(map[byte]map[string]*container) + + // array containers + cts[ContainerArray] = map[string]*container{ + "empty": doContainer(ContainerArray, arrayEmpty()), + "full": doContainer(ContainerArray, arrayFull()), + "firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()), + "lastBitSet": doContainer(ContainerArray, arrayLastBitSet()), + "firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()), + "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), + "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), + "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + } + + // bitmap containers + cts[ContainerBitmap] = map[string]*container{ + "empty": doContainer(ContainerBitmap, bitmapEmpty()), + "full": doContainer(ContainerBitmap, bitmapFull()), + "firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()), + "lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()), + "firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()), + "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), + "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), + "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + } + + // run containers + cts[ContainerRun] = map[string]*container{ + "empty": doContainer(ContainerRun, runEmpty()), + "full": doContainer(ContainerRun, runFull()), + "firstBitSet": doContainer(ContainerRun, runFirstBitSet()), + "lastBitSet": doContainer(ContainerRun, runLastBitSet()), + "firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()), + "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), + "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), + "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + } + + return cts +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 28b398c96..ce9ab217d 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -18,6 +18,8 @@ import ( "bytes" "fmt" "reflect" + "runtime" + "strings" "testing" ) @@ -2660,24 +2662,6 @@ func bitmapLast() []uint64 { return bitmap } -func bitmapFull() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } - return bitmap -} - -func bitmapEmpty() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0 - } - return bitmap -} - -var containerWidth uint64 = 65536 - // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the @@ -2759,3 +2743,373 @@ func bitmapVariousContainers() *Bitmap { bm.Optimize() return bm } + +/////////////////////////////////////////////////////////////////////////// + +func getFunctionName(i interface{}) string { + x := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() + y := strings.Split(x, ".") + y = y[len(y)-1:] + return y[0] +} + +func TestContainerCombinations(t *testing.T) { + + cts := setupContainerTests() + + containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun} + + // map used for a more descriptive print + cm := map[byte]string{ + ContainerArray: "array", + ContainerBitmap: "bitmap", + ContainerRun: "run", + } + + testOps := []testOp{ + // intersect + {intersect, "empty", "empty", "empty"}, + {intersect, "empty", "full", "empty"}, + {intersect, "empty", "firstBitSet", "empty"}, + {intersect, "empty", "lastBitSet", "empty"}, + {intersect, "empty", "firstBitUnset", "empty"}, + {intersect, "empty", "lastBitUnset", "empty"}, + {intersect, "empty", "innerBitsSet", "empty"}, + {intersect, "empty", "outerBitsSet", "empty"}, + // + {intersect, "full", "empty", "empty"}, + {intersect, "full", "full", "full"}, + {intersect, "full", "firstBitSet", "firstBitSet"}, + {intersect, "full", "lastBitSet", "lastBitSet"}, + {intersect, "full", "firstBitUnset", "firstBitUnset"}, + {intersect, "full", "lastBitUnset", "lastBitUnset"}, + {intersect, "full", "innerBitsSet", "innerBitsSet"}, + {intersect, "full", "outerBitsSet", "outerBitsSet"}, + // + {intersect, "firstBitSet", "empty", "empty"}, + {intersect, "firstBitSet", "full", "firstBitSet"}, + {intersect, "firstBitSet", "firstBitSet", "firstBitSet"}, + {intersect, "firstBitSet", "lastBitSet", "empty"}, + {intersect, "firstBitSet", "firstBitUnset", "empty"}, + {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, + {intersect, "firstBitSet", "innerBitsSet", "empty"}, + {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + // + {intersect, "lastBitSet", "empty", "empty"}, + {intersect, "lastBitSet", "full", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitSet", "empty"}, + {intersect, "lastBitSet", "lastBitSet", "lastBitSet"}, + {intersect, "lastBitSet", "firstBitUnset", "lastBitSet"}, + {intersect, "lastBitSet", "lastBitUnset", "empty"}, + {intersect, "lastBitSet", "innerBitsSet", "empty"}, + {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + // + {intersect, "firstBitUnset", "empty", "empty"}, + {intersect, "firstBitUnset", "full", "firstBitUnset"}, + {intersect, "firstBitUnset", "firstBitSet", "empty"}, + {intersect, "firstBitUnset", "lastBitSet", "lastBitSet"}, + {intersect, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, + {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + // + {intersect, "lastBitUnset", "empty", "empty"}, + {intersect, "lastBitUnset", "full", "lastBitUnset"}, + {intersect, "lastBitUnset", "firstBitSet", "firstBitSet"}, + {intersect, "lastBitUnset", "lastBitSet", "empty"}, + {intersect, "lastBitUnset", "firstBitUnset", "innerBitsSet"}, + {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, + {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + // + {intersect, "innerBitsSet", "empty", "empty"}, + {intersect, "innerBitsSet", "full", "innerBitsSet"}, + {intersect, "innerBitsSet", "firstBitSet", "empty"}, + {intersect, "innerBitsSet", "lastBitSet", "empty"}, + {intersect, "innerBitsSet", "firstBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, + {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + // + {intersect, "outerBitsSet", "empty", "empty"}, + {intersect, "outerBitsSet", "full", "outerBitsSet"}, + {intersect, "outerBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "outerBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "outerBitsSet", "firstBitUnset", "lastBitSet"}, + {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, + {intersect, "outerBitsSet", "innerBitsSet", "empty"}, + {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + + // union + {union, "empty", "empty", "empty"}, + {union, "empty", "full", "full"}, + {union, "empty", "firstBitSet", "firstBitSet"}, + {union, "empty", "lastBitSet", "lastBitSet"}, + {union, "empty", "firstBitUnset", "firstBitUnset"}, + {union, "empty", "lastBitUnset", "lastBitUnset"}, + {union, "empty", "innerBitsSet", "innerBitsSet"}, + {union, "empty", "outerBitsSet", "outerBitsSet"}, + // + {union, "full", "empty", "full"}, + {union, "full", "full", "full"}, + {union, "full", "firstBitSet", "full"}, + {union, "full", "lastBitSet", "full"}, + {union, "full", "firstBitUnset", "full"}, + {union, "full", "lastBitUnset", "full"}, + {union, "full", "innerBitsSet", "full"}, + {union, "full", "outerBitsSet", "full"}, + // + {union, "firstBitSet", "empty", "firstBitSet"}, + {union, "firstBitSet", "full", "full"}, + {union, "firstBitSet", "firstBitSet", "firstBitSet"}, + {union, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {union, "firstBitSet", "firstBitUnset", "full"}, + {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, + {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + // + {union, "lastBitSet", "empty", "lastBitSet"}, + {union, "lastBitSet", "full", "full"}, + {union, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {union, "lastBitSet", "lastBitSet", "lastBitSet"}, + {union, "lastBitSet", "firstBitUnset", "firstBitUnset"}, + {union, "lastBitSet", "lastBitUnset", "full"}, + {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + // + {union, "firstBitUnset", "empty", "firstBitUnset"}, + {union, "firstBitUnset", "full", "full"}, + {union, "firstBitUnset", "firstBitSet", "full"}, + {union, "firstBitUnset", "lastBitSet", "firstBitUnset"}, + {union, "firstBitUnset", "firstBitUnset", "firstBitUnset"}, + {union, "firstBitUnset", "lastBitUnset", "full"}, + {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "outerBitsSet", "full"}, + // + {union, "lastBitUnset", "empty", "lastBitUnset"}, + {union, "lastBitUnset", "full", "full"}, + {union, "lastBitUnset", "firstBitSet", "lastBitUnset"}, + {union, "lastBitUnset", "lastBitSet", "full"}, + {union, "lastBitUnset", "firstBitUnset", "full"}, + {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, + {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, + {union, "lastBitUnset", "outerBitsSet", "full"}, + // + {union, "innerBitsSet", "empty", "innerBitsSet"}, + {union, "innerBitsSet", "full", "full"}, + {union, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {union, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {union, "innerBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, + {union, "innerBitsSet", "outerBitsSet", "full"}, + // + {union, "outerBitsSet", "empty", "outerBitsSet"}, + {union, "outerBitsSet", "full", "full"}, + {union, "outerBitsSet", "firstBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "lastBitSet", "outerBitsSet"}, + {union, "outerBitsSet", "firstBitUnset", "full"}, + {union, "outerBitsSet", "lastBitUnset", "full"}, + {union, "outerBitsSet", "innerBitsSet", "full"}, + {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + + // difference + {difference, "empty", "empty", "empty"}, + {difference, "empty", "full", "empty"}, + {difference, "empty", "firstBitSet", "empty"}, + {difference, "empty", "lastBitSet", "empty"}, + {difference, "empty", "firstBitUnset", "empty"}, + {difference, "empty", "lastBitUnset", "empty"}, + {difference, "empty", "innerBitsSet", "empty"}, + {difference, "empty", "outerBitsSet", "empty"}, + // + {difference, "full", "empty", "full"}, + {difference, "full", "full", "empty"}, + {difference, "full", "firstBitSet", "firstBitUnset"}, + {difference, "full", "lastBitSet", "lastBitUnset"}, + {difference, "full", "firstBitUnset", "firstBitSet"}, + {difference, "full", "lastBitUnset", "lastBitSet"}, + {difference, "full", "innerBitsSet", "outerBitsSet"}, + {difference, "full", "outerBitsSet", "innerBitsSet"}, + // + {difference, "firstBitSet", "empty", "firstBitSet"}, + {difference, "firstBitSet", "full", "empty"}, + {difference, "firstBitSet", "firstBitSet", "empty"}, + {difference, "firstBitSet", "lastBitSet", "firstBitSet"}, + {difference, "firstBitSet", "firstBitUnset", "firstBitSet"}, + {difference, "firstBitSet", "lastBitUnset", "empty"}, + {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "outerBitsSet", "empty"}, + // + {difference, "lastBitSet", "empty", "lastBitSet"}, + {difference, "lastBitSet", "full", "empty"}, + {difference, "lastBitSet", "firstBitSet", "lastBitSet"}, + {difference, "lastBitSet", "lastBitSet", "empty"}, + {difference, "lastBitSet", "firstBitUnset", "empty"}, + {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, + {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, + {difference, "lastBitSet", "outerBitsSet", "empty"}, + // + {difference, "firstBitUnset", "empty", "firstBitUnset"}, + {difference, "firstBitUnset", "full", "empty"}, + {difference, "firstBitUnset", "firstBitSet", "firstBitUnset"}, + {difference, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {difference, "firstBitUnset", "firstBitUnset", "empty"}, + {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, + {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + // + {difference, "lastBitUnset", "empty", "lastBitUnset"}, + {difference, "lastBitUnset", "full", "empty"}, + {difference, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {difference, "lastBitUnset", "lastBitSet", "lastBitUnset"}, + {difference, "lastBitUnset", "firstBitUnset", "firstBitSet"}, + {difference, "lastBitUnset", "lastBitUnset", "empty"}, + {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + // + {difference, "innerBitsSet", "empty", "innerBitsSet"}, + {difference, "innerBitsSet", "full", "empty"}, + {difference, "innerBitsSet", "firstBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "lastBitSet", "innerBitsSet"}, + {difference, "innerBitsSet", "firstBitUnset", "empty"}, + {difference, "innerBitsSet", "lastBitUnset", "empty"}, + {difference, "innerBitsSet", "innerBitsSet", "empty"}, + {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + // + {difference, "outerBitsSet", "empty", "outerBitsSet"}, + {difference, "outerBitsSet", "full", "empty"}, + {difference, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {difference, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {difference, "outerBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, + {difference, "outerBitsSet", "outerBitsSet", "empty"}, + + // xor + {xor, "empty", "empty", "empty"}, + {xor, "empty", "full", "full"}, + {xor, "empty", "firstBitSet", "firstBitSet"}, + {xor, "empty", "lastBitSet", "lastBitSet"}, + {xor, "empty", "firstBitUnset", "firstBitUnset"}, + {xor, "empty", "lastBitUnset", "lastBitUnset"}, + {xor, "empty", "innerBitsSet", "innerBitsSet"}, + {xor, "empty", "outerBitsSet", "outerBitsSet"}, + // + {xor, "full", "empty", "full"}, + {xor, "full", "full", "empty"}, + {xor, "full", "firstBitSet", "firstBitUnset"}, + {xor, "full", "lastBitSet", "lastBitUnset"}, + {xor, "full", "firstBitUnset", "firstBitSet"}, + {xor, "full", "lastBitUnset", "lastBitSet"}, + {xor, "full", "innerBitsSet", "outerBitsSet"}, + {xor, "full", "outerBitsSet", "innerBitsSet"}, + // + {xor, "firstBitSet", "empty", "firstBitSet"}, + {xor, "firstBitSet", "full", "firstBitUnset"}, + {xor, "firstBitSet", "firstBitSet", "empty"}, + {xor, "firstBitSet", "lastBitSet", "outerBitsSet"}, + {xor, "firstBitSet", "firstBitUnset", "full"}, + {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, + {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, + {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + // + {xor, "lastBitSet", "empty", "lastBitSet"}, + {xor, "lastBitSet", "full", "lastBitUnset"}, + {xor, "lastBitSet", "firstBitSet", "outerBitsSet"}, + {xor, "lastBitSet", "lastBitSet", "empty"}, + {xor, "lastBitSet", "firstBitUnset", "innerBitsSet"}, + {xor, "lastBitSet", "lastBitUnset", "full"}, + {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, + {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + // + {xor, "firstBitUnset", "empty", "firstBitUnset"}, + {xor, "firstBitUnset", "full", "firstBitSet"}, + {xor, "firstBitUnset", "firstBitSet", "full"}, + {xor, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {xor, "firstBitUnset", "firstBitUnset", "empty"}, + {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, + {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + // + {xor, "lastBitUnset", "empty", "lastBitUnset"}, + {xor, "lastBitUnset", "full", "lastBitSet"}, + {xor, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {xor, "lastBitUnset", "lastBitSet", "full"}, + {xor, "lastBitUnset", "firstBitUnset", "outerBitsSet"}, + {xor, "lastBitUnset", "lastBitUnset", "empty"}, + {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + // + {xor, "innerBitsSet", "empty", "innerBitsSet"}, + {xor, "innerBitsSet", "full", "outerBitsSet"}, + {xor, "innerBitsSet", "firstBitSet", "lastBitUnset"}, + {xor, "innerBitsSet", "lastBitSet", "firstBitUnset"}, + {xor, "innerBitsSet", "firstBitUnset", "lastBitSet"}, + {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, + {xor, "innerBitsSet", "innerBitsSet", "empty"}, + {xor, "innerBitsSet", "outerBitsSet", "full"}, + // + {xor, "outerBitsSet", "empty", "outerBitsSet"}, + {xor, "outerBitsSet", "full", "innerBitsSet"}, + {xor, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {xor, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {xor, "outerBitsSet", "firstBitUnset", "lastBitUnset"}, + {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, + {xor, "outerBitsSet", "innerBitsSet", "full"}, + {xor, "outerBitsSet", "outerBitsSet", "empty"}, + } + for _, testOp := range testOps { + for _, x := range containerTypes { + for _, y := range containerTypes { + desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) + ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y]) + exp := testOp.exp + + // Convert to all container types and check result. + for _, ct := range containerTypes { + clone := ret.clone() + if ct == ContainerArray { + if clone.isBitmap() { + clone.bitmapToArray() + } else if clone.isRun() { + clone.runToArray() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + // Because xorRunRun resulting in an empty container returns an array container with a + // nil slice array, then we need to check len() on array first (look for 0). + if !(len(clone.array) == 0 && len(cts[ct][exp].array) == 0) && !reflect.DeepEqual(clone.array, cts[ct][exp].array) { + t.Fatalf("test %s expected array %X, but got %X", desc, cts[ct][exp].array, clone.array) + } + } else if ct == ContainerBitmap { + if clone.isArray() { + clone.arrayToBitmap() + } else if clone.isRun() { + clone.runToBitmap() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.bitmap, cts[ct][exp].bitmap) { + t.Fatalf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap, clone.bitmap) + } + } else if ct == ContainerRun { + if clone.isArray() { + clone.arrayToRun() + } else if clone.isBitmap() { + clone.bitmapToRun() + } + if clone.n != cts[ct][exp].n { + t.Fatalf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].n, clone.n) + } + if !reflect.DeepEqual(clone.runs, cts[ct][exp].runs) { + t.Fatalf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs, clone.runs) + } + } + } + } + } + } +} From 673232baf9261e85d440238bd1c77e770a3955e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 13:30:38 -0600 Subject: [PATCH 17/19] bug fixes found by running container operation tests --- roaring/roaring.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b7cdec1c9..b42da1418 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2181,7 +2181,7 @@ func unionArrayArray(a, b *container) *container { // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } output := &container{containerType: ContainerRun} @@ -2238,10 +2238,10 @@ func (c *container) runAppendInterval(v interval16) int { } func unionRunRun(a, b *container) *container { - if a.n == maxContainerVal { + if a.n == maxContainerVal+1 { return a.clone() } - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } na, nb := len(a.runs), len(b.runs) @@ -2272,9 +2272,12 @@ func unionRunRun(a, b *container) *container { } func unionBitmapRun(a, b *container) *container { - if b.n == maxContainerVal { + if b.n == maxContainerVal+1 { return b.clone() } + if a.n == maxContainerVal+1 { + return a.clone() + } output := a.clone() for j := 0; j < len(b.runs); j++ { output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) @@ -2289,7 +2292,7 @@ func (c *container) bitmapSetRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) xcnt := popcnt(X) ycnt := popcnt(Y) if x == y { @@ -2312,7 +2315,7 @@ func (c *container) bitmapXorRange(i, j uint64) { x := i >> 6 y := (j - 1) >> 6 var X uint64 = maxBitmap << (i % 64) - var Y uint64 = maxBitmap >> (64 - (j % 64)) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { cnt := popcnt(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip @@ -2784,7 +2787,9 @@ func xorArrayBitmap(a, b *container) *container { } } - if output.count() < ArrayMaxSize { + // It's possible that output was converted from bitmap to array in output.remove() + // so we only do this conversion if output is still a bitmap container. + if output.containerType == ContainerBitmap && output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3199,7 +3204,7 @@ func xorRunRun(a, b *container) *container { if nb == 0 { return a.clone() } - output := &container{} + output := &container{containerType: ContainerRun} lastI, lastJ := -1, -1 From 911af238ff8b6cf5ff2a5f4af52b0ecafc7ee19a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 14 Feb 2018 15:06:53 -0600 Subject: [PATCH 18/19] clean up roaring container helper functions --- roaring/roaring_helpers_test.go | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index de3dd002c..204b2fe2f 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -68,10 +68,7 @@ func arrayInnerBitsSet() []uint16 { } func arrayOuterBitsSet() []uint16 { - array := make([]uint16, 0) - array = append(array, uint16(0)) - array = append(array, uint16(65535)) - return array + return []uint16{0, 65535} } ////////////////// bitmap @@ -100,39 +97,27 @@ func bitmapLastBitSet() []uint64 { } func bitmapFirstBitUnset() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapFull() bitmap[0] = 0xFFFFFFFFFFFFFFFE - for i := 1; i < bitmapN; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } return bitmap } func bitmapLastBitUnset() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN-1; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } + bitmap := bitmapFull() bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF return bitmap } func bitmapInnerBitsSet() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapFull() bitmap[0] = 0xFFFFFFFFFFFFFFFE - for i := 1; i < bitmapN-1; i++ { - bitmap[i] = 0xFFFFFFFFFFFFFFFF - } bitmap[bitmapN-1] = 0x7FFFFFFFFFFFFFFF return bitmap } func bitmapOuterBitsSet() []uint64 { - bitmap := make([]uint64, bitmapN) + bitmap := bitmapEmpty() bitmap[0] = 0x0000000000000001 - for i := 1; i < bitmapN-1; i++ { - bitmap[i] = 0x0000000000000000 - } bitmap[bitmapN-1] = 0x8000000000000000 return bitmap } From 685a0075ae52779579adeed6a750be77d59cb263 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 15 Feb 2018 10:20:54 -0600 Subject: [PATCH 19/19] add evenBits/oddBits container tests and related bug fix --- roaring/roaring.go | 16 ++- roaring/roaring_helpers_test.go | 54 +++++++++ roaring/roaring_internal_test.go | 187 ++++++++++++++++++++++++++----- 3 files changed, 223 insertions(+), 34 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b42da1418..8d88b2088 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2476,10 +2476,18 @@ func differenceArrayRun(a, b *container) *container { if i < len(a.array) { // keep all array elements after end of runs - output.array = append(output.array, a.array[i:]...) - // TODO: consider handling container.n mutations in one place - // like we do with container.add(). - output.n += len(a.array[i:]) + // It's possible that output was converted from array to bitmap in output.add() + // so check container type before proceeding. + if output.containerType == ContainerArray { + output.array = append(output.array, a.array[i:]...) + // TODO: consider handling container.n mutations in one place + // like we do with container.add(). + output.n += len(a.array[i:]) + } else { + for _, v := range a.array[i:] { + output.add(v) + } + } } return output } diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 204b2fe2f..ea6d86cd9 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -71,6 +71,22 @@ func arrayOuterBitsSet() []uint16 { return []uint16{0, 65535} } +func arrayOddBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2*i + 1) + } + return array +} + +func arrayEvenBitsSet() []uint16 { + array := make([]uint16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + array[i] = uint16(2 * i) + } + return array +} + ////////////////// bitmap func bitmapEmpty() []uint64 { return make([]uint64, bitmapN) @@ -122,6 +138,22 @@ func bitmapOuterBitsSet() []uint64 { return bitmap } +func bitmapOddBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0xAAAAAAAAAAAAAAAA + } + return bitmap +} + +func bitmapEvenBitsSet() []uint64 { + bitmap := make([]uint64, bitmapN) + for i := 0; i < bitmapN; i++ { + bitmap[i] = 0x5555555555555555 + } + return bitmap +} + ////////////////// run func runEmpty() []interval16 { return make([]interval16, 0) @@ -170,6 +202,22 @@ func runOuterBitsSet() []interval16 { return run } +func runOddBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2*i + 1), last: uint16(2*i + 1)} + } + return run +} + +func runEvenBitsSet() []interval16 { + run := make([]interval16, containerWidth/2) + for i := 0; i < int(containerWidth/2); i++ { + run[i] = interval16{start: uint16(2 * i), last: uint16(2 * i)} + } + return run +} + /////////////////////////////////////////////////////////////////////////// type testOp struct { @@ -211,6 +259,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()), "innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()), "outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()), + "oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()), + "evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()), } // bitmap containers @@ -223,6 +273,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()), "innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()), "outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()), + "oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()), + "evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()), } // run containers @@ -235,6 +287,8 @@ func setupContainerTests() map[byte]map[string]*container { "lastBitUnset": doContainer(ContainerRun, runLastBitUnset()), "innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()), "outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()), + "oddBitsSet": doContainer(ContainerRun, runOddBitsSet()), + "evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()), } return cts diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index ce9ab217d..dca413811 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -1641,7 +1641,7 @@ func TestDifferenceBitmapRun(t *testing.T) { exp: []uint64{0x0000000000000000}, }, { - bitmap: bitmapLast(), + bitmap: bitmapLastBitSet(), runs: []interval16{{start: 65535, last: 65535}}, exp: bitmapEmpty(), }, @@ -1689,12 +1689,12 @@ func TestDifferenceBitmapArray(t *testing.T) { exp: []uint16{8, 9, 11, 12, 13, 14, 15}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10}, exp: []uint16{9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63}, }, { - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), array: []uint16{63}, exp: []uint16{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61}, }, @@ -2610,12 +2610,12 @@ func TestIntersectArrayBitmap(t *testing.T) { }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapOdds(), + bitmap: bitmapOddBitsSet(), exp: []uint16{1, 63, 543, 639, 65535}, }, { array: []uint16{0, 1, 63, 120, 543, 639, 12000, 65534, 65535}, - bitmap: bitmapEvens(), + bitmap: bitmapEvenBitsSet(), exp: []uint16{0, 120, 12000, 65534}, }, } @@ -2637,31 +2637,6 @@ func TestIntersectArrayBitmap(t *testing.T) { } } -func bitmapOdds() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0xAAAAAAAAAAAAAAAA - } - return bitmap -} - -func bitmapEvens() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN; i++ { - bitmap[i] = 0x5555555555555555 - } - return bitmap -} - -func bitmapLast() []uint64 { - bitmap := make([]uint64, bitmapN) - for i := 0; i < bitmapN-1; i++ { - bitmap[i] = 0 - } - bitmap[bitmapN-1] = 0x8000000000000000 - return bitmap -} - // rleCont returns a slice of numbers all in the range starting from // container_width*num, and ending at container_width*(num+1)-1. If left is // true, then the first 100 bits will be set, if mid is true, 100 bits in the @@ -2776,6 +2751,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "empty", "lastBitUnset", "empty"}, {intersect, "empty", "innerBitsSet", "empty"}, {intersect, "empty", "outerBitsSet", "empty"}, + {intersect, "empty", "oddBitsSet", "empty"}, + {intersect, "empty", "evenBitsSet", "empty"}, // {intersect, "full", "empty", "empty"}, {intersect, "full", "full", "full"}, @@ -2785,6 +2762,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "full", "lastBitUnset", "lastBitUnset"}, {intersect, "full", "innerBitsSet", "innerBitsSet"}, {intersect, "full", "outerBitsSet", "outerBitsSet"}, + {intersect, "full", "oddBitsSet", "oddBitsSet"}, + {intersect, "full", "evenBitsSet", "evenBitsSet"}, // {intersect, "firstBitSet", "empty", "empty"}, {intersect, "firstBitSet", "full", "firstBitSet"}, @@ -2794,6 +2773,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "firstBitSet", "lastBitUnset", "firstBitSet"}, {intersect, "firstBitSet", "innerBitsSet", "empty"}, {intersect, "firstBitSet", "outerBitsSet", "firstBitSet"}, + {intersect, "firstBitSet", "oddBitsSet", "empty"}, + {intersect, "firstBitSet", "evenBitsSet", "firstBitSet"}, // {intersect, "lastBitSet", "empty", "empty"}, {intersect, "lastBitSet", "full", "lastBitSet"}, @@ -2803,6 +2784,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "lastBitSet", "lastBitUnset", "empty"}, {intersect, "lastBitSet", "innerBitsSet", "empty"}, {intersect, "lastBitSet", "outerBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "oddBitsSet", "lastBitSet"}, + {intersect, "lastBitSet", "evenBitsSet", "empty"}, // {intersect, "firstBitUnset", "empty", "empty"}, {intersect, "firstBitUnset", "full", "firstBitUnset"}, @@ -2812,6 +2795,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "firstBitUnset", "lastBitUnset", "innerBitsSet"}, {intersect, "firstBitUnset", "innerBitsSet", "innerBitsSet"}, {intersect, "firstBitUnset", "outerBitsSet", "lastBitSet"}, + {intersect, "firstBitUnset", "oddBitsSet", "oddBitsSet"}, + //{intersect, "firstBitUnset", "evenBitsSet", ""}, // {intersect, "lastBitUnset", "empty", "empty"}, {intersect, "lastBitUnset", "full", "lastBitUnset"}, @@ -2821,6 +2806,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, {intersect, "lastBitUnset", "innerBitsSet", "innerBitsSet"}, {intersect, "lastBitUnset", "outerBitsSet", "firstBitSet"}, + //{intersect, "lastBitUnset", "oddBitsSet", ""}, + {intersect, "lastBitUnset", "evenBitsSet", "evenBitsSet"}, // {intersect, "innerBitsSet", "empty", "empty"}, {intersect, "innerBitsSet", "full", "innerBitsSet"}, @@ -2830,6 +2817,8 @@ func TestContainerCombinations(t *testing.T) { {intersect, "innerBitsSet", "lastBitUnset", "innerBitsSet"}, {intersect, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, {intersect, "innerBitsSet", "outerBitsSet", "empty"}, + //{intersect, "innerBitsSet", "oddBitsSet", ""}, + //{intersect, "innerBitsSet", "evenBitsSet", ""}, // {intersect, "outerBitsSet", "empty", "empty"}, {intersect, "outerBitsSet", "full", "outerBitsSet"}, @@ -2839,6 +2828,30 @@ func TestContainerCombinations(t *testing.T) { {intersect, "outerBitsSet", "lastBitUnset", "firstBitSet"}, {intersect, "outerBitsSet", "innerBitsSet", "empty"}, {intersect, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + {intersect, "outerBitsSet", "oddBitsSet", "lastBitSet"}, + {intersect, "outerBitsSet", "evenBitsSet", "firstBitSet"}, + // + {intersect, "oddBitsSet", "empty", "empty"}, + {intersect, "oddBitsSet", "full", "oddBitsSet"}, + {intersect, "oddBitsSet", "firstBitSet", "empty"}, + {intersect, "oddBitsSet", "lastBitSet", "lastBitSet"}, + {intersect, "oddBitsSet", "firstBitUnset", "oddBitsSet"}, + //{intersect, "oddBitsSet", "lastBitUnset", ""}, + //{intersect, "oddBitsSet", "innerBitsSet", ""}, + {intersect, "oddBitsSet", "outerBitsSet", "lastBitSet"}, + {intersect, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {intersect, "oddBitsSet", "evenBitsSet", "empty"}, + // + {intersect, "evenBitsSet", "empty", "empty"}, + {intersect, "evenBitsSet", "full", "evenBitsSet"}, + {intersect, "evenBitsSet", "firstBitSet", "firstBitSet"}, + {intersect, "evenBitsSet", "lastBitSet", "empty"}, + //{intersect, "evenBitsSet", "firstBitUnset", ""}, + {intersect, "evenBitsSet", "lastBitUnset", "evenBitsSet"}, + //{intersect, "evenBitsSet", "innerBitsSet", ""}, + {intersect, "evenBitsSet", "outerBitsSet", "firstBitSet"}, + {intersect, "evenBitsSet", "oddBitsSet", "empty"}, + {intersect, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, // union {union, "empty", "empty", "empty"}, @@ -2849,6 +2862,8 @@ func TestContainerCombinations(t *testing.T) { {union, "empty", "lastBitUnset", "lastBitUnset"}, {union, "empty", "innerBitsSet", "innerBitsSet"}, {union, "empty", "outerBitsSet", "outerBitsSet"}, + {union, "empty", "oddBitsSet", "oddBitsSet"}, + {union, "empty", "evenBitsSet", "evenBitsSet"}, // {union, "full", "empty", "full"}, {union, "full", "full", "full"}, @@ -2858,6 +2873,8 @@ func TestContainerCombinations(t *testing.T) { {union, "full", "lastBitUnset", "full"}, {union, "full", "innerBitsSet", "full"}, {union, "full", "outerBitsSet", "full"}, + {union, "full", "oddBitsSet", "full"}, + {union, "full", "evenBitsSet", "full"}, // {union, "firstBitSet", "empty", "firstBitSet"}, {union, "firstBitSet", "full", "full"}, @@ -2867,6 +2884,8 @@ func TestContainerCombinations(t *testing.T) { {union, "firstBitSet", "lastBitUnset", "lastBitUnset"}, {union, "firstBitSet", "innerBitsSet", "lastBitUnset"}, {union, "firstBitSet", "outerBitsSet", "outerBitsSet"}, + //{union, "firstBitSet", "oddBitsSet", ""}, + {union, "firstBitSet", "evenBitsSet", "evenBitsSet"}, // {union, "lastBitSet", "empty", "lastBitSet"}, {union, "lastBitSet", "full", "full"}, @@ -2876,6 +2895,8 @@ func TestContainerCombinations(t *testing.T) { {union, "lastBitSet", "lastBitUnset", "full"}, {union, "lastBitSet", "innerBitsSet", "firstBitUnset"}, {union, "lastBitSet", "outerBitsSet", "outerBitsSet"}, + {union, "lastBitSet", "oddBitsSet", "oddBitsSet"}, + //{union, "lastBitSet", "evenBitsSet", ""}, // {union, "firstBitUnset", "empty", "firstBitUnset"}, {union, "firstBitUnset", "full", "full"}, @@ -2885,6 +2906,8 @@ func TestContainerCombinations(t *testing.T) { {union, "firstBitUnset", "lastBitUnset", "full"}, {union, "firstBitUnset", "innerBitsSet", "firstBitUnset"}, {union, "firstBitUnset", "outerBitsSet", "full"}, + {union, "firstBitUnset", "oddBitsSet", "firstBitUnset"}, + {union, "firstBitUnset", "evenBitsSet", "full"}, // {union, "lastBitUnset", "empty", "lastBitUnset"}, {union, "lastBitUnset", "full", "full"}, @@ -2894,6 +2917,8 @@ func TestContainerCombinations(t *testing.T) { {union, "lastBitUnset", "lastBitUnset", "lastBitUnset"}, {union, "lastBitUnset", "innerBitsSet", "lastBitUnset"}, {union, "lastBitUnset", "outerBitsSet", "full"}, + {union, "lastBitUnset", "oddBitsSet", "full"}, + {union, "lastBitUnset", "evenBitsSet", "lastBitUnset"}, // {union, "innerBitsSet", "empty", "innerBitsSet"}, {union, "innerBitsSet", "full", "full"}, @@ -2903,6 +2928,8 @@ func TestContainerCombinations(t *testing.T) { {union, "innerBitsSet", "lastBitUnset", "lastBitUnset"}, {union, "innerBitsSet", "innerBitsSet", "innerBitsSet"}, {union, "innerBitsSet", "outerBitsSet", "full"}, + {union, "innerBitsSet", "oddBitsSet", "firstBitUnset"}, + {union, "innerBitsSet", "evenBitsSet", "lastBitUnset"}, // {union, "outerBitsSet", "empty", "outerBitsSet"}, {union, "outerBitsSet", "full", "full"}, @@ -2912,6 +2939,30 @@ func TestContainerCombinations(t *testing.T) { {union, "outerBitsSet", "lastBitUnset", "full"}, {union, "outerBitsSet", "innerBitsSet", "full"}, {union, "outerBitsSet", "outerBitsSet", "outerBitsSet"}, + //{union, "outerBitsSet", "oddBitsSet", ""}, + //{union, "outerBitsSet", "evenBitsSet", ""}, + // + {union, "oddBitsSet", "empty", "oddBitsSet"}, + {union, "oddBitsSet", "full", "full"}, + //{union, "oddBitsSet", "firstBitSet", ""}, + {union, "oddBitsSet", "lastBitSet", "oddBitsSet"}, + {union, "oddBitsSet", "firstBitUnset", "firstBitUnset"}, + {union, "oddBitsSet", "lastBitUnset", "full"}, + {union, "oddBitsSet", "innerBitsSet", "firstBitUnset"}, + //{union, "oddBitsSet", "outerBitsSet", ""}, + {union, "oddBitsSet", "oddBitsSet", "oddBitsSet"}, + {union, "oddBitsSet", "evenBitsSet", "full"}, + // + {union, "evenBitsSet", "empty", "evenBitsSet"}, + {union, "evenBitsSet", "full", "full"}, + {union, "evenBitsSet", "firstBitSet", "evenBitsSet"}, + //{union, "evenBitsSet", "lastBitSet", ""}, + {union, "evenBitsSet", "firstBitUnset", "full"}, + {union, "evenBitsSet", "lastBitUnset", "lastBitUnset"}, + {union, "evenBitsSet", "innerBitsSet", "lastBitUnset"}, + //{union, "evenBitsSet", "outerBitsSet", ""}, + {union, "evenBitsSet", "oddBitsSet", "full"}, + {union, "evenBitsSet", "evenBitsSet", "evenBitsSet"}, // difference {difference, "empty", "empty", "empty"}, @@ -2922,6 +2973,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "empty", "lastBitUnset", "empty"}, {difference, "empty", "innerBitsSet", "empty"}, {difference, "empty", "outerBitsSet", "empty"}, + {difference, "empty", "oddBitsSet", "empty"}, + {difference, "empty", "evenBitsSet", "empty"}, // {difference, "full", "empty", "full"}, {difference, "full", "full", "empty"}, @@ -2931,6 +2984,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "full", "lastBitUnset", "lastBitSet"}, {difference, "full", "innerBitsSet", "outerBitsSet"}, {difference, "full", "outerBitsSet", "innerBitsSet"}, + {difference, "full", "oddBitsSet", "evenBitsSet"}, + {difference, "full", "evenBitsSet", "oddBitsSet"}, // {difference, "firstBitSet", "empty", "firstBitSet"}, {difference, "firstBitSet", "full", "empty"}, @@ -2940,6 +2995,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "firstBitSet", "lastBitUnset", "empty"}, {difference, "firstBitSet", "innerBitsSet", "firstBitSet"}, {difference, "firstBitSet", "outerBitsSet", "empty"}, + {difference, "firstBitSet", "oddBitsSet", "firstBitSet"}, + {difference, "firstBitSet", "evenBitsSet", "empty"}, // {difference, "lastBitSet", "empty", "lastBitSet"}, {difference, "lastBitSet", "full", "empty"}, @@ -2949,6 +3006,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "lastBitSet", "lastBitUnset", "lastBitSet"}, {difference, "lastBitSet", "innerBitsSet", "lastBitSet"}, {difference, "lastBitSet", "outerBitsSet", "empty"}, + {difference, "lastBitSet", "oddBitsSet", "empty"}, + {difference, "lastBitSet", "evenBitsSet", "lastBitSet"}, // {difference, "firstBitUnset", "empty", "firstBitUnset"}, {difference, "firstBitUnset", "full", "empty"}, @@ -2958,6 +3017,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "firstBitUnset", "lastBitUnset", "lastBitSet"}, {difference, "firstBitUnset", "innerBitsSet", "lastBitSet"}, {difference, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + //{difference, "firstBitUnset", "oddBitsSet", ""}, + {difference, "firstBitUnset", "evenBitsSet", "oddBitsSet"}, // {difference, "lastBitUnset", "empty", "lastBitUnset"}, {difference, "lastBitUnset", "full", "empty"}, @@ -2967,6 +3028,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "lastBitUnset", "lastBitUnset", "empty"}, {difference, "lastBitUnset", "innerBitsSet", "firstBitSet"}, {difference, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + {difference, "lastBitUnset", "oddBitsSet", "evenBitsSet"}, + //{difference, "lastBitUnset", "evenBitsSet", ""}, // {difference, "innerBitsSet", "empty", "innerBitsSet"}, {difference, "innerBitsSet", "full", "empty"}, @@ -2976,6 +3039,8 @@ func TestContainerCombinations(t *testing.T) { {difference, "innerBitsSet", "lastBitUnset", "empty"}, {difference, "innerBitsSet", "innerBitsSet", "empty"}, {difference, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + //{difference, "innerBitsSet", "oddBitsSet", ""}, + //{difference, "innerBitsSet", "evenBitsSet", ""}, // {difference, "outerBitsSet", "empty", "outerBitsSet"}, {difference, "outerBitsSet", "full", "empty"}, @@ -2985,6 +3050,30 @@ func TestContainerCombinations(t *testing.T) { {difference, "outerBitsSet", "lastBitUnset", "lastBitSet"}, {difference, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, {difference, "outerBitsSet", "outerBitsSet", "empty"}, + {difference, "outerBitsSet", "oddBitsSet", "firstBitSet"}, + {difference, "outerBitsSet", "evenBitsSet", "lastBitSet"}, + // + {difference, "oddBitsSet", "empty", "oddBitsSet"}, + {difference, "oddBitsSet", "full", "empty"}, + {difference, "oddBitsSet", "firstBitSet", "oddBitsSet"}, + //{difference, "oddBitsSet", "lastBitSet", ""}, + {difference, "oddBitsSet", "firstBitUnset", "empty"}, + {difference, "oddBitsSet", "lastBitUnset", "lastBitSet"}, + {difference, "oddBitsSet", "innerBitsSet", "lastBitSet"}, + //{difference, "oddBitsSet", "outerBitsSet", ""}, + {difference, "oddBitsSet", "oddBitsSet", "empty"}, + {difference, "oddBitsSet", "evenBitsSet", "oddBitsSet"}, + // + {difference, "evenBitsSet", "empty", "evenBitsSet"}, + {difference, "evenBitsSet", "full", "empty"}, + //{difference, "evenBitsSet", "firstBitSet", ""}, + {difference, "evenBitsSet", "lastBitSet", "evenBitsSet"}, + {difference, "evenBitsSet", "firstBitUnset", "firstBitSet"}, + {difference, "evenBitsSet", "lastBitUnset", "empty"}, + {difference, "evenBitsSet", "innerBitsSet", "firstBitSet"}, + //{difference, "evenBitsSet", "outerBitsSet", ""}, + {difference, "evenBitsSet", "oddBitsSet", "evenBitsSet"}, + {difference, "evenBitsSet", "evenBitsSet", "empty"}, // xor {xor, "empty", "empty", "empty"}, @@ -2995,6 +3084,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "empty", "lastBitUnset", "lastBitUnset"}, {xor, "empty", "innerBitsSet", "innerBitsSet"}, {xor, "empty", "outerBitsSet", "outerBitsSet"}, + {xor, "empty", "oddBitsSet", "oddBitsSet"}, + {xor, "empty", "evenBitsSet", "evenBitsSet"}, // {xor, "full", "empty", "full"}, {xor, "full", "full", "empty"}, @@ -3004,6 +3095,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "full", "lastBitUnset", "lastBitSet"}, {xor, "full", "innerBitsSet", "outerBitsSet"}, {xor, "full", "outerBitsSet", "innerBitsSet"}, + {xor, "full", "oddBitsSet", "evenBitsSet"}, + {xor, "full", "evenBitsSet", "oddBitsSet"}, // {xor, "firstBitSet", "empty", "firstBitSet"}, {xor, "firstBitSet", "full", "firstBitUnset"}, @@ -3013,6 +3106,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "firstBitSet", "lastBitUnset", "innerBitsSet"}, {xor, "firstBitSet", "innerBitsSet", "lastBitUnset"}, {xor, "firstBitSet", "outerBitsSet", "lastBitSet"}, + //{xor, "firstBitSet", "oddBitsSet", ""}, + //{xor, "firstBitSet", "evenBitsSet", ""}, // {xor, "lastBitSet", "empty", "lastBitSet"}, {xor, "lastBitSet", "full", "lastBitUnset"}, @@ -3022,6 +3117,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "lastBitSet", "lastBitUnset", "full"}, {xor, "lastBitSet", "innerBitsSet", "firstBitUnset"}, {xor, "lastBitSet", "outerBitsSet", "firstBitSet"}, + //{xor, "lastBitSet", "oddBitsSet", ""}, + //{xor, "lastBitSet", "evenBitsSet", ""}, // {xor, "firstBitUnset", "empty", "firstBitUnset"}, {xor, "firstBitUnset", "full", "firstBitSet"}, @@ -3031,6 +3128,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "firstBitUnset", "lastBitUnset", "outerBitsSet"}, {xor, "firstBitUnset", "innerBitsSet", "lastBitSet"}, {xor, "firstBitUnset", "outerBitsSet", "lastBitUnset"}, + //{xor, "firstBitUnset", "oddBitsSet", ""}, + //{xor, "firstBitUnset", "evenBitsSet", ""}, // {xor, "lastBitUnset", "empty", "lastBitUnset"}, {xor, "lastBitUnset", "full", "lastBitSet"}, @@ -3040,6 +3139,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "lastBitUnset", "lastBitUnset", "empty"}, {xor, "lastBitUnset", "innerBitsSet", "firstBitSet"}, {xor, "lastBitUnset", "outerBitsSet", "firstBitUnset"}, + //{xor, "lastBitUnset", "oddBitsSet", ""}, + //{xor, "lastBitUnset", "evenBitsSet", ""}, // {xor, "innerBitsSet", "empty", "innerBitsSet"}, {xor, "innerBitsSet", "full", "outerBitsSet"}, @@ -3049,6 +3150,8 @@ func TestContainerCombinations(t *testing.T) { {xor, "innerBitsSet", "lastBitUnset", "firstBitSet"}, {xor, "innerBitsSet", "innerBitsSet", "empty"}, {xor, "innerBitsSet", "outerBitsSet", "full"}, + //{xor, "innerBitsSet", "oddBitsSet", ""}, + //{xor, "innerBitsSet", "evenBitsSet", ""}, // {xor, "outerBitsSet", "empty", "outerBitsSet"}, {xor, "outerBitsSet", "full", "innerBitsSet"}, @@ -3058,6 +3161,30 @@ func TestContainerCombinations(t *testing.T) { {xor, "outerBitsSet", "lastBitUnset", "firstBitUnset"}, {xor, "outerBitsSet", "innerBitsSet", "full"}, {xor, "outerBitsSet", "outerBitsSet", "empty"}, + //{xor, "outerBitsSet", "oddBitsSet", ""}, + //{xor, "outerBitsSet", "evenBitsSet", ""}, + // + {xor, "oddBitsSet", "empty", "oddBitsSet"}, + {xor, "oddBitsSet", "full", "evenBitsSet"}, + //{xor, "oddBitsSet", "firstBitSet", ""}, + //{xor, "oddBitsSet", "lastBitSet", ""}, + //{xor, "oddBitsSet", "firstBitUnset", ""}, + //{xor, "oddBitsSet", "lastBitUnset", ""}, + //{xor, "oddBitsSet", "innerBitsSet", ""}, + //{xor, "oddBitsSet", "outerBitsSet", ""}, + {xor, "oddBitsSet", "oddBitsSet", "empty"}, + {xor, "oddBitsSet", "evenBitsSet", "full"}, + // + {xor, "evenBitsSet", "empty", "evenBitsSet"}, + {xor, "evenBitsSet", "full", "oddBitsSet"}, + //{xor, "evenBitsSet", "firstBitSet", ""}, + //{xor, "evenBitsSet", "lastBitSet", ""}, + //{xor, "evenBitsSet", "firstBitUnset", ""}, + //{xor, "evenBitsSet", "lastBitUnset", ""}, + //{xor, "evenBitsSet", "innerBitsSet", ""}, + //{xor, "evenBitsSet", "outerBitsSet", ""}, + {xor, "evenBitsSet", "oddBitsSet", "full"}, + {xor, "evenBitsSet", "evenBitsSet", "empty"}, } for _, testOp := range testOps { for _, x := range containerTypes {