diff --git a/Gopkg.lock b/Gopkg.lock index 4114168b3..7837cadff 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -14,10 +14,16 @@ revision = "39b0596a2da3c92787b3319c6b5425a474b4e0da" [[projects]] + branch = "master" name = "github.com/DataDog/datadog-go" packages = ["statsd"] - revision = "0ddda6bee21174ef6c4873647cb0d6ec9cba996f" - version = "1.1.0" + revision = "4d2e5696ebe914940bd7459d2266fb7d555ea1b7" + +[[projects]] + branch = "master" + name = "github.com/StackExchange/wmi" + packages = ["."] + revision = "ea383cf3ba6ec950874b8486cd72356d007c768f" [[projects]] branch = "master" @@ -43,11 +49,17 @@ revision = "629574ca2a5df945712d3079857300b5e4da0236" version = "v1.4.2" +[[projects]] + name = "github.com/go-ole/go-ole" + packages = [".","oleutil"] + revision = "0e87ea779d9deb219633b828a023b32e1244dd57" + version = "v1.2.0" + [[projects]] name = "github.com/gogo/protobuf" packages = ["proto"] - revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" - version = "v0.5" + revision = "100ba4e885062801d56799d78530b73b178a78f3" + version = "v0.4" [[projects]] branch = "master" @@ -151,6 +163,12 @@ revision = "16398bac157da96aa88f98a2df640c7f32af1da2" version = "v1.0.1" +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] name = "github.com/rakyll/statik" packages = ["fs"] @@ -169,6 +187,12 @@ packages = ["."] revision = "e2103e2c35297fb7e17febb81e49b312087a2372" +[[projects]] + name = "github.com/shirou/gopsutil" + packages = ["host","internal/common","mem","process"] + revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" + version = "v2.17.11" + [[projects]] name = "github.com/sony/gobreaker" packages = ["."] @@ -226,7 +250,7 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix"] + packages = ["unix","windows"] revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] @@ -244,6 +268,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "210f654a7a072d5751f0814e4d71ef0758dd53b3dc59ed462619396ef8621d81" + inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 527744da4..377ef0fdc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -5,3 +5,14 @@ [[constraint]] name = "github.com/satori/go.uuid" version = "1.1.0" + +[[constraint]] + name = "github.com/shirou/gopsutil" + version = "2.17.11" + +[[constraint]] + # Required: the root import path of the project being constrained. + name = "github.com/DataDog/datadog-go" + # Recommended: the version constraint to enforce for the project. + # Only one of "branch", "version" or "revision" can be specified. + branch = "master" diff --git a/Makefile b/Makefile index 9987810d3..e1ed7c5d5 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ DEP := $(shell command -v dep 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) PROTOC := $(shell command -v protoc 2>/dev/null) VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +STATUS := $(shell git status --porcelain) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) @@ -65,9 +66,13 @@ endif @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" release: +ifeq ($(STATUS),"") make release-build GOOS=darwin GOARCH=amd64 make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 +else + @echo "Will not create release with unclean git status." +endif prerelease-build: vendor make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa" diff --git a/cache.go b/cache.go index a6f37a409..9c909ced4 100644 --- a/cache.go +++ b/cache.go @@ -314,19 +314,22 @@ func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } // Pair holds an id/count pair. type Pair struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Count uint64 `json:"count"` } func encodePair(p Pair) *internal.Pair { return &internal.Pair{ - Key: p.ID, + ID: p.ID, + Key: p.Key, Count: p.Count, } } func decodePair(pb *internal.Pair) Pair { return Pair{ - ID: pb.Key, + ID: pb.ID, + Key: pb.Key, Count: pb.Count, } } diff --git a/client_test.go b/client_test.go index 4ba603176..17aafd7f4 100644 --- a/client_test.go +++ b/client_test.go @@ -157,7 +157,7 @@ func TestClient_MultiNode(t *testing.T) { // Check the results before every node has the correct max slice value. pairs := result.Results[0].Pairs for _, pair := range pairs { - if pair.Key == 22 && pair.Count != 3 { + if pair.ID == 22 && pair.Count != 3 { t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) } } @@ -177,10 +177,10 @@ func TestClient_MultiNode(t *testing.T) { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } p := []*internal.Pair{ - {Key: 100, Count: 12}, - {Key: 22, Count: 11}, - {Key: 98, Count: 8}, - {Key: 99, Count: 7}} + {ID: 100, Count: 12}, + {ID: 22, Count: 11}, + {ID: 98, Count: 8}, + {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. if !reflect.DeepEqual(result.Results[0].Pairs, p) { diff --git a/diagnostics/diagnostics.go b/diagnostics/diagnostics.go index b1453db22..eb761e6c9 100644 --- a/diagnostics/diagnostics.go +++ b/diagnostics/diagnostics.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/shirou/gopsutil/host" + "github.com/shirou/gopsutil/mem" "github.com/sony/gobreaker" ) @@ -94,7 +96,7 @@ func (d *Diagnostics) schedule() { // Flush sends the current metrics. func (d *Diagnostics) Flush() error { d.mu.Lock() - d.metrics["uptime"] = (time.Now().Unix() - d.startTime) + d.metrics["Uptime"] = (time.Now().Unix() - d.startTime) buf, _ := d.Encode() d.mu.Unlock() @@ -203,6 +205,41 @@ func (d *Diagnostics) logger() *log.Logger { return log.New(d.logOutput, "", log.LstdFlags) } +// EnrichWithOSInfo adds OS information to the diagnostics payload. +func (d *Diagnostics) EnrichWithOSInfo() { + osInfo, err := host.Info() + if err != nil { + d.logOutput.Write([]byte(err.Error())) + } + d.Set("HostUptime", osInfo.Uptime) + + platform, family, version, err := host.PlatformInformation() + if err != nil { + d.logOutput.Write([]byte(err.Error())) + } + d.Set("OSPlatform", platform) + d.Set("OSFamily", family) + d.Set("OSVersion", version) + + kernelVersion, err := host.KernelVersion() + if err != nil { + d.logOutput.Write([]byte(err.Error())) + } + d.Set("OSKernelVersion", kernelVersion) +} + +// EnrichWithMemoryInfo adds memory information to the diagnostics payload. +func (d *Diagnostics) EnrichWithMemoryInfo() { + memory, err := mem.VirtualMemory() + if err != nil { + d.logOutput.Write([]byte(err.Error())) + } + d.Set("MemFree", memory.Free) + d.Set("MemTotal", memory.Total) + d.Set("MemUsed", memory.Used) + +} + // VersionSegments returns the numeric segments of the version as a slice of ints. func VersionSegments(segments string) []int { segments = strings.Trim(segments, "v") diff --git a/diagnostics/diagnostics_test.go b/diagnostics/diagnostics_test.go index 8e3a7cce4..7ac4906d4 100644 --- a/diagnostics/diagnostics_test.go +++ b/diagnostics/diagnostics_test.go @@ -49,7 +49,7 @@ func TestDiagnosticsClient(t *testing.T) { t.Fatal(err) } - output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`) + output2 := []byte(`{"gg":10,"ss":"ss","Uptime":0}`) if eq, err = compareJSON(data, output2); err != nil { t.Fatal(err) } diff --git a/docs/tutorials.md b/docs/tutorials.md index 4c2cdd7bd..9c19f3a67 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -2,13 +2,15 @@ title = "Tutorials" weight = 4 nav = [ - "How To Setup a Secure Cluster", + "Setting Up a Secure Cluster", + "Using Integer Field Values", + "Storing Row and Column Attributes", ] +++ ## Tutorials -### How To Setup a Secure Cluster +### Setting Up a Secure Cluster #### Introduction @@ -212,3 +214,228 @@ curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap #### What's Next? Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa. + + +### Using Integer Field Values + +#### Introduction + +Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. + +First, create an index called `patients`: +``` +curl localhost:10101/index/patients \ + -X POST +``` + +Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient. +``` +curl localhost:10101/index/patients/frame/measurements \ + -X POST \ + -d '{"options":{"rangeEnabled": true}}' +``` + +In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame. +``` +curl localhost:10101/index/patients/frame/measurements \ + -X POST \ + -d '{"options":{ + "rangeEnabled": true, + "fields": [ + {"name": "age", "type": "int", "min": 0, "max": 120}, + {"name": "weight", "type": "int", "min": 0, "max": 500}, + {"name": "tcells", "type": "int", "min": 0, "max": 2000} + ] + }}' +``` + +If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field). + +Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. + +This query sets the age, weight, and t-cell count for the patient with ID `1` in our system: +``` +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' +``` + +In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file. + +Assuming we have a file called `ages.csv` that is structured like this: +``` +1,34 +2,57 +3,19 +4,40 +5,32 +6,71 +7,28 +8,33 +9,63 +``` +where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command: +``` +pilosa import -i patients -f measurements --field age ages.csv +``` + +Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. + +In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. +``` +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Range(frame="measurements", age > 40)' +``` +You should get the following results: +``` +{"results":[{"attrs":{},"bits":[2,6,9]}]} +``` + +You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. + +To find the average age of all patients, run a `Sum` query: +``` +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Sum(frame="measurements", field="age")' +``` +The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. +``` +{"results":[{"sum":377,"count":9}]} +``` + +You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. +``` +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")' +``` +Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. +``` +{"results":[{"sum":191,"count":3}]} +``` + +### Storing Row and Column Attributes + +#### Introduction + +Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data. + +First, create an index called `books` to use for this tutorial: +``` +curl localhost:10101/index/books \ + -X POST +``` + +Next, create a frame in the `books` index called `members` which will represent library members who have read books. +``` +curl localhost:10101/index/books/frame/members \ + -X POST \ + -d '{}' +``` + +Now, let's add some books to our index. +``` +curl localhost:10101/index/books/query \ + -X POST \ + -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972) + SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000) + SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' +``` + +And add some members. +``` +curl localhost:10101/index/books/query \ + -X POST \ + -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") + SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins") + SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks") + SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") + SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' +``` + +At this point we can query one of the `member` records by querying that row. +``` +curl localhost:10101/index/books/query \ + -X POST \ + -d 'Bitmap(frame="members", rowID=10002)' +``` +You should get the following result set: +``` +{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} +``` + +Now let's add some data to the matrix such that each pair represents a member who has read that book. +``` +curl localhost:10101/index/books/query \ + -X POST \ + -d 'SetBit(frame="members", rowID=10001, columnID=3) + SetBit(frame="members", rowID=10001, columnID=5) + + SetBit(frame="members", rowID=10002, columnID=1) + SetBit(frame="members", rowID=10002, columnID=2) + SetBit(frame="members", rowID=10002, columnID=4) + + SetBit(frame="members", rowID=10003, columnID=3) + + SetBit(frame="members", rowID=10004, columnID=4) + SetBit(frame="members", rowID=10004, columnID=5) + + SetBit(frame="members", rowID=10005, columnID=1) + SetBit(frame="members", rowID=10005, columnID=2) + SetBit(frame="members", rowID=10005, columnID=3) + SetBit(frame="members", rowID=10005, columnID=4) + SetBit(frame="members", rowID=10005, columnID=5)' +``` + +Now pull the record for `Sue Perkins` again. +``` +curl localhost:10101/index/books/query \ + -X POST \ + -d 'Bitmap(frame="members", rowID=10002)' +``` +Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read. +``` +{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} +``` + +In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query. +``` +curl localhost:10101/index/books/query?columnAttrs=true \ + -X POST \ + -d 'Bitmap(frame="members", rowID=10002)' +``` + +Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute. + +``` +{ + "results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}], + "columnAttrs":[ + {"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}}, + {"id":2,"attrs":{"name":"No Name in the Street","year":1972}}, + {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}} + ] +} +``` + +Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members: +``` +curl localhost:10101/index/books/query?columnAttrs=true \ + -X POST \ + -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' +``` + +``` +{ + "results":[{"attrs":{},"bits":[4]}], + "columnAttrs":[ + {"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}} + ] +} +``` + +Notice that we don't get row attributes on a complex query, but we still get the column attributes—in this case book information. diff --git a/frame.go b/frame.go index 03a533dbb..84af0c4fd 100644 --- a/frame.go +++ b/frame.go @@ -541,8 +541,8 @@ func (f *Frame) view(name string) *View { return f.views[name] } // Views returns a list of all views in the frame. func (f *Frame) Views() []*View { - f.mu.Lock() - defer f.mu.Unlock() + f.mu.RLock() + defer f.mu.RUnlock() other := make([]*View, 0, len(f.views)) for _, view := range f.views { diff --git a/gossip/gossip.go b/gossip/gossip.go index 84daad8b3..243d19738 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" ) // Ensure GossipMemberSet implements interfaces. @@ -90,7 +91,7 @@ func (g *GossipMemberSet) Open() error { g.memberlist, err = memberlist.Create(g.config.memberlistConfig) g.mu.Unlock() if err != nil { - return fmt.Errorf("creating memberlist: %s", err) + return errors.Wrap(err, "creating memberlist") } g.broadcasts = &memberlist.TransmitLimitedQueue{ @@ -114,7 +115,7 @@ func (g *GossipMemberSet) Open() error { err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings()) g.mu.RUnlock() if err != nil { - return fmt.Errorf("joining member set: %s", err) + return errors.Wrap(err, "joinWithRetry") } return nil } @@ -178,7 +179,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport } // memberlist config - conf := memberlist.DefaultLocalConfig() + conf := memberlist.DefaultWANConfig() conf.Transport = transport.Net conf.Name = name conf.BindAddr = host @@ -216,7 +217,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport // NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port. func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) { - port, err := strconv.Atoi(cfg.Gossip.Port) if err != nil { return nil, fmt.Errorf("convert port: %s", err) @@ -431,7 +431,7 @@ type Transport struct { //func NewTransport(host string, port int) (*memberlist.NetTransport, error) { func NewTransport(host string, port int) (*Transport, error) { // memberlist config - conf := memberlist.DefaultLocalConfig() + conf := memberlist.DefaultWANConfig() conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port diff --git a/handler.go b/handler.go index 5ae1c009a..d3e4dc5fa 100644 --- a/handler.go +++ b/handler.go @@ -1623,10 +1623,15 @@ func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { + version := Version + if strings.HasPrefix(version, "v") { + // make the version string semver-compatible + version = version[1:] + } if err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: Version, + Version: version, }); err != nil { h.logger().Printf("write version response error: %s", err) } diff --git a/handler_test.go b/handler_test.go index c1e0a4c0b..7413cb3ea 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1181,9 +1181,13 @@ func TestHandler_Version(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/version", nil) h.ServeHTTP(w, r) + version := pilosa.Version + if strings.HasPrefix(version, "v") { + version = version[1:] + } if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `{"version":"`+pilosa.Version+`"}`+"\n" { + } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/holder.go b/holder.go index 8aeb6e1e8..b60546089 100644 --- a/holder.go +++ b/holder.go @@ -299,15 +299,14 @@ func (h *Holder) index(name string) *Index { return h.indexes[name] } // Indexes returns a list of all indexes in the holder. func (h *Holder) Indexes() []*Index { - h.mu.Lock() - defer h.mu.Unlock() - + h.mu.RLock() a := make([]*Index, 0, len(h.indexes)) for _, index := range h.indexes { a = append(a, index) } - sort.Sort(indexSlice(a)) + h.mu.RUnlock() + sort.Sort(indexSlice(a)) return a } @@ -559,6 +558,9 @@ type HolderSyncer struct { Cluster *Cluster RemoteClient *http.Client + // Stats + Stats StatsClient + // Signals that the sync should stop. Closing <-chan struct{} } @@ -575,6 +577,7 @@ func (s *HolderSyncer) IsClosing() bool { // SyncHolder compares the holder on host with the local holder and resolves differences. func (s *HolderSyncer) SyncHolder() error { + ti := time.Now() // Iterate over schema in sorted order. for _, di := range s.Holder.Schema() { // Verify syncer has not closed. @@ -587,6 +590,7 @@ func (s *HolderSyncer) SyncHolder() error { return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err) } + tf := time.Now() for _, fi := range di.Frames { // Verify syncer has not closed. if s.IsClosing() { @@ -621,7 +625,11 @@ func (s *HolderSyncer) SyncHolder() error { } } } + s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0) + tf = time.Now() // reset tf } + s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0) + ti = time.Now() // reset ti } return nil @@ -634,12 +642,14 @@ func (s *HolderSyncer) syncIndex(index string) error { if idx == nil { return nil } + indexTag := fmt.Sprintf("index:%s", index) // Read block checksums. blks, err := idx.ColumnAttrStore().Blocks() if err != nil { return err } + s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag}) // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { @@ -653,6 +663,7 @@ func (s *HolderSyncer) syncIndex(index string) error { } else if len(m) == 0 { continue } + s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.URI.HostPort()}) // Update local copy. if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil { @@ -676,12 +687,15 @@ func (s *HolderSyncer) syncFrame(index, name string) error { if f == nil { return nil } + indexTag := fmt.Sprintf("index:%s", index) + frameTag := fmt.Sprintf("frame:%s", name) // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { return err } + s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) { @@ -697,6 +711,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error { } else if len(m) == 0 { continue } + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.URI.HostPort()}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { diff --git a/holder_test.go b/holder_test.go index e1bfe9d6a..16f4c8abf 100644 --- a/holder_test.go +++ b/holder_test.go @@ -490,6 +490,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { URI: cluster.Nodes[0].URI, Cluster: cluster, RemoteClient: pilosa.GetHTTPClient(nil), + Stats: pilosa.NopStatsClient, } if err := syncer.SyncHolder(); err != nil { diff --git a/internal/public.pb.go b/internal/public.pb.go index 2b39cf4fb..22afdc0f9 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -44,6 +44,7 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Bitmap struct { Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } @@ -59,6 +60,13 @@ func (m *Bitmap) GetBits() []uint64 { return nil } +func (m *Bitmap) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + func (m *Bitmap) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -67,7 +75,8 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } @@ -76,11 +85,18 @@ func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } -func (m *Pair) GetKey() uint64 { +func (m *Pair) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + +func (m *Pair) GetKey() string { if m != nil { return m.Key } - return 0 + return "" } func (m *Pair) GetCount() uint64 { @@ -148,6 +164,7 @@ func (m *Bit) GetTimestamp() int64 { type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } @@ -163,6 +180,13 @@ func (m *ColumnAttrSet) GetID() uint64 { return 0 } +func (m *ColumnAttrSet) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { return m.Attrs @@ -548,6 +572,21 @@ func (m *Bitmap) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } return i, nil } @@ -566,16 +605,22 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Key != 0 { + if m.ID != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Key)) + i = encodeVarintPublic(dAtA, i, uint64(m.ID)) } if m.Count != 0 { dAtA[i] = 0x10 i++ i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } + if len(m.Key) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } return i, nil } @@ -672,6 +717,12 @@ func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.Key) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } return i, nil } @@ -1143,18 +1194,28 @@ func (m *Bitmap) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } return n } func (m *Pair) Size() (n int) { var l int _ = l - if m.Key != 0 { - n += 1 + sovPublic(uint64(m.Key)) + if m.ID != 0 { + n += 1 + sovPublic(uint64(m.ID)) } if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } + l = len(m.Key) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } return n } @@ -1197,6 +1258,10 @@ func (m *ColumnAttrSet) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + l = len(m.Key) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } return n } @@ -1523,6 +1588,35 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -1575,9 +1669,9 @@ func (m *Pair) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } - m.Key = 0 + m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1587,7 +1681,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Key |= (uint64(b) & 0x7F) << shift + m.ID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1611,6 +1705,35 @@ func (m *Pair) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -1906,6 +2029,35 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3434,46 +3586,47 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 651 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40, - 0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc, - 0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a, - 0x96, 0xfc, 0x62, 0x3c, 0x16, 0xed, 0x77, 0xb0, 0x61, 0xcd, 0x06, 0x7e, 0x80, 0x1d, 0x1f, 0xc0, - 0x92, 0x4f, 0x40, 0xe1, 0x47, 0xd0, 0xbd, 0xe3, 0x89, 0x1d, 0x16, 0xc0, 0x82, 0xdd, 0x9c, 0x73, - 0x1f, 0xbe, 0x8f, 0x73, 0x0d, 0xd3, 0xb2, 0xbe, 0x48, 0x93, 0xcb, 0x45, 0xa9, 0x0a, 0x5d, 0xf0, - 0x51, 0x92, 0x6b, 0xa9, 0xf2, 0x38, 0xf5, 0x43, 0x18, 0x84, 0x89, 0xce, 0xe2, 0x92, 0x73, 0x70, - 0xc3, 0x44, 0x57, 0x1e, 0x9b, 0x3b, 0x81, 0x2b, 0xe8, 0xcd, 0x1f, 0x41, 0xff, 0xa9, 0xd6, 0xaa, - 0xf2, 0x7a, 0x73, 0x27, 0x98, 0x2c, 0xf7, 0x17, 0x36, 0x6e, 0x81, 0xb4, 0x30, 0x46, 0x7f, 0x01, - 0xee, 0x8b, 0x38, 0x51, 0xfc, 0x00, 0x9c, 0x67, 0xf2, 0xd6, 0x63, 0x73, 0x16, 0xb8, 0x02, 0x9f, - 0xfc, 0x2e, 0xf4, 0x8f, 0x8b, 0x3a, 0xd7, 0x5e, 0x8f, 0x38, 0x03, 0xfc, 0x25, 0x8c, 0xd6, 0x75, - 0x46, 0x6f, 0x8c, 0x59, 0xd7, 0x19, 0xc5, 0x38, 0x02, 0x9f, 0xbb, 0x31, 0x8e, 0x8d, 0x79, 0x05, - 0x4e, 0x98, 0x68, 0x34, 0x8a, 0xe2, 0x5d, 0x74, 0xd2, 0x7c, 0xc4, 0x00, 0xfe, 0x00, 0x46, 0xc7, - 0x45, 0x5a, 0x67, 0x79, 0x74, 0xd2, 0x7c, 0x69, 0x8b, 0xf9, 0x43, 0x18, 0x9f, 0x27, 0x99, 0xac, - 0x74, 0x9c, 0x95, 0x9e, 0x43, 0x29, 0x5b, 0xc2, 0x5f, 0xc1, 0x9e, 0xf1, 0xc4, 0x4e, 0xd6, 0x52, - 0xf3, 0x7d, 0xe8, 0x6d, 0xb3, 0xf7, 0xa2, 0x93, 0x7f, 0x9c, 0xc0, 0x67, 0x06, 0x2e, 0xbe, 0xba, - 0x23, 0x18, 0x9b, 0x11, 0x70, 0x70, 0xcf, 0x6f, 0x4b, 0xd9, 0xd4, 0x45, 0x6f, 0x3e, 0x87, 0xc9, - 0x5a, 0xab, 0x24, 0xbf, 0x7e, 0x1d, 0xa7, 0xb5, 0xa4, 0xaa, 0xc6, 0xa2, 0x4b, 0x61, 0x47, 0x51, - 0xae, 0x8d, 0xd9, 0xa5, 0xa2, 0xb7, 0x18, 0x3b, 0x0a, 0x8b, 0x22, 0x35, 0xc6, 0xfe, 0x9c, 0x05, - 0x23, 0xd1, 0x12, 0x7c, 0x06, 0x70, 0x9a, 0x16, 0x71, 0x13, 0x3b, 0x98, 0xb3, 0x80, 0x89, 0x0e, - 0xe3, 0x1f, 0xc1, 0x10, 0x2b, 0x7d, 0x1e, 0x97, 0x6d, 0x6f, 0xec, 0x4f, 0xbd, 0x7d, 0x65, 0x30, - 0x7d, 0x59, 0x4b, 0x75, 0x2b, 0xe4, 0xdb, 0x5a, 0x56, 0xb4, 0x03, 0xc2, 0x4d, 0x97, 0x06, 0xf0, - 0x43, 0x18, 0xac, 0xd3, 0xe4, 0x52, 0x9a, 0x49, 0xb9, 0xa2, 0x41, 0xd8, 0x6b, 0x3b, 0xe1, 0x8a, - 0x7a, 0x1d, 0x89, 0x2e, 0x85, 0x91, 0x42, 0x66, 0x85, 0xb6, 0xcd, 0x34, 0x88, 0xfb, 0x30, 0x5d, - 0xdd, 0x5c, 0xa6, 0xf5, 0x95, 0x34, 0xa1, 0x03, 0xb2, 0xee, 0x70, 0x98, 0xbd, 0xc1, 0xa4, 0xdd, - 0xa1, 0xc9, 0xde, 0xa1, 0xfc, 0xf7, 0x0c, 0xf6, 0x9a, 0xf2, 0xab, 0xb2, 0xc8, 0x2b, 0x89, 0x3b, - 0x5a, 0x29, 0x65, 0x77, 0xb4, 0x52, 0x8a, 0x1f, 0xc1, 0x50, 0xc8, 0xaa, 0x4e, 0xb5, 0x5d, 0xf3, - 0xbd, 0x76, 0x14, 0x36, 0xb6, 0x4e, 0xb5, 0xb0, 0x5e, 0xfc, 0x09, 0xec, 0xef, 0xc8, 0x06, 0xfb, - 0xc2, 0xb8, 0xfb, 0x6d, 0xdc, 0x8e, 0x5d, 0xfc, 0xe6, 0xee, 0x7f, 0x61, 0x30, 0xe9, 0x64, 0xe6, - 0x81, 0x3d, 0x43, 0x2a, 0x6b, 0xb2, 0x3c, 0x68, 0x13, 0x19, 0x5e, 0xd8, 0x33, 0x9d, 0x02, 0x3b, - 0x6b, 0xc4, 0xc4, 0xce, 0x70, 0x85, 0x78, 0x7a, 0xf6, 0xfb, 0x9d, 0x15, 0x22, 0x2d, 0x8c, 0x91, - 0x7b, 0x30, 0x3c, 0x7e, 0x13, 0xe7, 0xd7, 0xf2, 0x8a, 0xc4, 0x34, 0x12, 0x16, 0xf2, 0x45, 0x7b, - 0x8a, 0x34, 0xfd, 0xc9, 0x92, 0xb7, 0x29, 0xac, 0x45, 0x6c, 0x7d, 0xfc, 0x4f, 0x0c, 0xf6, 0xa2, - 0xac, 0x2c, 0x94, 0xee, 0xa8, 0x21, 0xca, 0xaf, 0xe4, 0x8d, 0x55, 0x03, 0x01, 0x64, 0x4f, 0x55, - 0x9c, 0x19, 0xd9, 0x8f, 0x85, 0x01, 0xc8, 0x92, 0x2a, 0x48, 0x05, 0xae, 0x30, 0x80, 0xf6, 0x8f, - 0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b, - 0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a, - 0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1, - 0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6, - 0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00, - 0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00, + // 671 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xbb, 0x6e, 0xd4, 0x40, + 0x14, 0x65, 0xd6, 0xde, 0xd7, 0xdd, 0x4d, 0x14, 0x8d, 0x20, 0x58, 0x08, 0xad, 0x2c, 0x8b, 0xc2, + 0xd5, 0x46, 0x5a, 0x7a, 0x10, 0x9b, 0x87, 0x64, 0x45, 0x44, 0x70, 0x37, 0x84, 0xda, 0x49, 0x46, + 0xc1, 0x92, 0x5f, 0xd8, 0x63, 0x91, 0xfd, 0x0e, 0x1a, 0x6a, 0x1a, 0xf8, 0x01, 0x3a, 0x3e, 0x80, + 0x92, 0x4f, 0x40, 0xe1, 0x47, 0xd0, 0x9d, 0xf1, 0xd8, 0x5e, 0x22, 0x01, 0x05, 0xdd, 0x9c, 0x73, + 0x66, 0xae, 0xef, 0xe3, 0x5c, 0xc3, 0x34, 0xaf, 0xce, 0xe3, 0xe8, 0x62, 0x9e, 0x17, 0x99, 0xcc, + 0xf8, 0x28, 0x4a, 0xa5, 0x28, 0xd2, 0x30, 0xf6, 0xce, 0x60, 0xb0, 0x8c, 0x64, 0x12, 0xe6, 0x9c, + 0x83, 0xbd, 0x8c, 0x64, 0xe9, 0x30, 0xd7, 0xf2, 0x6d, 0x54, 0x67, 0xfe, 0x08, 0xfa, 0xcf, 0xa4, + 0x2c, 0x4a, 0xa7, 0xe7, 0x5a, 0xfe, 0x64, 0xb1, 0x3d, 0x37, 0xef, 0xe6, 0x44, 0xa3, 0x16, 0xe9, + 0xe5, 0xb1, 0x58, 0x97, 0x8e, 0xe5, 0x5a, 0xfe, 0x18, 0xd5, 0xd9, 0x7b, 0x02, 0xf6, 0x8b, 0x30, + 0x2a, 0xf8, 0x36, 0xf4, 0x82, 0x03, 0x87, 0xb9, 0xcc, 0xb7, 0xb1, 0x17, 0x1c, 0xf0, 0xbb, 0xd0, + 0xdf, 0xcf, 0xaa, 0x54, 0x3a, 0x3d, 0x45, 0x69, 0xc0, 0x77, 0xc0, 0x3a, 0x16, 0x6b, 0xc7, 0x72, + 0x99, 0x3f, 0x46, 0x3a, 0x7a, 0x0b, 0x18, 0xad, 0xaa, 0xa4, 0x51, 0x57, 0x55, 0xa2, 0x82, 0x58, + 0x48, 0xc7, 0xcd, 0x28, 0x56, 0x1d, 0xc5, 0x7b, 0x05, 0xd6, 0x32, 0x92, 0x24, 0x62, 0xf6, 0xae, + 0xf9, 0xaa, 0x06, 0xfc, 0x01, 0x8c, 0xf6, 0xb3, 0xb8, 0x4a, 0xd2, 0xe0, 0xa0, 0xfe, 0x76, 0x83, + 0xf9, 0x43, 0x18, 0x9f, 0x46, 0x89, 0x28, 0x65, 0x98, 0xe4, 0x2a, 0x09, 0x0b, 0x5b, 0xc2, 0x7b, + 0x0d, 0x5b, 0xfa, 0x26, 0x55, 0xbb, 0x12, 0xf2, 0x56, 0x4d, 0xff, 0xd6, 0xa5, 0xdb, 0x35, 0x7e, + 0x66, 0x60, 0x93, 0x66, 0x24, 0xd6, 0x48, 0xd4, 0xd2, 0xd3, 0x75, 0x2e, 0xea, 0x4c, 0xd5, 0x99, + 0xbb, 0x30, 0x59, 0xc9, 0x22, 0x4a, 0xaf, 0xce, 0xc2, 0xb8, 0x12, 0x75, 0xa0, 0x2e, 0x45, 0x35, + 0x06, 0xa9, 0xd4, 0xb2, 0xad, 0xca, 0x68, 0x30, 0xd5, 0xb8, 0xcc, 0xb2, 0x58, 0x8b, 0x7d, 0x97, + 0xf9, 0x23, 0x6c, 0x09, 0x3e, 0x03, 0x38, 0x8a, 0xb3, 0xb0, 0x7e, 0x3b, 0x70, 0x99, 0xcf, 0xb0, + 0xc3, 0x78, 0x7b, 0x30, 0xa4, 0x4c, 0x9f, 0x87, 0x79, 0x5b, 0x2d, 0xfb, 0x43, 0xb5, 0xde, 0x57, + 0x06, 0xd3, 0x97, 0x95, 0x28, 0xd6, 0x28, 0xde, 0x56, 0xa2, 0x54, 0x53, 0x51, 0xb8, 0xae, 0x52, + 0x03, 0xbe, 0x0b, 0x83, 0x55, 0x1c, 0x5d, 0x08, 0xdd, 0x3b, 0x1b, 0x6b, 0x44, 0xb5, 0xb6, 0x3d, + 0x2f, 0x55, 0xad, 0x23, 0xec, 0x52, 0xf4, 0x12, 0x45, 0x92, 0x49, 0x53, 0x4c, 0x8d, 0xb8, 0x07, + 0xd3, 0xc3, 0xeb, 0x8b, 0xb8, 0xba, 0x14, 0xfa, 0xe9, 0x40, 0xa9, 0x1b, 0x1c, 0x45, 0xaf, 0xb1, + 0x72, 0xfc, 0x50, 0x47, 0xef, 0x50, 0xde, 0x7b, 0x06, 0x5b, 0x75, 0xfa, 0x65, 0x9e, 0xa5, 0xa5, + 0xa0, 0x19, 0x1d, 0x16, 0x85, 0x99, 0xd1, 0x61, 0x51, 0xf0, 0x3d, 0x18, 0xa2, 0x28, 0xab, 0x58, + 0x9a, 0xc1, 0xdf, 0x6b, 0x5b, 0x61, 0xde, 0x56, 0xb1, 0x44, 0x73, 0x8b, 0x3f, 0x85, 0xed, 0x0d, + 0x23, 0xe9, 0x8d, 0x99, 0x2c, 0xee, 0xb7, 0xef, 0x36, 0x74, 0xfc, 0xed, 0xba, 0xf7, 0x85, 0xc1, + 0xa4, 0x13, 0x99, 0xfb, 0x66, 0x79, 0x55, 0x5a, 0x93, 0xc5, 0x4e, 0x1b, 0x48, 0xf3, 0x68, 0x96, + 0x7b, 0x0a, 0xec, 0xa4, 0x36, 0x13, 0x3b, 0xa1, 0x11, 0xd2, 0x72, 0x9a, 0xef, 0x77, 0x46, 0x48, + 0x34, 0x6a, 0x91, 0x3b, 0x30, 0xdc, 0x7f, 0x13, 0xa6, 0x57, 0xe2, 0x52, 0x99, 0x69, 0x84, 0x06, + 0xf2, 0x79, 0xbb, 0x9c, 0xaa, 0xfb, 0x93, 0x05, 0x6f, 0x43, 0x18, 0x05, 0x9b, 0x3b, 0xde, 0x27, + 0x06, 0x5b, 0x41, 0x92, 0x67, 0x85, 0xec, 0xb8, 0x21, 0x48, 0x2f, 0xc5, 0xb5, 0x71, 0x83, 0x02, + 0xc4, 0x1e, 0x15, 0x61, 0xa2, 0x6d, 0x3f, 0x46, 0x0d, 0x88, 0x55, 0xae, 0x50, 0x2e, 0xb0, 0x51, + 0x03, 0x35, 0x7f, 0x5a, 0xec, 0xd2, 0xb1, 0xb5, 0x73, 0x34, 0x22, 0x9f, 0x9b, 0xbd, 0x2e, 0x9d, + 0xbe, 0x92, 0x5a, 0x82, 0x7c, 0xde, 0x2c, 0x36, 0x79, 0xc3, 0xf2, 0x2d, 0xec, 0x30, 0xde, 0x47, + 0x06, 0x5c, 0x67, 0xaa, 0x7c, 0xff, 0xff, 0xd2, 0xa5, 0xbb, 0x91, 0x88, 0x75, 0x2b, 0xe9, 0x2e, + 0x81, 0xbf, 0x24, 0xbb, 0x0b, 0x03, 0x95, 0x85, 0x49, 0xb4, 0x46, 0xcb, 0x9d, 0x6f, 0x37, 0x33, + 0xf6, 0xfd, 0x66, 0xc6, 0x7e, 0xdc, 0xcc, 0xd8, 0x87, 0x9f, 0xb3, 0x3b, 0xe7, 0x03, 0xf5, 0x5b, + 0x7f, 0xfc, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x0f, 0xf2, 0x1f, 0x86, 0xe6, 0x05, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 47b04d8e7..025b50baa 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -4,11 +4,13 @@ package internal; message Bitmap { repeated uint64 Bits = 1; + repeated string Keys = 3; repeated Attr Attrs = 2; } message Pair { - uint64 Key = 1; + uint64 ID = 1; + string Key = 3; uint64 Count = 2; } @@ -25,6 +27,7 @@ message Bit { message ColumnAttrSet { uint64 ID = 1; + string Key = 3; repeated Attr Attrs = 2; } diff --git a/roaring/internal_test.go b/roaring/internal_test.go deleted file mode 100644 index 107851489..000000000 --- a/roaring/internal_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package roaring - -import ( - "reflect" - "testing" -) - -// Ensure iterator returns values from a bitmap. -func TestBitmapIterator(t *testing.T) { - for i, tt := range []struct { - bitmap []uint64 - values []uint16 - }{ - // Empty - { - bitmap: []uint64{6}, // 0110 - values: []uint16{1, 2}, - }, - - // Single uint64 bitmap - { - bitmap: []uint64{6}, // 0110 - values: []uint16{1, 2}, - }, - - // Multi uint64 bitmap - { - bitmap: []uint64{1 << 63, 1, 0, 1, 3 << 62}, - values: []uint16{63, 64, 192, 318, 319}, - }, - } { - itr := newBitmapIterator(tt.bitmap) - - var a []uint16 - for v, eof := itr.next(); !eof; v, eof = itr.next() { - a = append(a, v) - } - - if !reflect.DeepEqual(a, tt.values) { - t.Errorf("%d. unexpected values: exp=%+v, got=%+v", i, a, tt.values) - } - } -} diff --git a/roaring/roaring.go b/roaring/roaring.go index ad7b33bc4..055c81e36 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// package roaring implements roaring bitmaps with support for incremental changes. +// Package roaring implements roaring bitmaps with support for incremental changes. package roaring import ( @@ -50,11 +50,15 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - // manual allocation size tuned to our average client data - manualAlloc = 524288 - ContainerArray = byte(1) + //ContainerArray indicates a container of bit position values + ContainerArray = byte(1) + + //ContainerBitmap indicates a container of bits packed in a uint64 array block ContainerBitmap = byte(2) - ContainerRun = byte(3) + + //ContainerRun indicates a container of run encoded bits + ContainerRun = byte(3) + maxContainerVal = 0xffff ) @@ -128,7 +132,7 @@ func (b *Bitmap) add(v uint64) bool { // If index is negative then there's not an exact match // and a container needs to be added. if i < 0 { - b.insertAt(hb, newContainer(), int(-i-1)) + b.insertAt(hb, newContainer(), -i-1) i = -i - 1 } return b.containers[i].add(lowbits(v)) @@ -181,7 +185,7 @@ func (b *Bitmap) Max() uint64 { hb := b.keys[len(b.keys)-1] lb := b.containers[len(b.containers)-1].max() - return uint64(hb)<<16 | uint64(lb) + return hb<<16 | uint64(lb) } // Count returns the number of bits set in the bitmap. @@ -216,7 +220,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { } else { // Count first partial container and advance i so we don't recount it n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1)) - i += 1 + i++ } // Count last container. @@ -364,10 +368,8 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { if ni == 0 && nj == 0 { // eof(i,j) break } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j - key, container = kj[0], cj[0].clone() kj, cj = kj[1:], cj[1:] } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j - key, container = ki[0], ci[0].clone() ki, ci = ki[1:], ci[1:] } else { // i == j key, container = ki[0], intersect(ci[0], cj[0]) @@ -518,19 +520,39 @@ func (b *Bitmap) Optimize() { } } -//hoping this in-lines -func WriteUint16(w io.Writer, b []byte, v uint16) (int, error) { - binary.LittleEndian.PutUint16(b, v) - return w.Write(b) -} -func WriteUint32(w io.Writer, b []byte, v uint32) (int, error) { - binary.LittleEndian.PutUint32(b, v) - return w.Write(b) +type errWriter struct { + w io.Writer + err error + n int } -func WriteUint64(w io.Writer, b []byte, v uint64) (int, error) { +func (ew *errWriter) WriteUint16(b []byte, v uint16) { + if ew.err != nil { + return + } + var n int + binary.LittleEndian.PutUint16(b, v) + n, ew.err = ew.w.Write(b) + ew.n += n +} +func (ew *errWriter) WriteUint32(b []byte, v uint32) { + if ew.err != nil { + return + } + var n int + binary.LittleEndian.PutUint32(b, v) + n, ew.err = ew.w.Write(b) + ew.n += n +} + +func (ew *errWriter) WriteUint64(b []byte, v uint64) { + if ew.err != nil { + return + } + var n int binary.LittleEndian.PutUint64(b, v) - return w.Write(b) + n, ew.err = ew.w.Write(b) + ew.n += n } // WriteTo writes b to w. @@ -546,11 +568,15 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { byte8 := make([]byte, 8) // Build header before writing individual container blocks. - // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) + // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(containerType)+sizeof(cardinality) + sizeof(file offset) // Cookie header section. + ew := &errWriter{ + w: w, + n: 0, + } - WriteUint32(w, byte4, cookie) - WriteUint32(w, byte4, uint32(containerCount)) + ew.WriteUint32(byte4, cookie) + ew.WriteUint32(byte4, uint32(containerCount)) // Descriptive header section: encode keys and cardinality. // Key and cardinality are stored interleaved here, 12 bytes per container. @@ -562,9 +588,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //count := c.count() //assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n) if c.n > 0 { - WriteUint64(w, byte8, uint64(key)) - WriteUint16(w, byte2, uint16(c.container_type)) - WriteUint16(w, byte2, uint16(c.n-1)) + ew.WriteUint64(byte8, key) + ew.WriteUint16(byte2, uint16(c.containerType)) + ew.WriteUint16(byte2, uint16(c.n-1)) } } @@ -574,15 +600,15 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { for _, c := range b.containers { if c.n > 0 { - WriteUint32(w, byte4, uint32(offset)) + ew.WriteUint32(byte4, offset) offset += uint32(c.size()) } } + if ew.err != nil { + return int64(ew.n), ew.err + } n = int64(headerSize + (containerCount * (8 + 2 + 2 + 4))) - if err != nil { - return n, err - } // Container storage section: write each container block. for _, c := range b.containers { @@ -638,14 +664,14 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { if i >= len(b.keys) { b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8])) b.containers = append(b.containers, &container{ - container_type: byte(binary.LittleEndian.Uint16(buf[8:10])), - n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, - mapped: true, + containerType: byte(binary.LittleEndian.Uint16(buf[8:10])), + n: int(binary.LittleEndian.Uint16(buf[10:12])) + 1, + mapped: true, }) } else { b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) c := b.containers[i] - c.container_type = byte(binary.LittleEndian.Uint16(buf[8:10])) + c.containerType = byte(binary.LittleEndian.Uint16(buf[8:10])) c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1 c.mapped = true @@ -663,7 +689,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Map byte slice directly to the container data. c := b.containers[i] - switch c.container_type { + switch c.containerType { case ContainerRun: c.array = nil c.bitmap = nil @@ -692,18 +718,18 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { } // Unmarshal the op and apply it. - var op op - if err := op.UnmarshalBinary(buf); err != nil { + var opr op + if err := opr.UnmarshalBinary(buf); err != nil { // FIXME(benbjohnson): return error with position so file can be trimmed. return err } - op.apply(b) + opr.apply(b) // Increase the op count. b.opN++ // Move the buffer forward. - buf = buf[op.size():] + buf = buf[opr.size():] } return nil @@ -769,7 +795,7 @@ func (b *Bitmap) Check() error { return a } -//Perform a logical negate of the bits in the range [start,end]. +// Flip performs a logical negate of the bits in the range [start,end]. func (b *Bitmap) Flip(start, end uint64) *Bitmap { result := NewBitmap() itr := b.Iterator() @@ -810,7 +836,7 @@ type Iterator struct { } // eof returns true if the iterator is at the end of the bitmap. -func (itr *Iterator) eof() bool { return int(itr.i) >= len(itr.bitmap.containers) } +func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) } // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { @@ -825,7 +851,7 @@ func (itr *Iterator) Seek(seek uint64) { // Move to the correct value index inside the container. lb := lowbits(seek) - if int(itr.i) >= len(itr.bitmap.containers) { + if itr.i >= len(itr.bitmap.containers) { panic(fmt.Sprintf("data Corruption %d %d %d", itr.i, len(itr.bitmap.containers), seek)) } c := itr.bitmap.containers[itr.i] @@ -835,7 +861,7 @@ func (itr *Iterator) Seek(seek uint64) { if itr.j < 0 { itr.j = -itr.j - 1 } - if int(itr.j) < len(c.array) { + if itr.j < len(c.array) { itr.j-- return } @@ -878,7 +904,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { c := itr.bitmap.containers[itr.i] if c.isArray() { - if itr.j >= int(c.n-1) { + if itr.j >= c.n-1 { // Reached end of array, move to the next container. itr.i, itr.j = itr.i+1, -1 continue @@ -927,7 +953,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { itr.j++ // Find first non-zero bit in current bitmap, if possible. - hb := int(itr.j >> 6) + hb := itr.j >> 6 if hb >= len(c.bitmap) { itr.i, itr.j = itr.i+1, -1 @@ -935,14 +961,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } lb := c.bitmap[hb] >> (uint(itr.j) % 64) if lb != 0 { - itr.j = int(itr.j) + trailingZeroN(lb) + itr.j = itr.j + trailingZeroN(lb) return itr.peek(), false } // Otherwise iterate through remaining bitmaps to find next bit. for hb++; hb < len(c.bitmap); hb++ { if c.bitmap[hb] != 0 { - itr.j = int(hb<<6) + trailingZeroN(c.bitmap[hb]) + itr.j = hb<<6 + trailingZeroN(c.bitmap[hb]) return itr.peek(), false } } @@ -957,18 +983,18 @@ func (itr *Iterator) peek() uint64 { key := itr.bitmap.keys[itr.i] c := itr.bitmap.containers[itr.i] if c.isArray() { - return uint64(key)<<16 | uint64(c.array[itr.j]) + return key<<16 | uint64(c.array[itr.j]) } if c.isRun() { - return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint16(itr.k)) + return key<<16 | uint64(c.runs[itr.j].start+uint16(itr.k)) } - return uint64(key)<<16 | uint64(itr.j) + return key<<16 | uint64(itr.j) } -// The maximum size of array containers. +// ArrayMaxSize represents the maximum size of array containers. const ArrayMaxSize = 4096 -// The maximum size of run length encoded containers. +// RunMaxSize represents the maximum size of run length encoded containers. const RunMaxSize = 2048 // container represents a container for uint32 integers. @@ -978,12 +1004,12 @@ const RunMaxSize = 2048 // an array or RLE container is used, depending on the contents. For containers // with more than 4,096 values, the values are encoded into bitmaps. type container struct { - container_type byte // array, bitmap, or run - n int // number of integers in container - array []uint16 // used for array containers - bitmap []uint64 // used for bitmap containers - runs []interval16 // used for RLE containers - mapped bool // mapped directly to a byte slice when true + mapped bool // mapped directly to a byte slice when true + containerType byte // array, bitmap, or run + n int // number of integers in container + array []uint16 // used for array containers + bitmap []uint64 // used for bitmap containers + runs []interval16 // used for RLE containers } type interval16 struct { @@ -998,22 +1024,22 @@ func (iv interval16) runlen() int { // newContainer returns a new instance of container. func newContainer() *container { - return &container{container_type: ContainerArray} + return &container{containerType: ContainerArray} } // isArray returns true if the container is an array container. func (c *container) isArray() bool { - return c.container_type == ContainerArray + return c.containerType == ContainerArray } // isBitmap returns true if the container is a bitmap container. func (c *container) isBitmap() bool { - return c.container_type == ContainerBitmap + return c.containerType == ContainerBitmap } // isRun returns true if the container is a run-length-encoded container. func (c *container) isRun() bool { - return c.container_type == ContainerRun + return c.containerType == ContainerRun } // unmap creates copies of the containers data in the heap. @@ -1025,7 +1051,7 @@ func (c *container) unmap() { return } - switch c.container_type { + switch c.containerType { case ContainerArray: tmp := make([]uint16, len(c.array)) copy(tmp, c.array) @@ -1091,7 +1117,7 @@ func (c *container) bitmapCountRange(start, end int) int { } // Count partial ending word. - if int(j) < len(c.bitmap) { + if j < len(c.bitmap) { off := 64 - (uint(end) % 64) n += popcount(c.bitmap[j] << off) } @@ -1111,7 +1137,7 @@ func (c *container) runCountRange(start, end int) (n int) { } // iv is superset of range if int(iv.start) < start && int(iv.last) > end { - return int(end - start) + return end - start } // iv is subset of range if int(iv.start) >= start && int(iv.last) < end { @@ -1190,20 +1216,23 @@ func (c *container) runAdd(v uint16) bool { c.runs = []interval16{{start: v, last: v}} return true } - i := 0 - var iv interval16 - for i, iv = range c.runs { - if iv.last >= v { - break - } + + i := sort.Search(len(c.runs), + func(i int) bool { return c.runs[i].last >= v }) + + if i == len(c.runs) { + i-- } + + iv := c.runs[i] if v >= iv.start && iv.last >= v { return false } + c.unmap() if iv.last < v { if iv.last == v-1 { - c.runs[i].last += 1 + c.runs[i].last++ } else { c.runs = append(c.runs, interval16{start: v, last: v}) } @@ -1215,10 +1244,10 @@ func (c *container) runAdd(v uint16) bool { return true } // just before an interval - c.runs[i].start -= 1 + c.runs[i].start-- } else if i > 0 && v-1 == c.runs[i-1].last { // just after an interval - c.runs[i-1].last += 1 + c.runs[i-1].last++ } else { // alone newIv := interval16{start: v, last: v} @@ -1252,7 +1281,7 @@ func (c *container) arrayCountRuns() (r int) { prev := -2 for _, v := range c.array { if prev+1 != int(v) { - r += 1 + r++ } prev = int(v) } @@ -1391,9 +1420,9 @@ func (c *container) runRemove(v uint16) bool { if v == c.runs[i].last && v == c.runs[i].start { c.runs = append(c.runs[:i], c.runs[i+1:]...) } else if v == c.runs[i].last { - c.runs[i].last -= 1 + c.runs[i].last-- } else if v == c.runs[i].start { - c.runs[i].start += 1 + c.runs[i].start++ } else if v > c.runs[i].start { last := c.runs[i].last c.runs[i].last = v - 1 @@ -1430,10 +1459,13 @@ func (c *container) bitmapMax() uint16 { } // Find the highest set bit. - for j := uint16(63); j >= 0; j-- { + for j := uint16(63); ; j-- { if v&(1<= na && j >= nb { @@ -2148,7 +2180,7 @@ func unionArrayRun(a, b *container) *container { if b.n == maxContainerVal { return b.clone() } - output := &container{container_type: ContainerRun} + output := &container{containerType: ContainerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 @@ -2185,18 +2217,18 @@ func (c *container) runAppendInterval(v interval16) int { if len(c.runs) == 0 { c.runs = append(c.runs, v) return int(v.last-v.start) + 1 - } else { - last := c.runs[len(c.runs)-1] - if last.last == maxContainerVal { //protect against overflow - return 0 - } - if last.last+1 >= v.start && v.last > last.last { - c.runs[len(c.runs)-1].last = v.last - return int(v.last - last.last) - } else if last.last+1 < v.start { - c.runs = append(c.runs, v) - return int(v.last-v.start) + 1 - } + } + + last := c.runs[len(c.runs)-1] + if last.last == maxContainerVal { //protect against overflow + return 0 + } + if last.last+1 >= v.start && v.last > last.last { + c.runs[len(c.runs)-1].last = v.last + return int(v.last - last.last) + } else if last.last+1 < v.start { + c.runs = append(c.runs, v) + return int(v.last-v.start) + 1 } return 0 } @@ -2210,8 +2242,8 @@ func unionRunRun(a, b *container) *container { } na, nb := len(a.runs), len(b.runs) output := &container{ - runs: make([]interval16, 0, na+nb), - container_type: ContainerRun, + runs: make([]interval16, 0, na+nb), + containerType: ContainerRun, } var va, vb interval16 for i, j := 0, 0; i < na || j < nb; { @@ -2330,8 +2362,8 @@ func unionArrayBitmap(a, b *container) *container { func unionBitmapBitmap(a, b *container) *container { output := &container{ - bitmap: make([]uint64, bitmapN), - container_type: ContainerBitmap, + bitmap: make([]uint64, bitmapN), + containerType: ContainerBitmap, } for i := 0; i < bitmapN; i++ { @@ -2373,7 +2405,7 @@ func difference(a, b *container) *container { // differenceArrayArray computes the difference bween two arrays. func differenceArrayArray(a, b *container) *container { - output := &container{container_type: ContainerArray} + output := &container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na; { va := a.array[i] @@ -2404,14 +2436,14 @@ func differenceArrayRun(a, b *container) *container { return a.clone() } - output := &container{array: make([]uint16, 0, a.n), container_type: ContainerArray} + output := &container{array: make([]uint16, 0, a.n), containerType: ContainerArray} // cardinality upper bound: card(A) i := 0 // array index j := 0 // run index // handle overlap - for i < int(a.n) { + for i < a.n { // keep all array elements before beginning of runs if a.array[i] < b.runs[j].start { @@ -2440,7 +2472,7 @@ func differenceArrayRun(a, b *container) *container { output.array = append(output.array, a.array[i:]...) // TODO: consider handling container.n mutations in one place // like we do with container.add(). - output.n += int(len(a.array[i:])) + output.n += len(a.array[i:]) } return output } @@ -2464,7 +2496,7 @@ func differenceRunArray(a, b *container) *container { if a.n == 0 || b.n == 0 { return a.clone() } - output := &container{runs: make([]interval16, 0, len(a.runs)), container_type: ContainerRun} + output := &container{runs: make([]interval16, 0, len(a.runs)), containerType: ContainerRun} bidx := 0 vb := b.array[bidx] @@ -2512,7 +2544,7 @@ func differenceRunBitmap(a, b *container) *container { if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { return b.flipBitmap() } - output := &container{container_type: ContainerRun} + output := &container{containerType: ContainerRun} output.n = a.n if len(a.runs) == 0 { return output @@ -2555,61 +2587,6 @@ func differenceRunBitmap(a, b *container) *container { } } - if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 { - output.runToArray() - } else if len(output.runs) > RunMaxSize { - output.runToBitmap() - } - return output -} - -func differenceRunIterator(a *container, itr containerIterator) *container { - - output := &container{runs: make([]interval16, 0, a.n), container_type: ContainerRun} - - vb, eof := itr.next() - j := 0 - vr := a.runs[j] - working := !eof - for working { - switch { - case vb < vr.start: //before - case vb > vr.last: //after - if vr.start <= vr.last { - output.n += output.runAppendInterval(vr) - } - j++ - if j < len(a.runs) { - vr = a.runs[j] - } else { - working = false - } - case vb == vr.start: //begining of run - vr.start++ - case vb == a.runs[j].last: //end of run - vr.last-- - if vr.last >= vr.start { - output.n += output.runAppendInterval(vr) - } - j++ - if j < len(a.runs) { - vr = a.runs[j] - } else { - working = false - } - case vb > vr.start: //inside run - output.n += output.runAppendInterval(interval16{start: vr.start, last: vb - 1}) - vr.start = vb + 1 - - } - vb, eof = itr.next() - if eof { - working = false - } - } - if vr.start <= vr.last { - output.n += output.runAppendInterval(vr) - } if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() } else if len(output.runs) > RunMaxSize { @@ -2633,7 +2610,7 @@ func differenceRunRun(a, b *container) *container { alen := len(a.runs) blen := len(b.runs) - output := &container{runs: make([]interval16, 0, alen+blen), container_type: ContainerRun} // TODO allocate max then truncate? or something else + output := &container{runs: make([]interval16, 0, alen+blen), containerType: ContainerRun} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -2641,7 +2618,7 @@ func differenceRunRun(a, b *container) *container { switch { case alast < bstart: // current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run - output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)}) + output.runs = append(output.runs, interval16{start: astart, last: alast}) apos++ if apos < alen { astart = a.runs[apos].start @@ -2657,7 +2634,7 @@ func differenceRunRun(a, b *container) *container { default: // overlap if astart < bstart { - output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(bstart - 1)}) + output.runs = append(output.runs, interval16{start: astart, last: bstart - 1}) } if alast > blast { astart = blast + 1 @@ -2671,7 +2648,7 @@ func differenceRunRun(a, b *container) *container { } } if apos < alen { - output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)}) + output.runs = append(output.runs, interval16{start: astart, last: alast}) apos++ if apos < alen { output.runs = append(output.runs, a.runs[apos:]...) @@ -2683,7 +2660,7 @@ func differenceRunRun(a, b *container) *container { } func differenceArrayBitmap(a, b *container) *container { - output := &container{container_type: ContainerArray} + output := &container{containerType: ContainerArray} for _, va := range a.array { bmidx := va / 64 bidx := va % 64 @@ -2714,7 +2691,7 @@ func differenceBitmapArray(a, b *container) *container { } func differenceBitmapBitmap(a, b *container) *container { - output := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + output := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} for i := range a.bitmap { v := a.bitmap[i] & (^b.bitmap[i]) @@ -2757,7 +2734,7 @@ func xor(a, b *container) *container { } func xorArrayArray(a, b *container) *container { - output := &container{container_type: ContainerArray} + output := &container{containerType: ContainerArray} na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na || j < nb; { if i < na && j >= nb { @@ -2804,8 +2781,8 @@ func xorArrayBitmap(a, b *container) *container { func xorBitmapBitmap(a, b *container) *container { output := &container{ - bitmap: make([]uint64, bitmapN), - container_type: ContainerBitmap, + bitmap: make([]uint64, bitmapN), + containerType: ContainerBitmap, } for i := 0; i < bitmapN; i++ { v := a.bitmap[i] ^ b.bitmap[i] @@ -2843,7 +2820,6 @@ func (op *op) apply(b *Bitmap) bool { default: panic(fmt.Sprintf("invalid op type: %d", op.typ)) } - return false } // WriteTo writes op to the w. @@ -2887,7 +2863,7 @@ func (op *op) UnmarshalBinary(data []byte) error { // size returns the encoded size of the op, in bytes. func (*op) size() int { return 1 + 8 + 4 } -func highbits(v uint64) uint64 { return uint64(v >> 16) } +func highbits(v uint64) uint64 { return v >> 16 } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } // search32 returns the index of value in a. If value is not found, it works the @@ -2990,7 +2966,7 @@ func trailingZeroN(v uint64) int { if y := v << 2; y != 0 { n, v = n-2, y } - return int(n - int64(uint64(v<<1)>>63)) + return int(n - int64(v<<1>>63)) } // bit population count, taken from @@ -3005,110 +2981,6 @@ func popcount(x uint64) (n uint64) { return x >> 56 } -// Returns eof as true if there are no values left in the iterator. -type containerIterator interface { - next() (uint16, bool) -} - -// arrayIterator represents an iterator over container array values. -type arrayIterator struct { - array []uint16 - i int -} - -func newArrayIterator(array []uint16) *arrayIterator { - return &arrayIterator{ - array: array, - i: -1, - } -} - -// next returns the next value in the array. -func (itr *arrayIterator) next() (v uint16, eof bool) { - - itr.i++ - if itr.i >= len(itr.array) { - return 0, true - } - return itr.array[itr.i], false -} - -// bitmapIterator represents an iterator over container bitmap values. -type bitmapIterator struct { - bitmap []uint64 - i int -} - -func newBitmapIterator(bitmap []uint64) *bitmapIterator { - return &bitmapIterator{ - bitmap: bitmap, - i: -1, - } -} - -// next returns the next value in the bitmap. -// Returns eof as true if there are no values left in the iterator. -func (itr *bitmapIterator) next() (v uint16, eof bool) { - if itr.i+1 >= int(len(itr.bitmap)*64) { - return 0, true - } - itr.i++ - - // Find first non-zero bit in current bitmap, if possible. - hb := int(itr.i >> 6) - lb := itr.bitmap[hb] >> (uint(itr.i) % 64) - if lb != 0 { - itr.i = int(itr.i) + trailingZeroN(lb) - return uint16(itr.i), false - } - - // Otherwise iterate through remaining bitmaps to find next bit. - for hb++; hb < len(itr.bitmap); hb++ { - if itr.bitmap[hb] != 0 { - itr.i = int(hb<<6) + trailingZeroN(itr.bitmap[hb]) - return uint16(itr.i), false - } - } - - return 0, true -} - -// bufBitmapIterator wraps an iterator to provide the ability to unread values. -type bufBitmapIterator struct { - buf struct { - v uint16 - eof bool - full bool - } - itr *bitmapIterator -} - -// newBufBitmapIterator returns a buffered iterator that wraps a bitmapIterator. -func newBufBitmapIterator(itr *bitmapIterator) *bufBitmapIterator { - return &bufBitmapIterator{itr: itr} -} - -// next returns the next pair in the bitmap. -// If a value has been buffered then it is returned and the buffer is cleared. -func (itr *bufBitmapIterator) next() (v uint16, eof bool) { - if itr.buf.full { - itr.buf.full = false - return itr.buf.v, itr.buf.eof - } - - // Read value onto buffer in case of unread. - itr.buf.v, itr.buf.eof = itr.itr.next() - return itr.buf.v, itr.buf.eof -} - -// unread pushes previous pair on to the buffer. Panics if the buffer is already full. -func (itr *bufBitmapIterator) unread() { - if itr.buf.full { - panic("roaring.bufBitmapIterator: buffer full") - } - itr.buf.full = true -} - // ErrorList represents a list of errors. type ErrorList []error @@ -3144,29 +3016,22 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) { } } -// assert panics with a formatted message if condition is false. -func assert(condition bool, format string, a ...interface{}) { - if !condition { - panic(fmt.Sprintf(format, a...)) - } -} - // xorArrayRun computes the exclusive or of an array and a run container. func xorArrayRun(a, b *container) *container { - output := &container{container_type: ContainerRun} + output := &container{containerType: ContainerRun} na, nb := len(a.array), len(b.runs) var vb interval16 var va uint16 - last_i, last_j := -1, -1 + lastI, lastJ := -1, -1 for i, j := 0, 0; i < na || j < nb; { - if i < na && i != last_i { + if i < na && i != lastI { va = a.array[i] } - if j < nb && j != last_j { + if j < nb && j != lastJ { vb = b.runs[j] } - last_i = i - last_j = j + lastI = i + lastJ = j if i < na && (j >= nb || va < vb.start) { //before output.n += output.runAppendInterval(interval16{start: va, last: va}) @@ -3216,91 +3081,91 @@ func xorArrayRun(a, b *container) *container { } // xorCompare computes first exclusive run between two runs. -func xorCompare(x *xorstm) (r1 interval16, has_data bool) { - has_data = false - if !x.va_valid || !x.vb_valid { - if x.vb_valid { - x.vb_valid = false +func xorCompare(x *xorstm) (r1 interval16, hasData bool) { + hasData = false + if !x.vaValid || !x.vbValid { + if x.vbValid { + x.vbValid = false r1 = x.vb - has_data = true + hasData = true return } - if x.va_valid { - x.va_valid = false + if x.vaValid { + x.vaValid = false r1 = x.va - has_data = true + hasData = true return } return } if x.va.last < x.vb.start { //va before - x.va_valid = false + x.vaValid = false r1 = x.va - has_data = true + hasData = true } else if x.vb.last < x.va.start { //vb before - x.vb_valid = false + x.vbValid = false r1 = x.vb - has_data = true + hasData = true } else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal - x.va_valid = false - x.vb_valid = false + x.vaValid = false + x.vbValid = false } else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { //vb inside - x.vb_valid = false + x.vbValid = false if x.va.start != x.vb.start { r1 = interval16{start: x.va.start, last: x.vb.start - 1} - has_data = true + hasData = true } if x.vb.last == maxContainerVal { // Check for overflow - x.va_valid = false + x.vaValid = false } else { x.va.start = x.vb.last + 1 if x.va.start > x.va.last { - x.va_valid = false + x.vaValid = false } } } else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside - x.va_valid = false + x.vaValid = false if x.vb.start != x.va.start { r1 = interval16{start: x.vb.start, last: x.va.start - 1} - has_data = true + hasData = true } if x.va.last == maxContainerVal { //check for overflow - x.vb_valid = false + x.vbValid = false } else { x.vb.start = x.va.last + 1 if x.vb.start > x.vb.last { - x.vb_valid = false + x.vbValid = false } } } else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap - x.va_valid = false + x.vaValid = false r1 = interval16{start: x.va.start, last: x.vb.start - 1} - has_data = true + hasData = true if x.va.last == maxContainerVal { // check for overflow - x.vb_valid = false + x.vbValid = false } else { x.vb.start = x.va.last + 1 if x.vb.start > x.vb.last { - x.vb_valid = false + x.vbValid = false } } } else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap - x.vb_valid = false + x.vbValid = false r1 = interval16{start: x.vb.start, last: x.va.start - 1} - has_data = true + hasData = true if x.vb.last == maxContainerVal { // check for overflow - x.va_valid = false + x.vaValid = false } else { x.va.start = x.vb.last + 1 if x.va.start > x.va.last { - x.va_valid = false + x.vaValid = false } } } @@ -3309,8 +3174,8 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) { //stm is state machine used to "xor" iterate over runs. type xorstm struct { - va_valid, vb_valid bool - va, vb interval16 + vaValid, vbValid bool + va, vb interval16 } // xorRunRun computes the exclusive or of two run containers. @@ -3324,36 +3189,36 @@ func xorRunRun(a, b *container) *container { } output := &container{} - last_i, last_j := -1, -1 + lastI, lastJ := -1, -1 state := &xorstm{} for i, j := 0, 0; i < na || j < nb; { - if i < na && last_i != i { + if i < na && lastI != i { state.va = a.runs[i] - state.va_valid = true + state.vaValid = true } - if j < nb && last_j != j { + if j < nb && lastJ != j { state.vb = b.runs[j] - state.vb_valid = true + state.vbValid = true } - last_i, last_j = i, j + lastI, lastJ = i, j r1, ok := xorCompare(state) if ok { output.n += output.runAppendInterval(r1) } - if !state.va_valid { + if !state.vaValid { i++ } - if !state.vb_valid { + if !state.vbValid { j++ } } - if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 { + if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() } else if len(output.runs) > RunMaxSize { output.runToBitmap() @@ -3368,7 +3233,7 @@ func xorBitmapRun(a, b *container) *container { output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } - if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 { + if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { output.runToArray() } else if len(output.runs) > RunMaxSize { output.runToBitmap() diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index f5c12aa83..4b2813908 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -27,11 +27,11 @@ func (iv interval16) String() string { } func (c *container) String() string { - return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.container_type) + return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.containerType) } func TestRunAppendInterval(t *testing.T) { - a := container{container_type: ContainerRun} + a := container{containerType: ContainerRun} tests := []struct { base []interval16 app interval16 @@ -80,7 +80,7 @@ func TestInterval16RunLen(t *testing.T) { } func TestContainerRunAdd(t *testing.T) { - c := container{runs: make([]interval16, 0), container_type: ContainerRun} + c := container{runs: make([]interval16, 0), containerType: ContainerRun} tests := []struct { op uint16 exp []interval16 @@ -111,7 +111,7 @@ func TestContainerRunAdd(t *testing.T) { } func TestContainerRunAdd2(t *testing.T) { - c := container{runs: make([]interval16, 0), container_type: ContainerRun} + c := container{runs: make([]interval16, 0), containerType: ContainerRun} ret := c.add(0) if !ret { t.Fatalf("result of adding new bit should be true: %v", c.runs) @@ -126,7 +126,7 @@ func TestContainerRunAdd2(t *testing.T) { } func TestRunCountRange(t *testing.T) { - c := container{runs: make([]interval16, 0), container_type: ContainerRun} + c := container{runs: make([]interval16, 0), containerType: ContainerRun} cnt := c.runCountRange(2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) @@ -179,7 +179,7 @@ func TestRunCountRange(t *testing.T) { } func TestRunContains(t *testing.T) { - c := container{runs: make([]interval16, 0), container_type: ContainerRun} + c := container{runs: make([]interval16, 0), containerType: ContainerRun} if c.runContains(5) { t.Fatalf("empty run container should not contain 5") } @@ -201,7 +201,7 @@ func TestRunContains(t *testing.T) { } func TestBitmapCountRange(t *testing.T) { - c := container{container_type: ContainerBitmap} + c := container{containerType: ContainerBitmap} tests := []struct { start int end int @@ -227,11 +227,11 @@ func TestBitmapCountRange(t *testing.T) { func TestIntersectionCountArrayBitmap3(t *testing.T) { a, b := &container{}, &container{} - a.container_type = ContainerBitmap + a.containerType = ContainerBitmap a.bitmap = getFullBitmap() a.n = maxContainerVal + 1 - b.container_type = ContainerBitmap + b.containerType = ContainerBitmap b.bitmap = getFullBitmap() b.n = maxContainerVal + 1 res := intersectBitmapBitmap(a, b) @@ -288,9 +288,9 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { for i, test := range tests { a.array = test.array - a.container_type = ContainerArray + a.containerType = ContainerArray b.bitmap = test.bitmap - b.container_type = ContainerBitmap + b.containerType = ContainerBitmap ret := intersectionCountArrayBitmap(a, b) if ret != test.exp { t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp) @@ -299,7 +299,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { } func TestRunRemove(t *testing.T) { - c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} + c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} tests := []struct { op uint16 exp []interval16 @@ -333,7 +333,7 @@ func TestRunRemove(t *testing.T) { } func TestRunMax(t *testing.T) { - c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} + c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun} max := c.max() if max != 16 { t.Fatalf("max for %v should be 16", c.runs) @@ -347,8 +347,8 @@ func TestRunMax(t *testing.T) { } func TestIntersectionCountArrayRun(t *testing.T) { - a := &container{container_type: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} - b := &container{container_type: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} + a := &container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}} + b := &container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}} ret := intersectionCountArrayRun(a, b) if ret != 3 { @@ -357,16 +357,16 @@ func TestIntersectionCountArrayRun(t *testing.T) { } func TestIntersectionCountBitmapRun(t *testing.T) { - a := &container{container_type: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} - b := &container{container_type: ContainerRun, runs: []interval16{{start: 63, last: 64}}} + a := &container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}} + b := &container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}} ret := intersectionCountBitmapRun(a, b) if ret != 1 { t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret) } - a = &container{container_type: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} - b = &container{container_type: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} + a = &container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}} + b = &container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}} ret = intersectionCountBitmapRun(a, b) if ret != 14 { @@ -414,8 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) { bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6}, } for i, test := range tests { - a.container_type = ContainerRun - b.container_type = ContainerRun + a.containerType = ContainerRun + b.containerType = ContainerRun a.runs = test.aruns b.runs = test.bruns ret := intersectionCountRunRun(a, b) @@ -456,8 +456,8 @@ func TestIntersectArrayRun(t *testing.T) { } for i, test := range tests { - a.container_type = ContainerArray - b.container_type = ContainerRun + a.containerType = ContainerArray + b.containerType = ContainerRun a.array = test.array b.runs = test.runs ret := intersectArrayRun(a, b) @@ -514,8 +514,8 @@ func TestIntersectRunRun(t *testing.T) { }, } for i, test := range tests { - a.container_type = ContainerRun - b.container_type = ContainerRun + a.containerType = ContainerRun + b.containerType = ContainerRun a.runs = test.aruns b.runs = test.bruns ret := intersectRunRun(a, b) @@ -579,8 +579,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { for i, v := range test.exp { exp[i] = v } - a.container_type = ContainerBitmap - b.container_type = ContainerRun + a.containerType = ContainerBitmap + b.containerType = ContainerRun ret := intersectBitmapRun(a, b) if ret.isArray() { ret.arrayToBitmap() @@ -640,8 +640,8 @@ func TestIntersectBitmapRunArray(t *testing.T) { a.bitmap[i] = v } b.runs = test.runs - a.container_type = ContainerBitmap - b.container_type = ContainerRun + a.containerType = ContainerBitmap + b.containerType = ContainerRun ret := intersectBitmapRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -658,19 +658,19 @@ func TestUnionMixed(t *testing.T) { // array container a := &container{} a.array = []uint16{1, 4, 5, 7, 10, 11, 12} - a.container_type = ContainerArray + a.containerType = ContainerArray a.n = 7 // bitmap container b := &container{bitmap: make([]uint64, bitmapN)} b.bitmap[0] = uint64(0x3) b.n = 2 - b.container_type = ContainerBitmap + b.containerType = ContainerBitmap // run container r := &container{} r.runs = []interval16{{start: 5, last: 10}} - r.container_type = ContainerRun + r.containerType = ContainerRun r.n = 6 t.Run("various container Unions", func(t *testing.T) { @@ -711,10 +711,10 @@ func TestIntersectMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = 6 - a.container_type = ContainerRun + a.containerType = ContainerRun b.array = []uint16{1, 4, 5, 7, 10, 11, 12} b.n = 7 - b.container_type = ContainerArray + b.containerType = ContainerArray res := intersect(a, b) if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) { t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array) @@ -730,7 +730,7 @@ func TestIntersectMixed(t *testing.T) { } c.bitmap = []uint64{0x60} c.n = 2 - c.container_type = ContainerBitmap + c.containerType = ContainerBitmap res = intersect(c, a) if !reflect.DeepEqual(res.array, []uint16{5, 6}) { @@ -760,15 +760,15 @@ func TestDifferenceMixed(t *testing.T) { a.runs = []interval16{{start: 5, last: 10}} a.n = a.runCountRange(0, 100) - a.container_type = ContainerRun + a.containerType = ContainerRun b.array = []uint16{0, 2, 4, 6, 8, 10, 12} b.n = len(b.array) - b.container_type = ContainerArray + b.containerType = ContainerArray d.array = []uint16{1, 3, 5, 7, 9, 11, 12} d.n = len(d.array) - d.container_type = ContainerArray + d.containerType = ContainerArray res := difference(a, b) @@ -788,7 +788,7 @@ func TestDifferenceMixed(t *testing.T) { c.bitmap = []uint64{0x64} c.n = c.countRange(0, 100) - c.container_type = ContainerBitmap + c.containerType = ContainerBitmap res = difference(c, a) if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) { t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap) @@ -883,8 +883,8 @@ func TestUnionRunRun(t *testing.T) { for i, test := range tests { a.runs = test.aruns b.runs = test.bruns - a.container_type = ContainerRun - b.container_type = ContainerRun + a.containerType = ContainerRun + b.containerType = ContainerRun ret := unionRunRun(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -925,8 +925,8 @@ func TestUnionArrayRun(t *testing.T) { for i, test := range tests { a.array = test.array b.runs = test.runs - a.container_type = ContainerArray - b.container_type = ContainerRun + a.containerType = ContainerArray + b.containerType = ContainerRun ret := unionArrayRun(a, b) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) @@ -935,7 +935,7 @@ func TestUnionArrayRun(t *testing.T) { } func TestBitmapSetRange(t *testing.T) { - c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -975,7 +975,7 @@ func TestBitmapSetRange(t *testing.T) { } func TestArrayToBitmap(t *testing.T) { - a := &container{container_type: ContainerArray} + a := &container{containerType: ContainerArray} tests := []struct { array []uint16 exp []uint64 @@ -1006,7 +1006,7 @@ func TestArrayToBitmap(t *testing.T) { } func TestBitmapToArray(t *testing.T) { - a := &container{container_type: ContainerBitmap} + a := &container{containerType: ContainerBitmap} tests := []struct { bitmap []uint64 exp []uint16 @@ -1037,7 +1037,7 @@ func TestBitmapToArray(t *testing.T) { } func TestRunToBitmap(t *testing.T) { - a := &container{container_type: ContainerRun} + a := &container{containerType: ContainerRun} tests := []struct { runs []interval16 exp []uint64 @@ -1091,7 +1091,7 @@ func getFullBitmap() []uint64 { } func TestBitmapToRun(t *testing.T) { - a := &container{container_type: ContainerBitmap} + a := &container{containerType: ContainerBitmap} tests := []struct { bitmap []uint64 exp []interval16 @@ -1169,7 +1169,7 @@ func TestBitmapToRun(t *testing.T) { } func TestArrayToRun(t *testing.T) { - a := &container{container_type: ContainerArray} + a := &container{containerType: ContainerArray} tests := []struct { array []uint16 exp []interval16 @@ -1203,7 +1203,7 @@ func TestArrayToRun(t *testing.T) { } func TestRunToArray(t *testing.T) { - a := &container{container_type: ContainerRun} + a := &container{containerType: ContainerRun} tests := []struct { runs []interval16 exp []uint16 @@ -1237,7 +1237,7 @@ func TestRunToArray(t *testing.T) { } func TestBitmapZeroRange(t *testing.T) { - c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 start uint64 @@ -1273,7 +1273,7 @@ func TestBitmapZeroRange(t *testing.T) { if test.expN != c.n { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n) } - for i, _ := range test.bitmap { + for i := range test.bitmap { c.bitmap[i] = 0 } } @@ -1281,8 +1281,8 @@ func TestBitmapZeroRange(t *testing.T) { } func TestUnionBitmapRun(t *testing.T) { - a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1313,14 +1313,14 @@ func TestUnionBitmapRun(t *testing.T) { if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) } - for i, _ := range test.bitmap { + for i := range test.bitmap { a.bitmap[i] = 0 } } } func TestBitmapCountRuns(t *testing.T) { - c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 exp int @@ -1353,7 +1353,7 @@ func TestBitmapCountRuns(t *testing.T) { t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret) } - for j, _ := range test.bitmap { + for j := range test.bitmap { c.bitmap[j] = 0 } } @@ -1370,7 +1370,7 @@ func TestBitmapCountRuns(t *testing.T) { } func TestArrayCountRuns(t *testing.T) { - c := &container{container_type: ContainerArray} + c := &container{containerType: ContainerArray} tests := []struct { array []uint16 exp int @@ -1411,8 +1411,8 @@ func TestArrayCountRuns(t *testing.T) { } func TestDifferenceArrayRun(t *testing.T) { - a := &container{container_type: ContainerArray} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerArray} + b := &container{containerType: ContainerRun} tests := []struct { array []uint16 runs []interval16 @@ -1437,8 +1437,8 @@ func TestDifferenceArrayRun(t *testing.T) { } func TestDifferenceRunArray(t *testing.T) { - a := &container{container_type: ContainerRun} - b := &container{container_type: ContainerArray} + a := &container{containerType: ContainerRun} + b := &container{containerType: ContainerArray} tests := []struct { runs []interval16 array []uint16 @@ -1508,8 +1508,8 @@ func MakeLastBitSet() []uint64 { } func TestDifferenceRunBitmap(t *testing.T) { - a := &container{container_type: ContainerRun} - b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &container{containerType: ContainerRun} + b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { runs []interval16 bitmap []uint64 @@ -1571,8 +1571,8 @@ func TestDifferenceRunBitmap(t *testing.T) { } func TestDifferenceBitmapRun(t *testing.T) { - a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + b := &container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -1599,8 +1599,8 @@ func TestDifferenceBitmapRun(t *testing.T) { } func TestDifferenceBitmapArray(t *testing.T) { - b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)} - a := &container{container_type: ContainerArray} + b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)} + a := &container{containerType: ContainerArray} tests := []struct { bitmap []uint64 array []uint16 @@ -1649,8 +1649,8 @@ func TestDifferenceBitmapArray(t *testing.T) { } func TestDifferenceBitmapBitmap(t *testing.T) { - a := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} - b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + a := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + b := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { abitmap []uint64 bbitmap []uint64 @@ -1679,8 +1679,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) { } func TestDifferenceRunRun(t *testing.T) { - a := &container{container_type: ContainerRun} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerRun} + b := &container{containerType: ContainerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -1713,7 +1713,7 @@ func TestDifferenceRunRun(t *testing.T) { } func TestWriteReadArray(t *testing.T) { - ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, container_type: ContainerArray} + ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray} ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}} ba2 := &Bitmap{} var buf bytes.Buffer @@ -1732,7 +1732,7 @@ func TestWriteReadArray(t *testing.T) { func TestWriteReadBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, container_type: ContainerBitmap} + cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap} for i := 0; i < 129; i++ { cb.bitmap[i] = 0x5555555555555555 } @@ -1754,7 +1754,7 @@ func TestWriteReadBitmap(t *testing.T) { func TestWriteReadFullBitmap(t *testing.T) { // create bitmap containing > 4096 bits - cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, container_type: ContainerBitmap} + cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap} for i := 0; i < bitmapN; i++ { cb.bitmap[i] = 0xffffffffffffffff } @@ -1782,7 +1782,7 @@ func TestWriteReadFullBitmap(t *testing.T) { } func TestWriteReadRun(t *testing.T) { - cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, container_type: ContainerRun} + cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun} br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}} br2 := &Bitmap{} var buf bytes.Buffer @@ -1806,21 +1806,21 @@ func TestXorArrayRun(t *testing.T) { exp *container }{ { - a: &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray}, - b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}, - exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, container_type: ContainerArray, n: 12}, + a: &container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray}, + b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12}, }, { - a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}, - b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}, - exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, container_type: ContainerArray, n: 12}, + a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray}, + b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}, + exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12}, }, { - a: &container{array: []uint16{65535}, container_type: ContainerArray}, - b: &container{runs: []interval16{{start: 65534, last: 65535}}, container_type: ContainerRun}, - exp: &container{array: []uint16{65534}, container_type: ContainerArray, n: 1}, + a: &container{array: []uint16{65535}, containerType: ContainerArray}, + b: &container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun}, + exp: &container{array: []uint16{65534}, containerType: ContainerArray, n: 1}, }, { - a: &container{array: []uint16{65535}, container_type: ContainerArray}, - b: &container{runs: []interval16{{start: 65535, last: 65535}}, container_type: ContainerRun}, - exp: &container{array: []uint16{}, container_type: ContainerArray, n: 0}, + a: &container{array: []uint16{65535}, containerType: ContainerArray}, + b: &container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun}, + exp: &container{array: []uint16{}, containerType: ContainerArray, n: 0}, }, } @@ -1841,8 +1841,8 @@ func TestXorArrayRun(t *testing.T) { //special case that didn't fit the xorrunrun table testing below. func TestXorRunRun1(t *testing.T) { - a := &container{container_type: ContainerRun} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerRun} + b := &container{containerType: ContainerRun} a.runs = []interval16{{start: 4, last: 10}} b.runs = []interval16{{start: 5, last: 10}} ret := xorRunRun(a, b) @@ -1856,8 +1856,8 @@ func TestXorRunRun1(t *testing.T) { } func TestXorRunRun(t *testing.T) { - a := &container{container_type: ContainerRun} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerRun} + b := &container{containerType: ContainerRun} tests := []struct { aruns []interval16 bruns []interval16 @@ -1954,7 +1954,7 @@ func TestXorRunRun(t *testing.T) { } func TestBitmapFlip(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} ttable := []struct { original uint64 @@ -1986,7 +1986,7 @@ func TestBitmapFlip(t *testing.T) { } func TestBitmapXorRange(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap} + c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { bitmap []uint64 start uint64 @@ -2054,8 +2054,8 @@ func TestBitmapXorRange(t *testing.T) { } func TestXorBitmapRun(t *testing.T) { - a := &container{container_type: ContainerBitmap} - b := &container{container_type: ContainerRun} + a := &container{containerType: ContainerBitmap} + b := &container{containerType: ContainerRun} tests := []struct { bitmap []uint64 runs []interval16 @@ -2390,65 +2390,6 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) { } } -func Test_BufBitmapIterator_Next(t *testing.T) { - b := NewBitmap() - for i := uint64(0); i < 4097; i++ { - b.Add(i) - } - if !b.containers[0].isBitmap() { - t.Fatalf("wrong container type") - } - - bin := []uint16{} - - itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap)) - x := uint16(0) - - for i := 0; i < 10; i++ { - x, _ = itr.next() - bin = append(bin, x) - } - exp := []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - if !reflect.DeepEqual(bin, exp) { - t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin) - } - - // ensure that unread points next back one such that the last value is repeated - itr.unread() - x, _ = itr.next() - bin = append(bin, x) - exp = append(exp, uint16(9)) - if !reflect.DeepEqual(bin, exp) { - t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin) - } -} - -func Test_BufBitmapIterator_UnreadPanic(t *testing.T) { - - defer func() { - if r := recover(); r == nil { - t.Errorf("BufBitmapIterator unread did not panic") - } - }() - - b := NewBitmap() - for i := uint64(0); i < 4097; i++ { - b.Add(i) - } - if !b.containers[0].isBitmap() { - t.Fatalf("wrong container type") - } - - itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap)) - for i := 0; i < 10; i++ { - itr.next() - } - - // ensure that unreading back-to-back panics - itr.unread() - itr.unread() -} - func TestSearc64(t *testing.T) { tests := []struct { a []uint64 @@ -2528,9 +2469,9 @@ func TestSearc64(t *testing.T) { } func TestIntersectArrayBitmap(t *testing.T) { - a, b := &container{container_type: ContainerArray}, &container{ - container_type: ContainerBitmap, - bitmap: make([]uint64, bitmapN), + a, b := &container{containerType: ContainerArray}, &container{ + containerType: ContainerBitmap, + bitmap: make([]uint64, bitmapN), } tests := []struct { array []uint16 @@ -2576,11 +2517,11 @@ func TestIntersectArrayBitmap(t *testing.T) { for i, test := range tests { a.array = test.array - a.container_type = ContainerArray + a.containerType = ContainerArray for i, bmval := range test.bitmap { b.bitmap[i] = bmval } - b.container_type = ContainerBitmap + b.containerType = ContainerBitmap ret := intersectArrayBitmap(a, b).array if len(ret) == 0 && len(test.exp) == 0 { continue diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f6ac33036..6378ee67f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1059,32 +1059,34 @@ func TestBitmapBufIterator(t *testing.T) { } var benchmarkBitmapIntersectionCountData struct { - a, b *roaring.Bitmap + a, b, r *roaring.Bitmap } -func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) { +func getBenchData() *struct{ a, b, r *roaring.Bitmap } { data := &benchmarkBitmapIntersectionCountData if data.a == nil { const max = (1 << 24) / 64 // Build bitmap with array container. data.a = roaring.NewBitmap() - for i, n := 0, rand.Intn(roaring.ArrayMaxSize); i < n; i++ { + for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ { data.a.Add(uint64(rand.Intn(max))) } // Build bitmap with bitmap container. data.b = roaring.NewBitmap() - for i, n := 0, roaring.ArrayMaxSize*2; i < n; i++ { + for i, n := 0, MaxContainerVal/3; i < n; i++ { data.b.Add(uint64(i * 3)) } - } - // Reset timer & benchmark. - b.ResetTimer() - for i := 0; i < b.N; i++ { - data.a.IntersectionCount(data.b) + // build bitmap with run container + data.r = roaring.NewBitmap() + for i, n := 0, MaxContainerVal; i < n; i++ { + data.r.Add(uint64(i)) + } + } + return data } // GenerateUint64Slice generates between [0, n) random uint64 numbers between min and max. @@ -1137,3 +1139,115 @@ func TestBitmap_Intersect(t *testing.T) { t.Fatalf("Counts do not match %d %d", bm0.Count(), result.Count()) } } + +func BenchmarkBitmap_IntersectionCount_ArrayRun(b *testing.B) { + data := getBenchData() + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.a.IntersectionCount(data.r) + } +} + +func BenchmarkBitmap_IntersectionCount_BitmapRun(b *testing.B) { + data := getBenchData() + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.b.IntersectionCount(data.r) + } +} + +func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) { + data := getBenchData() + // Reset timer & benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + data.a.IntersectionCount(data.b) + } +} + +const ( + NumRows = uint64(10000) + NumColums = uint64(16) + MaxContainerVal = 0xffff +) + +func BenchmarkContainerLinear(b *testing.B) { + + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for row := uint64(1); row < NumRows; row++ { + for col := uint64(1); col < NumColums; col++ { + bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + } + } + } +} + +func BenchmarkContainerReverse(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for row := NumRows - 1; row >= 1; row-- { + for col := NumColums - 1; col >= 1; col-- { + bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + } + } + } +} + +func BenchmarkContainerColumn(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for col := uint64(1); col < NumColums; col++ { + for row := uint64(1); row < NumRows; row++ { + bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + } + } + } +} + +func BenchmarkContainerOutsideIn(b *testing.B) { + middle := NumRows / uint64(2) + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + + for col := uint64(1); col < NumColums; col++ { + for row := uint64(1); row < middle; row++ { + bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + } + } + } +} + +func BenchmarkContainerInsideOut(b *testing.B) { + middle := NumRows / uint64(2) + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for col := uint64(1); col < NumColums; col++ { + for row := uint64(1); row <= middle; row++ { + bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal)) + bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal)) + } + } + } +} + +func BenchmarkSliceAscending(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for col := uint64(0); col < pilosa.SliceWidth; col++ { + bm.Add(col) + } + } +} + +func BenchmarkSliceDescending(b *testing.B) { + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- { + bm.Add(col) + } + } +} diff --git a/server.go b/server.go index aea2c8518..94f823345 100644 --- a/server.go +++ b/server.go @@ -88,6 +88,7 @@ type Server struct { MaxWritesPerRequest int LogOutput io.Writer + logger *log.Logger defaultClient InternalClient } @@ -111,9 +112,9 @@ func NewServer() *Server { LogOutput: os.Stderr, } + s.logger = log.New(s.LogOutput, "", log.LstdFlags) s.Handler.Holder = s.Holder - return s } @@ -313,7 +314,7 @@ func GetHTTPClient(t *tls.Config) *http.Client { } // Logger returns a logger that writes to LogOutput -func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } +func (s *Server) Logger() *log.Logger { return s.logger } func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) @@ -339,6 +340,7 @@ func (s *Server) monitorAntiEntropy() { syncer.Cluster = s.Cluster syncer.Closing = s.closing syncer.RemoteClient = s.RemoteClient + syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. if err := syncer.SyncHolder(); err != nil { @@ -588,32 +590,17 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.Holder.NodeID) s.diagnostics.Set("ClusterID", s.Cluster.ID) + s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - numFrames := 0 - numSlices := uint64(0) - for _, index := range s.Holder.Indexes() { - numSlices += index.MaxSlice() + 1 - for _, f := range index.Frames() { - numFrames++ - if f.rangeEnabled { - s.diagnostics.Set("BSIEnabled", true) - } - if f.timeQuantum != "" { - s.diagnostics.Set("TimeQuantumEnabled", true) - } - } - } - - s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes())) - s.diagnostics.Set("NumFrames", numFrames) - s.diagnostics.Set("NumSlices", numSlices) + enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder) openFiles, err := CountOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } s.diagnostics.Set("GoRoutines", runtime.NumGoroutine()) + s.diagnostics.EnrichWithMemoryInfo() s.diagnostics.CheckVersion() s.diagnostics.Flush() } @@ -711,3 +698,39 @@ type StatusHandler interface { ClusterStatus() (proto.Message, error) HandleRemoteStatus(proto.Message) error } + +type diagnosticsFrameProperties struct { + BSIFieldCount int + TimeQuantumEnabled bool +} + +func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) { + // NOTE: this function is not in the diagnostics package, since circular imports are not allowed. + var numSlices uint64 + numFrames := 0 + numIndexes := 0 + bsiFieldCount := 0 + timeQuantumEnabled := false + + for _, index := range holder.Indexes() { + numSlices += index.MaxSlice() + 1 + numIndexes += 1 + for _, frame := range index.Frames() { + numFrames += 1 + if frame.rangeEnabled { + if fields, err := frame.GetFields(); err == nil { + bsiFieldCount += len(fields) + } + } + if frame.TimeQuantum() != "" { + timeQuantumEnabled = true + } + } + } + + d.Set("NumIndexes", numIndexes) + d.Set("NumFrames", numFrames) + d.Set("NumSlices", numSlices) + d.Set("BSIFieldCount", bsiFieldCount) + d.Set("TimeQuantumEnabled", timeQuantumEnabled) +} diff --git a/server/server_test.go b/server/server_test.go index b96dbec26..c7b5038f7 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -277,24 +277,17 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { m0 := MustRunMain() - defer m0.Close() - - m1 := MustRunMain() - defer m1.Close() - - // Update cluster config. - m0.Server.Cluster.Nodes = []*pilosa.Node{ - {URI: m0.Server.URI}, - {URI: m1.Server.URI}, - } - m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes + // TODO: this test used to start a two node cluster, but there was a race + // condition with anti-entropy. We need some general code for starting up + // arbitrarily sized Pilosa clusters for testing, and then we should + // re-instate the multi-node nature of this test. // Create frames. client := m0.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) + t.Fatal("create index:", err) } else if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) + t.Fatal("create frame:", err) } // Write data on first cluster. @@ -307,12 +300,12 @@ func TestMain_FrameRestore(t *testing.T) { SetBit(rowID=1, frame="f", columnID=600000) SetBit(rowID=1, frame="f", columnID=800000) `); err != nil { - t.Fatal(err) + t.Fatal("setting bits:", err) } // Query row on first cluster. if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { - t.Fatal(err) + t.Fatal("bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -324,20 +317,20 @@ func TestMain_FrameRestore(t *testing.T) { // Import from first cluster. client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) if err != nil { - t.Fatal(err) + t.Fatal("new client:", err) } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal(err) + t.Fatal("create new index:", err) } else if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) + t.Fatal("create new frame:", err) } else if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil { - t.Fatal(err) + t.Fatal("restore frame:", err) } // Query row on second cluster. if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil { - t.Fatal(err) + t.Fatal("another bitmap query:", err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { - t.Fatalf("unexpected result: %s", res) + t.Fatalf("2unexpected result: %s", res) } } @@ -419,6 +412,7 @@ func NewMain() *Main { // MustRunMain returns a new, running Main. Panic on error. func MustRunMain() *Main { m := NewMain() + m.Config.Metric.Diagnostics = false // Disable diagnostics. if err := m.Run(); err != nil { panic(err) } @@ -443,8 +437,6 @@ func (m *Main) Reopen() error { m.Server.Network = *test.Network m.Config = config - println("dbg/network", *test.Network) - // Run new program. if err := m.Run(); err != nil { return err diff --git a/test/pilosa.go b/test/pilosa.go new file mode 100644 index 000000000..2d264e6ae --- /dev/null +++ b/test/pilosa.go @@ -0,0 +1,250 @@ +package test + +import ( + "bytes" + "io" + "io/ioutil" + "net" + "os" + "strconv" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" +) + +//////////////////////////////////////////////////////////////////////////////////// +// Main represents a test wrapper for main.Main. +type Main struct { + *server.Command + + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer +} + +// NewMain returns a new instance of Main with a temporary data directory and random port. +func NewMain() *Main { + path, err := ioutil.TempDir("", "pilosa-") + if err != nil { + panic(err) + } + + m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} + m.Server.Network = *Network + m.Config.DataDir = path + m.Config.Bind = "localhost:0" + m.Config.Cluster.Type = "static" + m.Command.Stdin = &m.Stdin + m.Command.Stdout = &m.Stdout + m.Command.Stderr = &m.Stderr + + if testing.Verbose() { + m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) + m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr) + } + + return m +} + +/* +// MustRunMain returns a new, running Main. Panic on error. +func MustRunMain() *Main { + m := NewMain() + m.Config.Metric.Diagnostics = false // Disable diagnostics. + if err := m.Run(); err != nil { + panic(err) + } + return m +} +*/ + +// Close closes the program and removes the underlying data directory. +func (m *Main) Close() error { + defer os.RemoveAll(m.Config.DataDir) + return m.Command.Close() +} + +// RunWithTransport runs Main and returns the dynamically allocated gossip port. +func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) { + defer close(m.Started) + + m.Config.Cluster.Type = "gossip" + + /* + TEST: + - SetupServer (just static settings from config) + - OpenListener (sets Server.Name to use in gossip) + - NewTransport (gossip) + - SetupNetworking (does the gossip or static stuff) - uses Server.Name + - Open server + + PRODUCTION: + - SetupServer (just static settings from config) + - SetupNetworking (does the gossip or static stuff) - calls NewTransport + - Open server - calls OpenListener + */ + + // SetupServer + err = m.SetupServer() + if err != nil { + return seed, coord, err + } + + // Open server listener. + // This is used to set Server.Name, which is used as the node + // name for identifying a memberlist node. + err = m.Server.OpenListener() + if err != nil { + return seed, coord, err + } + + // Open gossip transport to use in SetupServer. + transport, err := gossip.NewTransport(host, bindPort) + if err != nil { + return seed, coord, err + } + m.GossipTransport = transport + + if joinSeed != "" { + m.Config.Gossip.Seed = joinSeed + } else { + m.Config.Gossip.Seed = transport.URI.String() + } + seed = m.Config.Gossip.Seed + + // SetupNetworking + err = m.SetupNetworking() + if err != nil { + return seed, coord, err + } + + if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil { + return seed, coord, err + } + + if coordinator != nil { + coord = *coordinator + } else { + coord = m.Server.URI + } + m.Server.Cluster.Coordinator = coord + m.Server.Cluster.Static = false + + // Initialize server. + err = m.Server.Open() + if err != nil { + return seed, coord, err + } + + return seed, coord, nil +} + +//////////////////////////////////////////////////////////////////////////////////// + +func MustNewRunningServer(t *testing.T) *server.Command { + s, err := newServer() + if err != nil { + t.Fatalf("getting new server: %v", err) + } + + err = s.Run() + if err != nil { + t.Fatalf("running new pilosa server: %v", err) + } + return s +} + +func newServer() (*server.Command, error) { + s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard) + + port, err := findPort() + if err != nil { + return nil, errors.Wrap(err, "getting port") + } + s.Config.Bind = "localhost:" + strconv.Itoa(port) + + gport, err := findPort() + if err != nil { + return nil, errors.Wrap(err, "getting gossip port") + } + s.Config.GossipPort = strconv.Itoa(gport) + + s.Config.GossipSeed = "localhost:" + s.Config.GossipPort + s.Config.Cluster.Type = "gossip" + s.Config.Metric.Diagnostics = false + td, err := ioutil.TempDir("", "") + if err != nil { + return nil, errors.Wrap(err, "temp dir") + } + s.Config.DataDir = td + return s, nil +} + +func findPort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", ":0") + if err != nil { + return 0, errors.Wrap(err, "resolving new port addr") + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, errors.Wrap(err, "listening to get new port") + } + port := l.Addr().(*net.TCPAddr).Port + err = l.Close() + if err != nil { + return port, errors.Wrap(err, "closing listener") + } + return port, nil + +} + +func MustFindPort(t *testing.T) int { + port, err := findPort() + if err != nil { + t.Fatalf("allocating new port: %v", err) + } + return port +} + +type Cluster struct { + Servers []*Main +} + +func MustNewServerCluster(t *testing.T, size int) *Cluster { + cluster, err := NewServerCluster(size) + if err != nil { + t.Fatalf("new cluster: %v", err) + } + return cluster +} + +func NewServerCluster(size int) (cluster *Cluster, err error) { + if size == 0 { + return nil, errors.New("cluster must contain at least one node") + } + + cluster = &Cluster{ + Servers: make([]*Main, size), + } + + gossipHost := "localhost" + gossipPort := 0 + var gossipSeed string + var coordinator pilosa.URI + + for i := 0; i < size; i++ { + m := NewMain() + + gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, &coordinator) + if err != nil { + return nil, errors.Wrap(err, "RunWithTransport") + } + + cluster.Servers[i] = m + } + + return cluster, nil +} diff --git a/test/pilosa_test.go b/test/pilosa_test.go new file mode 100644 index 000000000..8478e6bba --- /dev/null +++ b/test/pilosa_test.go @@ -0,0 +1,46 @@ +package test_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +func TestNewCluster(t *testing.T) { + cluster := test.MustNewServerCluster(t, 3) + + response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status") + if err != nil { + t.Fatalf("getting schema: %v", err) + } + dec := json.NewDecoder(response.Body) + body := struct { + State string + Nodes []struct { + Scheme string + Host string + Port int + } + }{} + + err = dec.Decode(&body) + if err != nil { + t.Fatalf("decoding status response: %v", err) + } + + bytes, err := json.MarshalIndent(body, "", " ") + if err != nil { + t.Fatalf("encoding: %v", err) + } + + if len(body.Nodes) != 3 { + t.Fatalf("wrong number of nodes in status: %s", bytes) + } + + if body.State != pilosa.ClusterStateNormal { + t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State) + } +}