scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into cluster-resize

Browse source
This commit is contained in:
Travis Turner 2018-01-17 12:55:12 -06:00
commit c0fab2bcee
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
24 changed files with 1365 additions and 703 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
Gopkg.lock generated
View file

@ -14,10 +14,16 @@
revision = "39b0596a2da3c92787b3319c6b5425a474b4e0da"
[[projects]]
branch = "master"
name = "github.com/DataDog/datadog-go"
packages = ["statsd"]
revision = "0ddda6bee21174ef6c4873647cb0d6ec9cba996f"
version = "1.1.0"
revision = "4d2e5696ebe914940bd7459d2266fb7d555ea1b7"
[[projects]]
branch = "master"
name = "github.com/StackExchange/wmi"
packages = ["."]
revision = "ea383cf3ba6ec950874b8486cd72356d007c768f"
[[projects]]
branch = "master"
@ -43,11 +49,17 @@
revision = "629574ca2a5df945712d3079857300b5e4da0236"
version = "v1.4.2"
[[projects]]
name = "github.com/go-ole/go-ole"
packages = [".","oleutil"]
revision = "0e87ea779d9deb219633b828a023b32e1244dd57"
version = "v1.2.0"
[[projects]]
name = "github.com/gogo/protobuf"
packages = ["proto"]
revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
version = "v0.5"
revision = "100ba4e885062801d56799d78530b73b178a78f3"
version = "v0.4"
[[projects]]
branch = "master"
@ -151,6 +163,12 @@
revision = "16398bac157da96aa88f98a2df640c7f32af1da2"
version = "v1.0.1"
[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
name = "github.com/rakyll/statik"
packages = ["fs"]
@ -169,6 +187,12 @@
packages = ["."]
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
[[projects]]
name = "github.com/shirou/gopsutil"
packages = ["host","internal/common","mem","process"]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
[[projects]]
name = "github.com/sony/gobreaker"
packages = ["."]
@ -226,7 +250,7 @@
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix"]
packages = ["unix","windows"]
revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6"
[[projects]]
@ -244,6 +268,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "210f654a7a072d5751f0814e4d71ef0758dd53b3dc59ed462619396ef8621d81"
inputs-digest = "d91110a10c830f7a9cc439b9578840d97d9921e84d08242316da8d4a18c68c56"
solver-name = "gps-cdcl"
solver-version = 1

11
Gopkg.toml
View file

@ -5,3 +5,14 @@
[[constraint]]
name = "github.com/satori/go.uuid"
version = "1.1.0"
[[constraint]]
name = "github.com/shirou/gopsutil"
version = "2.17.11"
[[constraint]]
# Required: the root import path of the project being constrained.
name = "github.com/DataDog/datadog-go"
# Recommended: the version constraint to enforce for the project.
# Only one of "branch", "version" or "revision" can be specified.
branch = "master"

5
Makefile
View file

@ -4,6 +4,7 @@ DEP := $(shell command -v dep 2>/dev/null)
STATIK := $(shell command -v statik 2>/dev/null)
PROTOC := $(shell command -v protoc 2>/dev/null)
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
STATUS := $(shell git status --porcelain)
IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
CLONE_URL=github.com/pilosa/pilosa
PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
@ -65,9 +66,13 @@ endif
@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
release:
ifeq ($(STATUS),"")
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
else
@echo "Will not create release with unclean git status."
endif
prerelease-build: vendor
make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa"

7
cache.go
View file

@ -314,19 +314,22 @@ func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
// Pair holds an id/count pair.
type Pair struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Count uint64 `json:"count"`
}
func encodePair(p Pair) *internal.Pair {
return &internal.Pair{
Key: p.ID,
ID: p.ID,
Key: p.Key,
Count: p.Count,
}
}
func decodePair(pb *internal.Pair) Pair {
return Pair{
ID: pb.Key,
ID: pb.ID,
Key: pb.Key,
Count: pb.Count,
}
}

10
client_test.go
View file

@ -157,7 +157,7 @@ func TestClient_MultiNode(t *testing.T) {
// Check the results before every node has the correct max slice value.
pairs := result.Results[0].Pairs
for _, pair := range pairs {
if pair.Key == 22 && pair.Count != 3 {
if pair.ID == 22 && pair.Count != 3 {
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
}
}
@ -177,10 +177,10 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result))
}
p := []*internal.Pair{
{Key: 100, Count: 12},
{Key: 22, Count: 11},
{Key: 98, Count: 8},
{Key: 99, Count: 7}}
{ID: 100, Count: 12},
{ID: 22, Count: 11},
{ID: 98, Count: 8},
{ID: 99, Count: 7}}
// Valdidate the Top 4 result counts.
if !reflect.DeepEqual(result.Results[0].Pairs, p) {

39
diagnostics/diagnostics.go
View file

@ -13,6 +13,8 @@ import (
"sync"
"time"
"github.com/shirou/gopsutil/host"
"github.com/shirou/gopsutil/mem"
"github.com/sony/gobreaker"
)
@ -94,7 +96,7 @@ func (d *Diagnostics) schedule() {
// Flush sends the current metrics.
func (d *Diagnostics) Flush() error {
d.mu.Lock()
d.metrics["uptime"] = (time.Now().Unix() - d.startTime)
d.metrics["Uptime"] = (time.Now().Unix() - d.startTime)
buf, _ := d.Encode()
d.mu.Unlock()
@ -203,6 +205,41 @@ func (d *Diagnostics) logger() *log.Logger {
return log.New(d.logOutput, "", log.LstdFlags)
}
// EnrichWithOSInfo adds OS information to the diagnostics payload.
func (d *Diagnostics) EnrichWithOSInfo() {
osInfo, err := host.Info()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("HostUptime", osInfo.Uptime)
platform, family, version, err := host.PlatformInformation()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSPlatform", platform)
d.Set("OSFamily", family)
d.Set("OSVersion", version)
kernelVersion, err := host.KernelVersion()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("OSKernelVersion", kernelVersion)
}
// EnrichWithMemoryInfo adds memory information to the diagnostics payload.
func (d *Diagnostics) EnrichWithMemoryInfo() {
memory, err := mem.VirtualMemory()
if err != nil {
d.logOutput.Write([]byte(err.Error()))
}
d.Set("MemFree", memory.Free)
d.Set("MemTotal", memory.Total)
d.Set("MemUsed", memory.Used)
}
// VersionSegments returns the numeric segments of the version as a slice of ints.
func VersionSegments(segments string) []int {
segments = strings.Trim(segments, "v")

2
diagnostics/diagnostics_test.go
View file

@ -49,7 +49,7 @@ func TestDiagnosticsClient(t *testing.T) {
t.Fatal(err)
}
output2 := []byte(`{"gg":10,"ss":"ss","uptime":0}`)
output2 := []byte(`{"gg":10,"ss":"ss","Uptime":0}`)
if eq, err = compareJSON(data, output2); err != nil {
t.Fatal(err)
}

231
docs/tutorials.md
View file

@ -2,13 +2,15 @@
title = "Tutorials"
weight = 4
nav = [
"How To Setup a Secure Cluster",
"Setting Up a Secure Cluster",
"Using Integer Field Values",
"Storing Row and Column Attributes",
]
+++
## Tutorials
### How To Setup a Secure Cluster
### Setting Up a Secure Cluster
#### Introduction
@ -212,3 +214,228 @@ curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap
#### What's Next?
Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
### Using Integer Field Values
#### Introduction
Pilosa can store integer values associated to the columns in an index, and those values are used to support range and aggregate queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients.
First, create an index called `patients`:
```
curl localhost:10101/index/patients \
-X POST
```
Next, create a frame in the `patients` index called `measurements` which will represent information gathered about each patient.
```
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{"rangeEnabled": true}}'
```
In addition to storing rows of bits, a frame can also contain fields that store integer values. The next step creates three fields (`age`, `weight`, `tcells`) in the `measurements` frame.
```
curl localhost:10101/index/patients/frame/measurements \
-X POST \
-d '{"options":{
"rangeEnabled": true,
"fields": [
{"name": "age", "type": "int", "min": 0, "max": 120},
{"name": "weight", "type": "int", "min": 0, "max": 500},
{"name": "tcells", "type": "int", "min": 0, "max": 2000}
]
}}'
```
If you need to, you can add fields to an existing frame by posting to the [Create Field endpoint](../api-reference/#create-field).
Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL.
This query sets the age, weight, and t-cell count for the patient with ID `1` in our system:
```
curl localhost:10101/index/patients/query \
-X POST \
-d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)'
```
In the case where we need to load a lot of data at once, we can use the `pilosa import` command. This method lets us import data into Pilosa from a CSV file.
Assuming we have a file called `ages.csv` that is structured like this:
```
1,34
2,57
3,19
4,40
5,32
6,71
7,28
8,33
9,63
```
where the first column of the CSV represents the patient `ID` and the second column represents the patient's`age`, then we can import the data into our `age` field by running this command:
```
pilosa import -i patients -f measurements --field age ages.csv
```
Now that we have some data in our index, let's run a few queries to demonstrate how to use that data.
In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field.
```
curl localhost:10101/index/patients/query \
-X POST \
-d 'Range(frame="measurements", age > 40)'
```
You should get the following results:
```
{"results":[{"attrs":{},"bits":[2,6,9]}]}
```
You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation.
To find the average age of all patients, run a `Sum` query:
```
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(frame="measurements", field="age")'
```
The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`.
```
{"results":[{"sum":377,"count":9}]}
```
You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40.
```
curl localhost:10101/index/patients/query \
-X POST \
-d 'Sum(Range(frame="measurements", age > 40), frame="measurements", field="age")'
```
Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query.
```
{"results":[{"sum":191,"count":3}]}
```
### Storing Row and Column Attributes
#### Introduction
Pilosa can store arbitrary values associated to any row or column. In Pilosa, these are referred to as `attributes`, and they can be of type `string`, `integer`, `boolean`, or `float`. In this tutorial we will store some attribute data and then run some queries that return that data.
First, create an index called `books` to use for this tutorial:
```
curl localhost:10101/index/books \
-X POST
```
Next, create a frame in the `books` index called `members` which will represent library members who have read books.
```
curl localhost:10101/index/books/frame/members \
-X POST \
-d '{}'
```
Now, let's add some books to our index.
```
curl localhost:10101/index/books/query \
-X POST \
-d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960)
SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972)
SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000)
SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003)
SetColumnAttrs(columnID=5, name="The Forever War", year=2008)'
```
And add some members.
```
curl localhost:10101/index/books/query \
-X POST \
-d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith")
SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins")
SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks")
SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez")
SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")'
```
At this point we can query one of the `member` records by querying that row.
```
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
You should get the following result set:
```
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]}
```
Now let's add some data to the matrix such that each pair represents a member who has read that book.
```
curl localhost:10101/index/books/query \
-X POST \
-d 'SetBit(frame="members", rowID=10001, columnID=3)
SetBit(frame="members", rowID=10001, columnID=5)
SetBit(frame="members", rowID=10002, columnID=1)
SetBit(frame="members", rowID=10002, columnID=2)
SetBit(frame="members", rowID=10002, columnID=4)
SetBit(frame="members", rowID=10003, columnID=3)
SetBit(frame="members", rowID=10004, columnID=4)
SetBit(frame="members", rowID=10004, columnID=5)
SetBit(frame="members", rowID=10005, columnID=1)
SetBit(frame="members", rowID=10005, columnID=2)
SetBit(frame="members", rowID=10005, columnID=3)
SetBit(frame="members", rowID=10005, columnID=4)
SetBit(frame="members", rowID=10005, columnID=5)'
```
Now pull the record for `Sue Perkins` again.
```
curl localhost:10101/index/books/query \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Notice that the result set now contains a list of integers in the `bits` attribute. These integers match the column IDs of the books that Sue has read.
```
{"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]}
```
In order to retrieve the attribute information that we stored for each book, we need to add a URL parameter `columnAttrs=true` to the query.
```
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Bitmap(frame="members", rowID=10002)'
```
Here, the `book` attributes will be included in the result set at the `columnAttrs` attribute.
```
{
"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}],
"columnAttrs":[
{"id":1,"attrs":{"name":"To Kill a Mockingbird","year":1960}},
{"id":2,"attrs":{"name":"No Name in the Street","year":1972}},
{"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
]
}
```
Finally, if we want to find out which books were read by both `Sue` and `Pedro`, we just perform an `Intersect` query on those two members:
```
curl localhost:10101/index/books/query?columnAttrs=true \
-X POST \
-d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))'
```
```
{
"results":[{"attrs":{},"bits":[4]}],
"columnAttrs":[
{"id":4,"attrs":{"name":"Out Stealing Horses","year":2003}}
]
}
```
Notice that we don't get row attributes on a complex query, but we still get the column attributes—in this case book information.

4
frame.go
View file

@ -541,8 +541,8 @@ func (f *Frame) view(name string) *View { return f.views[name] }
// Views returns a list of all views in the frame.
func (f *Frame) Views() []*View {
f.mu.Lock()
defer f.mu.Unlock()
f.mu.RLock()
defer f.mu.RUnlock()
other := make([]*View, 0, len(f.views))
for _, view := range f.views {

10
gossip/gossip.go
View file

@ -32,6 +32,7 @@ import (
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pkg/errors"
)
// Ensure GossipMemberSet implements interfaces.
@ -90,7 +91,7 @@ func (g *GossipMemberSet) Open() error {
g.memberlist, err = memberlist.Create(g.config.memberlistConfig)
g.mu.Unlock()
if err != nil {
return fmt.Errorf("creating memberlist: %s", err)
return errors.Wrap(err, "creating memberlist")
}
g.broadcasts = &memberlist.TransmitLimitedQueue{
@ -114,7 +115,7 @@ func (g *GossipMemberSet) Open() error {
err = g.joinWithRetry(pilosa.NodeSet(pilosa.Nodes(nodes).URIs()).ToHostPortStrings())
g.mu.RUnlock()
if err != nil {
return fmt.Errorf("joining member set: %s", err)
return errors.Wrap(err, "joinWithRetry")
}
return nil
}
@ -178,7 +179,7 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport
}
// memberlist config
conf := memberlist.DefaultLocalConfig()
conf := memberlist.DefaultWANConfig()
conf.Transport = transport.Net
conf.Name = name
conf.BindAddr = host
@ -216,7 +217,6 @@ func NewGossipMemberSetWithTransport(name string, cfg *pilosa.Config, transport
// NewGossipMemberSet returns a new instance of GossipMemberSet given a gossip port.
func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server) (*GossipMemberSet, error) {
port, err := strconv.Atoi(cfg.Gossip.Port)
if err != nil {
return nil, fmt.Errorf("convert port: %s", err)
@ -431,7 +431,7 @@ type Transport struct {
//func NewTransport(host string, port int) (*memberlist.NetTransport, error) {
func NewTransport(host string, port int) (*Transport, error) {
// memberlist config
conf := memberlist.DefaultLocalConfig()
conf := memberlist.DefaultWANConfig()
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port

7
handler.go
View file

@ -1623,10 +1623,15 @@ func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) {
// handleGetVersion handles /version requests.
func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
version := Version
if strings.HasPrefix(version, "v") {
// make the version string semver-compatible
version = version[1:]
}
if err := json.NewEncoder(w).Encode(struct {
Version string `json:"version"`
}{
Version: Version,
Version: version,
}); err != nil {
h.logger().Printf("write version response error: %s", err)
}

6
handler_test.go
View file

@ -1181,9 +1181,13 @@ func TestHandler_Version(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
version := pilosa.Version
if strings.HasPrefix(version, "v") {
version = version[1:]
}
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+pilosa.Version+`"}`+"\n" {
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}

23
holder.go
View file

@ -299,15 +299,14 @@ func (h *Holder) index(name string) *Index { return h.indexes[name] }
// Indexes returns a list of all indexes in the holder.
func (h *Holder) Indexes() []*Index {
h.mu.Lock()
defer h.mu.Unlock()
h.mu.RLock()
a := make([]*Index, 0, len(h.indexes))
for _, index := range h.indexes {
a = append(a, index)
}
sort.Sort(indexSlice(a))
h.mu.RUnlock()
sort.Sort(indexSlice(a))
return a
}
@ -559,6 +558,9 @@ type HolderSyncer struct {
Cluster *Cluster
RemoteClient *http.Client
// Stats
Stats StatsClient
// Signals that the sync should stop.
Closing <-chan struct{}
}
@ -575,6 +577,7 @@ func (s *HolderSyncer) IsClosing() bool {
// SyncHolder compares the holder on host with the local holder and resolves differences.
func (s *HolderSyncer) SyncHolder() error {
ti := time.Now()
// Iterate over schema in sorted order.
for _, di := range s.Holder.Schema() {
// Verify syncer has not closed.
@ -587,6 +590,7 @@ func (s *HolderSyncer) SyncHolder() error {
return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err)
}
tf := time.Now()
for _, fi := range di.Frames {
// Verify syncer has not closed.
if s.IsClosing() {
@ -621,7 +625,11 @@ func (s *HolderSyncer) SyncHolder() error {
}
}
}
s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0)
tf = time.Now() // reset tf
}
s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0)
ti = time.Now() // reset ti
}
return nil
@ -634,12 +642,14 @@ func (s *HolderSyncer) syncIndex(index string) error {
if idx == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
// Read block checksums.
blks, err := idx.ColumnAttrStore().Blocks()
if err != nil {
return err
}
s.Stats.CountWithCustomTags("ColumnAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) {
@ -653,6 +663,7 @@ func (s *HolderSyncer) syncIndex(index string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("ColumnAttrDiff", int64(len(m)), 1.0, []string{indexTag, node.URI.HostPort()})
// Update local copy.
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
@ -676,12 +687,15 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
if f == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
frameTag := fmt.Sprintf("frame:%s", name)
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return err
}
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterURI(s.URI) {
@ -697,6 +711,7 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.URI.HostPort()})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {

1
holder_test.go
View file

@ -490,6 +490,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
URI: cluster.Nodes[0].URI,
Cluster: cluster,
RemoteClient: pilosa.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
}
if err := syncer.SyncHolder(); err != nil {

257
internal/public.pb.go
View file

@ -44,6 +44,7 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Bitmap struct {
Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
@ -59,6 +60,13 @@ func (m *Bitmap) GetBits() []uint64 {
return nil
}
func (m *Bitmap) GetKeys() []string {
if m != nil {
return m.Keys
}
return nil
}
func (m *Bitmap) GetAttrs() []*Attr {
if m != nil {
return m.Attrs
@ -67,7 +75,8 @@ func (m *Bitmap) GetAttrs() []*Attr {
}
type Pair struct {
Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"`
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"`
Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
@ -76,11 +85,18 @@ func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} }
func (m *Pair) GetKey() uint64 {
func (m *Pair) GetID() uint64 {
if m != nil {
return m.ID
}
return 0
}
func (m *Pair) GetKey() string {
if m != nil {
return m.Key
}
return 0
return ""
}
func (m *Pair) GetCount() uint64 {
@ -148,6 +164,7 @@ func (m *Bit) GetTimestamp() int64 {
type ColumnAttrSet struct {
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
@ -163,6 +180,13 @@ func (m *ColumnAttrSet) GetID() uint64 {
return 0
}
func (m *ColumnAttrSet) GetKey() string {
if m != nil {
return m.Key
}
return ""
}
func (m *ColumnAttrSet) GetAttrs() []*Attr {
if m != nil {
return m.Attrs
@ -548,6 +572,21 @@ func (m *Bitmap) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if len(m.Keys) > 0 {
for _, s := range m.Keys {
dAtA[i] = 0x1a
i++
l = len(s)
for l >= 1<<7 {
dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
l >>= 7
i++
}
dAtA[i] = uint8(l)
i++
i += copy(dAtA[i:], s)
}
}
return i, nil
}
@ -566,16 +605,22 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.Key != 0 {
if m.ID != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Key))
i = encodeVarintPublic(dAtA, i, uint64(m.ID))
}
if m.Count != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Count))
}
if len(m.Key) > 0 {
dAtA[i] = 0x1a
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Key)))
i += copy(dAtA[i:], m.Key)
}
return i, nil
}
@ -672,6 +717,12 @@ func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if len(m.Key) > 0 {
dAtA[i] = 0x1a
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Key)))
i += copy(dAtA[i:], m.Key)
}
return i, nil
}
@ -1143,18 +1194,28 @@ func (m *Bitmap) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if len(m.Keys) > 0 {
for _, s := range m.Keys {
l = len(s)
n += 1 + l + sovPublic(uint64(l))
}
}
return n
}
func (m *Pair) Size() (n int) {
var l int
_ = l
if m.Key != 0 {
n += 1 + sovPublic(uint64(m.Key))
if m.ID != 0 {
n += 1 + sovPublic(uint64(m.ID))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
}
l = len(m.Key)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
return n
}
@ -1197,6 +1258,10 @@ func (m *ColumnAttrSet) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
l = len(m.Key)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
return n
}
@ -1523,6 +1588,35 @@ func (m *Bitmap) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex]))
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -1575,9 +1669,9 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
}
m.Key = 0
m.ID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1587,7 +1681,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.Key |= (uint64(b) & 0x7F) << shift
m.ID |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -1611,6 +1705,35 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
break
}
}
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Key = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -1906,6 +2029,35 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Key = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -3434,46 +3586,47 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 651 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc,
0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a,
0x96, 0xfc, 0x62, 0x3c, 0x16, 0xed, 0x77, 0xb0, 0x61, 0xcd, 0x06, 0x7e, 0x80, 0x1d, 0x1f, 0xc0,
0x92, 0x4f, 0x40, 0xe1, 0x47, 0xd0, 0xbd, 0xe3, 0x89, 0x1d, 0x16, 0xc0, 0x82, 0xdd, 0x9c, 0x73,
0x1f, 0xbe, 0x8f, 0x73, 0x0d, 0xd3, 0xb2, 0xbe, 0x48, 0x93, 0xcb, 0x45, 0xa9, 0x0a, 0x5d, 0xf0,
0x51, 0x92, 0x6b, 0xa9, 0xf2, 0x38, 0xf5, 0x43, 0x18, 0x84, 0x89, 0xce, 0xe2, 0x92, 0x73, 0x70,
0xc3, 0x44, 0x57, 0x1e, 0x9b, 0x3b, 0x81, 0x2b, 0xe8, 0xcd, 0x1f, 0x41, 0xff, 0xa9, 0xd6, 0xaa,
0xf2, 0x7a, 0x73, 0x27, 0x98, 0x2c, 0xf7, 0x17, 0x36, 0x6e, 0x81, 0xb4, 0x30, 0x46, 0x7f, 0x01,
0xee, 0x8b, 0x38, 0x51, 0xfc, 0x00, 0x9c, 0x67, 0xf2, 0xd6, 0x63, 0x73, 0x16, 0xb8, 0x02, 0x9f,
0xfc, 0x2e, 0xf4, 0x8f, 0x8b, 0x3a, 0xd7, 0x5e, 0x8f, 0x38, 0x03, 0xfc, 0x25, 0x8c, 0xd6, 0x75,
0x46, 0x6f, 0x8c, 0x59, 0xd7, 0x19, 0xc5, 0x38, 0x02, 0x9f, 0xbb, 0x31, 0x8e, 0x8d, 0x79, 0x05,
0x4e, 0x98, 0x68, 0x34, 0x8a, 0xe2, 0x5d, 0x74, 0xd2, 0x7c, 0xc4, 0x00, 0xfe, 0x00, 0x46, 0xc7,
0x45, 0x5a, 0x67, 0x79, 0x74, 0xd2, 0x7c, 0x69, 0x8b, 0xf9, 0x43, 0x18, 0x9f, 0x27, 0x99, 0xac,
0x74, 0x9c, 0x95, 0x9e, 0x43, 0x29, 0x5b, 0xc2, 0x5f, 0xc1, 0x9e, 0xf1, 0xc4, 0x4e, 0xd6, 0x52,
0xf3, 0x7d, 0xe8, 0x6d, 0xb3, 0xf7, 0xa2, 0x93, 0x7f, 0x9c, 0xc0, 0x67, 0x06, 0x2e, 0xbe, 0xba,
0x23, 0x18, 0x9b, 0x11, 0x70, 0x70, 0xcf, 0x6f, 0x4b, 0xd9, 0xd4, 0x45, 0x6f, 0x3e, 0x87, 0xc9,
0x5a, 0xab, 0x24, 0xbf, 0x7e, 0x1d, 0xa7, 0xb5, 0xa4, 0xaa, 0xc6, 0xa2, 0x4b, 0x61, 0x47, 0x51,
0xae, 0x8d, 0xd9, 0xa5, 0xa2, 0xb7, 0x18, 0x3b, 0x0a, 0x8b, 0x22, 0x35, 0xc6, 0xfe, 0x9c, 0x05,
0x23, 0xd1, 0x12, 0x7c, 0x06, 0x70, 0x9a, 0x16, 0x71, 0x13, 0x3b, 0x98, 0xb3, 0x80, 0x89, 0x0e,
0xe3, 0x1f, 0xc1, 0x10, 0x2b, 0x7d, 0x1e, 0x97, 0x6d, 0x6f, 0xec, 0x4f, 0xbd, 0x7d, 0x65, 0x30,
0x7d, 0x59, 0x4b, 0x75, 0x2b, 0xe4, 0xdb, 0x5a, 0x56, 0xb4, 0x03, 0xc2, 0x4d, 0x97, 0x06, 0xf0,
0x43, 0x18, 0xac, 0xd3, 0xe4, 0x52, 0x9a, 0x49, 0xb9, 0xa2, 0x41, 0xd8, 0x6b, 0x3b, 0xe1, 0x8a,
0x7a, 0x1d, 0x89, 0x2e, 0x85, 0x91, 0x42, 0x66, 0x85, 0xb6, 0xcd, 0x34, 0x88, 0xfb, 0x30, 0x5d,
0xdd, 0x5c, 0xa6, 0xf5, 0x95, 0x34, 0xa1, 0x03, 0xb2, 0xee, 0x70, 0x98, 0xbd, 0xc1, 0xa4, 0xdd,
0xa1, 0xc9, 0xde, 0xa1, 0xfc, 0xf7, 0x0c, 0xf6, 0x9a, 0xf2, 0xab, 0xb2, 0xc8, 0x2b, 0x89, 0x3b,
0x5a, 0x29, 0x65, 0x77, 0xb4, 0x52, 0x8a, 0x1f, 0xc1, 0x50, 0xc8, 0xaa, 0x4e, 0xb5, 0x5d, 0xf3,
0xbd, 0x76, 0x14, 0x36, 0xb6, 0x4e, 0xb5, 0xb0, 0x5e, 0xfc, 0x09, 0xec, 0xef, 0xc8, 0x06, 0xfb,
0xc2, 0xb8, 0xfb, 0x6d, 0xdc, 0x8e, 0x5d, 0xfc, 0xe6, 0xee, 0x7f, 0x61, 0x30, 0xe9, 0x64, 0xe6,
0x81, 0x3d, 0x43, 0x2a, 0x6b, 0xb2, 0x3c, 0x68, 0x13, 0x19, 0x5e, 0xd8, 0x33, 0x9d, 0x02, 0x3b,
0x6b, 0xc4, 0xc4, 0xce, 0x70, 0x85, 0x78, 0x7a, 0xf6, 0xfb, 0x9d, 0x15, 0x22, 0x2d, 0x8c, 0x91,
0x7b, 0x30, 0x3c, 0x7e, 0x13, 0xe7, 0xd7, 0xf2, 0x8a, 0xc4, 0x34, 0x12, 0x16, 0xf2, 0x45, 0x7b,
0x8a, 0x34, 0xfd, 0xc9, 0x92, 0xb7, 0x29, 0xac, 0x45, 0x6c, 0x7d, 0xfc, 0x4f, 0x0c, 0xf6, 0xa2,
0xac, 0x2c, 0x94, 0xee, 0xa8, 0x21, 0xca, 0xaf, 0xe4, 0x8d, 0x55, 0x03, 0x01, 0x64, 0x4f, 0x55,
0x9c, 0x19, 0xd9, 0x8f, 0x85, 0x01, 0xc8, 0x92, 0x2a, 0x48, 0x05, 0xae, 0x30, 0x80, 0xf6, 0x8f,
0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b,
0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a,
0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1,
0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6,
0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00,
0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00,
// 671 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xbb, 0x6e, 0xd4, 0x40,
0x14, 0x65, 0xd6, 0xde, 0xd7, 0xdd, 0x4d, 0x14, 0x8d, 0x20, 0x58, 0x08, 0xad, 0x2c, 0x8b, 0xc2,
0xd5, 0x46, 0x5a, 0x7a, 0x10, 0x9b, 0x87, 0x64, 0x45, 0x44, 0x70, 0x37, 0x84, 0xda, 0x49, 0x46,
0xc1, 0x92, 0x5f, 0xd8, 0x63, 0x91, 0xfd, 0x0e, 0x1a, 0x6a, 0x1a, 0xf8, 0x01, 0x3a, 0x3e, 0x80,
0x92, 0x4f, 0x40, 0xe1, 0x47, 0xd0, 0x9d, 0xf1, 0xd8, 0x5e, 0x22, 0x01, 0x05, 0xdd, 0x9c, 0x73,
0x66, 0xae, 0xef, 0xe3, 0x5c, 0xc3, 0x34, 0xaf, 0xce, 0xe3, 0xe8, 0x62, 0x9e, 0x17, 0x99, 0xcc,
0xf8, 0x28, 0x4a, 0xa5, 0x28, 0xd2, 0x30, 0xf6, 0xce, 0x60, 0xb0, 0x8c, 0x64, 0x12, 0xe6, 0x9c,
0x83, 0xbd, 0x8c, 0x64, 0xe9, 0x30, 0xd7, 0xf2, 0x6d, 0x54, 0x67, 0xfe, 0x08, 0xfa, 0xcf, 0xa4,
0x2c, 0x4a, 0xa7, 0xe7, 0x5a, 0xfe, 0x64, 0xb1, 0x3d, 0x37, 0xef, 0xe6, 0x44, 0xa3, 0x16, 0xe9,
0xe5, 0xb1, 0x58, 0x97, 0x8e, 0xe5, 0x5a, 0xfe, 0x18, 0xd5, 0xd9, 0x7b, 0x02, 0xf6, 0x8b, 0x30,
0x2a, 0xf8, 0x36, 0xf4, 0x82, 0x03, 0x87, 0xb9, 0xcc, 0xb7, 0xb1, 0x17, 0x1c, 0xf0, 0xbb, 0xd0,
0xdf, 0xcf, 0xaa, 0x54, 0x3a, 0x3d, 0x45, 0x69, 0xc0, 0x77, 0xc0, 0x3a, 0x16, 0x6b, 0xc7, 0x72,
0x99, 0x3f, 0x46, 0x3a, 0x7a, 0x0b, 0x18, 0xad, 0xaa, 0xa4, 0x51, 0x57, 0x55, 0xa2, 0x82, 0x58,
0x48, 0xc7, 0xcd, 0x28, 0x56, 0x1d, 0xc5, 0x7b, 0x05, 0xd6, 0x32, 0x92, 0x24, 0x62, 0xf6, 0xae,
0xf9, 0xaa, 0x06, 0xfc, 0x01, 0x8c, 0xf6, 0xb3, 0xb8, 0x4a, 0xd2, 0xe0, 0xa0, 0xfe, 0x76, 0x83,
0xf9, 0x43, 0x18, 0x9f, 0x46, 0x89, 0x28, 0x65, 0x98, 0xe4, 0x2a, 0x09, 0x0b, 0x5b, 0xc2, 0x7b,
0x0d, 0x5b, 0xfa, 0x26, 0x55, 0xbb, 0x12, 0xf2, 0x56, 0x4d, 0xff, 0xd6, 0xa5, 0xdb, 0x35, 0x7e,
0x66, 0x60, 0x93, 0x66, 0x24, 0xd6, 0x48, 0xd4, 0xd2, 0xd3, 0x75, 0x2e, 0xea, 0x4c, 0xd5, 0x99,
0xbb, 0x30, 0x59, 0xc9, 0x22, 0x4a, 0xaf, 0xce, 0xc2, 0xb8, 0x12, 0x75, 0xa0, 0x2e, 0x45, 0x35,
0x06, 0xa9, 0xd4, 0xb2, 0xad, 0xca, 0x68, 0x30, 0xd5, 0xb8, 0xcc, 0xb2, 0x58, 0x8b, 0x7d, 0x97,
0xf9, 0x23, 0x6c, 0x09, 0x3e, 0x03, 0x38, 0x8a, 0xb3, 0xb0, 0x7e, 0x3b, 0x70, 0x99, 0xcf, 0xb0,
0xc3, 0x78, 0x7b, 0x30, 0xa4, 0x4c, 0x9f, 0x87, 0x79, 0x5b, 0x2d, 0xfb, 0x43, 0xb5, 0xde, 0x57,
0x06, 0xd3, 0x97, 0x95, 0x28, 0xd6, 0x28, 0xde, 0x56, 0xa2, 0x54, 0x53, 0x51, 0xb8, 0xae, 0x52,
0x03, 0xbe, 0x0b, 0x83, 0x55, 0x1c, 0x5d, 0x08, 0xdd, 0x3b, 0x1b, 0x6b, 0x44, 0xb5, 0xb6, 0x3d,
0x2f, 0x55, 0xad, 0x23, 0xec, 0x52, 0xf4, 0x12, 0x45, 0x92, 0x49, 0x53, 0x4c, 0x8d, 0xb8, 0x07,
0xd3, 0xc3, 0xeb, 0x8b, 0xb8, 0xba, 0x14, 0xfa, 0xe9, 0x40, 0xa9, 0x1b, 0x1c, 0x45, 0xaf, 0xb1,
0x72, 0xfc, 0x50, 0x47, 0xef, 0x50, 0xde, 0x7b, 0x06, 0x5b, 0x75, 0xfa, 0x65, 0x9e, 0xa5, 0xa5,
0xa0, 0x19, 0x1d, 0x16, 0x85, 0x99, 0xd1, 0x61, 0x51, 0xf0, 0x3d, 0x18, 0xa2, 0x28, 0xab, 0x58,
0x9a, 0xc1, 0xdf, 0x6b, 0x5b, 0x61, 0xde, 0x56, 0xb1, 0x44, 0x73, 0x8b, 0x3f, 0x85, 0xed, 0x0d,
0x23, 0xe9, 0x8d, 0x99, 0x2c, 0xee, 0xb7, 0xef, 0x36, 0x74, 0xfc, 0xed, 0xba, 0xf7, 0x85, 0xc1,
0xa4, 0x13, 0x99, 0xfb, 0x66, 0x79, 0x55, 0x5a, 0x93, 0xc5, 0x4e, 0x1b, 0x48, 0xf3, 0x68, 0x96,
0x7b, 0x0a, 0xec, 0xa4, 0x36, 0x13, 0x3b, 0xa1, 0x11, 0xd2, 0x72, 0x9a, 0xef, 0x77, 0x46, 0x48,
0x34, 0x6a, 0x91, 0x3b, 0x30, 0xdc, 0x7f, 0x13, 0xa6, 0x57, 0xe2, 0x52, 0x99, 0x69, 0x84, 0x06,
0xf2, 0x79, 0xbb, 0x9c, 0xaa, 0xfb, 0x93, 0x05, 0x6f, 0x43, 0x18, 0x05, 0x9b, 0x3b, 0xde, 0x27,
0x06, 0x5b, 0x41, 0x92, 0x67, 0x85, 0xec, 0xb8, 0x21, 0x48, 0x2f, 0xc5, 0xb5, 0x71, 0x83, 0x02,
0xc4, 0x1e, 0x15, 0x61, 0xa2, 0x6d, 0x3f, 0x46, 0x0d, 0x88, 0x55, 0xae, 0x50, 0x2e, 0xb0, 0x51,
0x03, 0x35, 0x7f, 0x5a, 0xec, 0xd2, 0xb1, 0xb5, 0x73, 0x34, 0x22, 0x9f, 0x9b, 0xbd, 0x2e, 0x9d,
0xbe, 0x92, 0x5a, 0x82, 0x7c, 0xde, 0x2c, 0x36, 0x79, 0xc3, 0xf2, 0x2d, 0xec, 0x30, 0xde, 0x47,
0x06, 0x5c, 0x67, 0xaa, 0x7c, 0xff, 0xff, 0xd2, 0xa5, 0xbb, 0x91, 0x88, 0x75, 0x2b, 0xe9, 0x2e,
0x81, 0xbf, 0x24, 0xbb, 0x0b, 0x03, 0x95, 0x85, 0x49, 0xb4, 0x46, 0xcb, 0x9d, 0x6f, 0x37, 0x33,
0xf6, 0xfd, 0x66, 0xc6, 0x7e, 0xdc, 0xcc, 0xd8, 0x87, 0x9f, 0xb3, 0x3b, 0xe7, 0x03, 0xf5, 0x5b,
0x7f, 0xfc, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x0f, 0xf2, 0x1f, 0x86, 0xe6, 0x05, 0x00, 0x00,
}

5
internal/public.proto
View file

@ -4,11 +4,13 @@ package internal;
message Bitmap {
repeated uint64 Bits = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
}
message Pair {
uint64 Key = 1;
uint64 ID = 1;
string Key = 3;
uint64 Count = 2;
}
@ -25,6 +27,7 @@ message Bit {
message ColumnAttrSet {
uint64 ID = 1;
string Key = 3;
repeated Attr Attrs = 2;
}

57
roaring/internal_test.go
View file

@ -1,57 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package roaring
import (
"reflect"
"testing"
)
// Ensure iterator returns values from a bitmap.
func TestBitmapIterator(t *testing.T) {
for i, tt := range []struct {
bitmap []uint64
values []uint16
}{
// Empty
{
bitmap: []uint64{6}, // 0110
values: []uint16{1, 2},
},
// Single uint64 bitmap
{
bitmap: []uint64{6}, // 0110
values: []uint16{1, 2},
},
// Multi uint64 bitmap
{
bitmap: []uint64{1 << 63, 1, 0, 1, 3 << 62},
values: []uint16{63, 64, 192, 318, 319},
},
} {
itr := newBitmapIterator(tt.bitmap)
var a []uint16
for v, eof := itr.next(); !eof; v, eof = itr.next() {
a = append(a, v)
}
if !reflect.DeepEqual(a, tt.values) {
t.Errorf("%d. unexpected values: exp=%+v, got=%+v", i, a, tt.values)
}
}
}

567
roaring/roaring.go
View file

File diff suppressed because it is too large Load diff

261
roaring/roaring_internal_test.go
View file

@ -27,11 +27,11 @@ func (iv interval16) String() string {
}
func (c *container) String() string {
return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.container_type)
return fmt.Sprintf("<%s container n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.containerType)
}
func TestRunAppendInterval(t *testing.T) {
a := container{container_type: ContainerRun}
a := container{containerType: ContainerRun}
tests := []struct {
base []interval16
app interval16
@ -80,7 +80,7 @@ func TestInterval16RunLen(t *testing.T) {
}
func TestContainerRunAdd(t *testing.T) {
c := container{runs: make([]interval16, 0), container_type: ContainerRun}
c := container{runs: make([]interval16, 0), containerType: ContainerRun}
tests := []struct {
op uint16
exp []interval16
@ -111,7 +111,7 @@ func TestContainerRunAdd(t *testing.T) {
}
func TestContainerRunAdd2(t *testing.T) {
c := container{runs: make([]interval16, 0), container_type: ContainerRun}
c := container{runs: make([]interval16, 0), containerType: ContainerRun}
ret := c.add(0)
if !ret {
t.Fatalf("result of adding new bit should be true: %v", c.runs)
@ -126,7 +126,7 @@ func TestContainerRunAdd2(t *testing.T) {
}
func TestRunCountRange(t *testing.T) {
c := container{runs: make([]interval16, 0), container_type: ContainerRun}
c := container{runs: make([]interval16, 0), containerType: ContainerRun}
cnt := c.runCountRange(2, 9)
if cnt != 0 {
t.Fatalf("should get 0 from empty container, but got: %v", cnt)
@ -179,7 +179,7 @@ func TestRunCountRange(t *testing.T) {
}
func TestRunContains(t *testing.T) {
c := container{runs: make([]interval16, 0), container_type: ContainerRun}
c := container{runs: make([]interval16, 0), containerType: ContainerRun}
if c.runContains(5) {
t.Fatalf("empty run container should not contain 5")
}
@ -201,7 +201,7 @@ func TestRunContains(t *testing.T) {
}
func TestBitmapCountRange(t *testing.T) {
c := container{container_type: ContainerBitmap}
c := container{containerType: ContainerBitmap}
tests := []struct {
start int
end int
@ -227,11 +227,11 @@ func TestBitmapCountRange(t *testing.T) {
func TestIntersectionCountArrayBitmap3(t *testing.T) {
a, b := &container{}, &container{}
a.container_type = ContainerBitmap
a.containerType = ContainerBitmap
a.bitmap = getFullBitmap()
a.n = maxContainerVal + 1
b.container_type = ContainerBitmap
b.containerType = ContainerBitmap
b.bitmap = getFullBitmap()
b.n = maxContainerVal + 1
res := intersectBitmapBitmap(a, b)
@ -288,9 +288,9 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
for i, test := range tests {
a.array = test.array
a.container_type = ContainerArray
a.containerType = ContainerArray
b.bitmap = test.bitmap
b.container_type = ContainerBitmap
b.containerType = ContainerBitmap
ret := intersectionCountArrayBitmap(a, b)
if ret != test.exp {
t.Fatalf("test #%v intersectCountArrayBitmap fail received: %v exp: %v", i, ret, test.exp)
@ -299,7 +299,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
}
func TestRunRemove(t *testing.T) {
c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
tests := []struct {
op uint16
exp []interval16
@ -333,7 +333,7 @@ func TestRunRemove(t *testing.T) {
}
func TestRunMax(t *testing.T) {
c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun}
max := c.max()
if max != 16 {
t.Fatalf("max for %v should be 16", c.runs)
@ -347,8 +347,8 @@ func TestRunMax(t *testing.T) {
}
func TestIntersectionCountArrayRun(t *testing.T) {
a := &container{container_type: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
b := &container{container_type: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
a := &container{containerType: ContainerArray, array: []uint16{1, 5, 10, 11, 12}}
b := &container{containerType: ContainerRun, runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
ret := intersectionCountArrayRun(a, b)
if ret != 3 {
@ -357,16 +357,16 @@ func TestIntersectionCountArrayRun(t *testing.T) {
}
func TestIntersectionCountBitmapRun(t *testing.T) {
a := &container{container_type: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
b := &container{container_type: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
a := &container{containerType: ContainerBitmap, bitmap: []uint64{0x8000000000000000}}
b := &container{containerType: ContainerRun, runs: []interval16{{start: 63, last: 64}}}
ret := intersectionCountBitmapRun(a, b)
if ret != 1 {
t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret)
}
a = &container{container_type: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
b = &container{container_type: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
a = &container{containerType: ContainerBitmap, bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
b = &container{containerType: ContainerRun, runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
ret = intersectionCountBitmapRun(a, b)
if ret != 14 {
@ -414,8 +414,8 @@ func TestIntersectionCountRunRun(t *testing.T) {
bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
}
for i, test := range tests {
a.container_type = ContainerRun
b.container_type = ContainerRun
a.containerType = ContainerRun
b.containerType = ContainerRun
a.runs = test.aruns
b.runs = test.bruns
ret := intersectionCountRunRun(a, b)
@ -456,8 +456,8 @@ func TestIntersectArrayRun(t *testing.T) {
}
for i, test := range tests {
a.container_type = ContainerArray
b.container_type = ContainerRun
a.containerType = ContainerArray
b.containerType = ContainerRun
a.array = test.array
b.runs = test.runs
ret := intersectArrayRun(a, b)
@ -514,8 +514,8 @@ func TestIntersectRunRun(t *testing.T) {
},
}
for i, test := range tests {
a.container_type = ContainerRun
b.container_type = ContainerRun
a.containerType = ContainerRun
b.containerType = ContainerRun
a.runs = test.aruns
b.runs = test.bruns
ret := intersectRunRun(a, b)
@ -579,8 +579,8 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
for i, v := range test.exp {
exp[i] = v
}
a.container_type = ContainerBitmap
b.container_type = ContainerRun
a.containerType = ContainerBitmap
b.containerType = ContainerRun
ret := intersectBitmapRun(a, b)
if ret.isArray() {
ret.arrayToBitmap()
@ -640,8 +640,8 @@ func TestIntersectBitmapRunArray(t *testing.T) {
a.bitmap[i] = v
}
b.runs = test.runs
a.container_type = ContainerBitmap
b.container_type = ContainerRun
a.containerType = ContainerBitmap
b.containerType = ContainerRun
ret := intersectBitmapRun(a, b)
if !reflect.DeepEqual(ret.array, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@ -658,19 +658,19 @@ func TestUnionMixed(t *testing.T) {
// array container
a := &container{}
a.array = []uint16{1, 4, 5, 7, 10, 11, 12}
a.container_type = ContainerArray
a.containerType = ContainerArray
a.n = 7
// bitmap container
b := &container{bitmap: make([]uint64, bitmapN)}
b.bitmap[0] = uint64(0x3)
b.n = 2
b.container_type = ContainerBitmap
b.containerType = ContainerBitmap
// run container
r := &container{}
r.runs = []interval16{{start: 5, last: 10}}
r.container_type = ContainerRun
r.containerType = ContainerRun
r.n = 6
t.Run("various container Unions", func(t *testing.T) {
@ -711,10 +711,10 @@ func TestIntersectMixed(t *testing.T) {
a.runs = []interval16{{start: 5, last: 10}}
a.n = 6
a.container_type = ContainerRun
a.containerType = ContainerRun
b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
b.n = 7
b.container_type = ContainerArray
b.containerType = ContainerArray
res := intersect(a, b)
if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) {
t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array)
@ -730,7 +730,7 @@ func TestIntersectMixed(t *testing.T) {
}
c.bitmap = []uint64{0x60}
c.n = 2
c.container_type = ContainerBitmap
c.containerType = ContainerBitmap
res = intersect(c, a)
if !reflect.DeepEqual(res.array, []uint16{5, 6}) {
@ -760,15 +760,15 @@ func TestDifferenceMixed(t *testing.T) {
a.runs = []interval16{{start: 5, last: 10}}
a.n = a.runCountRange(0, 100)
a.container_type = ContainerRun
a.containerType = ContainerRun
b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
b.n = len(b.array)
b.container_type = ContainerArray
b.containerType = ContainerArray
d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
d.n = len(d.array)
d.container_type = ContainerArray
d.containerType = ContainerArray
res := difference(a, b)
@ -788,7 +788,7 @@ func TestDifferenceMixed(t *testing.T) {
c.bitmap = []uint64{0x64}
c.n = c.countRange(0, 100)
c.container_type = ContainerBitmap
c.containerType = ContainerBitmap
res = difference(c, a)
if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
@ -883,8 +883,8 @@ func TestUnionRunRun(t *testing.T) {
for i, test := range tests {
a.runs = test.aruns
b.runs = test.bruns
a.container_type = ContainerRun
b.container_type = ContainerRun
a.containerType = ContainerRun
b.containerType = ContainerRun
ret := unionRunRun(a, b)
if !reflect.DeepEqual(ret.runs, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@ -925,8 +925,8 @@ func TestUnionArrayRun(t *testing.T) {
for i, test := range tests {
a.array = test.array
b.runs = test.runs
a.container_type = ContainerArray
b.container_type = ContainerRun
a.containerType = ContainerArray
b.containerType = ContainerRun
ret := unionArrayRun(a, b)
if !reflect.DeepEqual(ret.array, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@ -935,7 +935,7 @@ func TestUnionArrayRun(t *testing.T) {
}
func TestBitmapSetRange(t *testing.T) {
c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@ -975,7 +975,7 @@ func TestBitmapSetRange(t *testing.T) {
}
func TestArrayToBitmap(t *testing.T) {
a := &container{container_type: ContainerArray}
a := &container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp []uint64
@ -1006,7 +1006,7 @@ func TestArrayToBitmap(t *testing.T) {
}
func TestBitmapToArray(t *testing.T) {
a := &container{container_type: ContainerBitmap}
a := &container{containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
exp []uint16
@ -1037,7 +1037,7 @@ func TestBitmapToArray(t *testing.T) {
}
func TestRunToBitmap(t *testing.T) {
a := &container{container_type: ContainerRun}
a := &container{containerType: ContainerRun}
tests := []struct {
runs []interval16
exp []uint64
@ -1091,7 +1091,7 @@ func getFullBitmap() []uint64 {
}
func TestBitmapToRun(t *testing.T) {
a := &container{container_type: ContainerBitmap}
a := &container{containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
exp []interval16
@ -1169,7 +1169,7 @@ func TestBitmapToRun(t *testing.T) {
}
func TestArrayToRun(t *testing.T) {
a := &container{container_type: ContainerArray}
a := &container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp []interval16
@ -1203,7 +1203,7 @@ func TestArrayToRun(t *testing.T) {
}
func TestRunToArray(t *testing.T) {
a := &container{container_type: ContainerRun}
a := &container{containerType: ContainerRun}
tests := []struct {
runs []interval16
exp []uint16
@ -1237,7 +1237,7 @@ func TestRunToArray(t *testing.T) {
}
func TestBitmapZeroRange(t *testing.T) {
c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
start uint64
@ -1273,7 +1273,7 @@ func TestBitmapZeroRange(t *testing.T) {
if test.expN != c.n {
t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n)
}
for i, _ := range test.bitmap {
for i := range test.bitmap {
c.bitmap[i] = 0
}
}
@ -1281,8 +1281,8 @@ func TestBitmapZeroRange(t *testing.T) {
}
func TestUnionBitmapRun(t *testing.T) {
a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -1313,14 +1313,14 @@ func TestUnionBitmapRun(t *testing.T) {
if ret.n != test.expN {
t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
}
for i, _ := range test.bitmap {
for i := range test.bitmap {
a.bitmap[i] = 0
}
}
}
func TestBitmapCountRuns(t *testing.T) {
c := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
c := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
bitmap []uint64
exp int
@ -1353,7 +1353,7 @@ func TestBitmapCountRuns(t *testing.T) {
t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret)
}
for j, _ := range test.bitmap {
for j := range test.bitmap {
c.bitmap[j] = 0
}
}
@ -1370,7 +1370,7 @@ func TestBitmapCountRuns(t *testing.T) {
}
func TestArrayCountRuns(t *testing.T) {
c := &container{container_type: ContainerArray}
c := &container{containerType: ContainerArray}
tests := []struct {
array []uint16
exp int
@ -1411,8 +1411,8 @@ func TestArrayCountRuns(t *testing.T) {
}
func TestDifferenceArrayRun(t *testing.T) {
a := &container{container_type: ContainerArray}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerArray}
b := &container{containerType: ContainerRun}
tests := []struct {
array []uint16
runs []interval16
@ -1437,8 +1437,8 @@ func TestDifferenceArrayRun(t *testing.T) {
}
func TestDifferenceRunArray(t *testing.T) {
a := &container{container_type: ContainerRun}
b := &container{container_type: ContainerArray}
a := &container{containerType: ContainerRun}
b := &container{containerType: ContainerArray}
tests := []struct {
runs []interval16
array []uint16
@ -1508,8 +1508,8 @@ func MakeLastBitSet() []uint64 {
}
func TestDifferenceRunBitmap(t *testing.T) {
a := &container{container_type: ContainerRun}
b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
a := &container{containerType: ContainerRun}
b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
tests := []struct {
runs []interval16
bitmap []uint64
@ -1571,8 +1571,8 @@ func TestDifferenceRunBitmap(t *testing.T) {
}
func TestDifferenceBitmapRun(t *testing.T) {
a := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
b := &container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -1599,8 +1599,8 @@ func TestDifferenceBitmapRun(t *testing.T) {
}
func TestDifferenceBitmapArray(t *testing.T) {
b := &container{container_type: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
a := &container{container_type: ContainerArray}
b := &container{containerType: ContainerBitmap, bitmap: make([]uint64, bitmapN)}
a := &container{containerType: ContainerArray}
tests := []struct {
bitmap []uint64
array []uint16
@ -1649,8 +1649,8 @@ func TestDifferenceBitmapArray(t *testing.T) {
}
func TestDifferenceBitmapBitmap(t *testing.T) {
a := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
a := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
b := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
tests := []struct {
abitmap []uint64
bbitmap []uint64
@ -1679,8 +1679,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
}
func TestDifferenceRunRun(t *testing.T) {
a := &container{container_type: ContainerRun}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerRun}
b := &container{containerType: ContainerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@ -1713,7 +1713,7 @@ func TestDifferenceRunRun(t *testing.T) {
}
func TestWriteReadArray(t *testing.T) {
ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, container_type: ContainerArray}
ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, containerType: ContainerArray}
ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}
ba2 := &Bitmap{}
var buf bytes.Buffer
@ -1732,7 +1732,7 @@ func TestWriteReadArray(t *testing.T) {
func TestWriteReadBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, container_type: ContainerBitmap}
cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, containerType: ContainerBitmap}
for i := 0; i < 129; i++ {
cb.bitmap[i] = 0x5555555555555555
}
@ -1754,7 +1754,7 @@ func TestWriteReadBitmap(t *testing.T) {
func TestWriteReadFullBitmap(t *testing.T) {
// create bitmap containing > 4096 bits
cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, container_type: ContainerBitmap}
cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, containerType: ContainerBitmap}
for i := 0; i < bitmapN; i++ {
cb.bitmap[i] = 0xffffffffffffffff
}
@ -1782,7 +1782,7 @@ func TestWriteReadFullBitmap(t *testing.T) {
}
func TestWriteReadRun(t *testing.T) {
cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, container_type: ContainerRun}
cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, containerType: ContainerRun}
br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
br2 := &Bitmap{}
var buf bytes.Buffer
@ -1806,21 +1806,21 @@ func TestXorArrayRun(t *testing.T) {
exp *container
}{
{
a: &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, container_type: ContainerArray, n: 12},
a: &container{array: []uint16{1, 5, 10, 11, 12}, containerType: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, containerType: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, container_type: ContainerArray, n: 12},
a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, containerType: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, containerType: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, containerType: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65534, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{65534}, container_type: ContainerArray, n: 1},
a: &container{array: []uint16{65535}, containerType: ContainerArray},
b: &container{runs: []interval16{{start: 65534, last: 65535}}, containerType: ContainerRun},
exp: &container{array: []uint16{65534}, containerType: ContainerArray, n: 1},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65535, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{}, container_type: ContainerArray, n: 0},
a: &container{array: []uint16{65535}, containerType: ContainerArray},
b: &container{runs: []interval16{{start: 65535, last: 65535}}, containerType: ContainerRun},
exp: &container{array: []uint16{}, containerType: ContainerArray, n: 0},
},
}
@ -1841,8 +1841,8 @@ func TestXorArrayRun(t *testing.T) {
//special case that didn't fit the xorrunrun table testing below.
func TestXorRunRun1(t *testing.T) {
a := &container{container_type: ContainerRun}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerRun}
b := &container{containerType: ContainerRun}
a.runs = []interval16{{start: 4, last: 10}}
b.runs = []interval16{{start: 5, last: 10}}
ret := xorRunRun(a, b)
@ -1856,8 +1856,8 @@ func TestXorRunRun1(t *testing.T) {
}
func TestXorRunRun(t *testing.T) {
a := &container{container_type: ContainerRun}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerRun}
b := &container{containerType: ContainerRun}
tests := []struct {
aruns []interval16
bruns []interval16
@ -1954,7 +1954,7 @@ func TestXorRunRun(t *testing.T) {
}
func TestBitmapFlip(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
ttable := []struct {
original uint64
@ -1986,7 +1986,7 @@ func TestBitmapFlip(t *testing.T) {
}
func TestBitmapXorRange(t *testing.T) {
c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap}
tests := []struct {
bitmap []uint64
start uint64
@ -2054,8 +2054,8 @@ func TestBitmapXorRange(t *testing.T) {
}
func TestXorBitmapRun(t *testing.T) {
a := &container{container_type: ContainerBitmap}
b := &container{container_type: ContainerRun}
a := &container{containerType: ContainerBitmap}
b := &container{containerType: ContainerRun}
tests := []struct {
bitmap []uint64
runs []interval16
@ -2390,65 +2390,6 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) {
}
}
func Test_BufBitmapIterator_Next(t *testing.T) {
b := NewBitmap()
for i := uint64(0); i < 4097; i++ {
b.Add(i)
}
if !b.containers[0].isBitmap() {
t.Fatalf("wrong container type")
}
bin := []uint16{}
itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap))
x := uint16(0)
for i := 0; i < 10; i++ {
x, _ = itr.next()
bin = append(bin, x)
}
exp := []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
if !reflect.DeepEqual(bin, exp) {
t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
}
// ensure that unread points next back one such that the last value is repeated
itr.unread()
x, _ = itr.next()
bin = append(bin, x)
exp = append(exp, uint16(9))
if !reflect.DeepEqual(bin, exp) {
t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
}
}
func Test_BufBitmapIterator_UnreadPanic(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Errorf("BufBitmapIterator unread did not panic")
}
}()
b := NewBitmap()
for i := uint64(0); i < 4097; i++ {
b.Add(i)
}
if !b.containers[0].isBitmap() {
t.Fatalf("wrong container type")
}
itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap))
for i := 0; i < 10; i++ {
itr.next()
}
// ensure that unreading back-to-back panics
itr.unread()
itr.unread()
}
func TestSearc64(t *testing.T) {
tests := []struct {
a []uint64
@ -2528,9 +2469,9 @@ func TestSearc64(t *testing.T) {
}
func TestIntersectArrayBitmap(t *testing.T) {
a, b := &container{container_type: ContainerArray}, &container{
container_type: ContainerBitmap,
bitmap: make([]uint64, bitmapN),
a, b := &container{containerType: ContainerArray}, &container{
containerType: ContainerBitmap,
bitmap: make([]uint64, bitmapN),
}
tests := []struct {
array []uint16
@ -2576,11 +2517,11 @@ func TestIntersectArrayBitmap(t *testing.T) {
for i, test := range tests {
a.array = test.array
a.container_type = ContainerArray
a.containerType = ContainerArray
for i, bmval := range test.bitmap {
b.bitmap[i] = bmval
}
b.container_type = ContainerBitmap
b.containerType = ContainerBitmap
ret := intersectArrayBitmap(a, b).array
if len(ret) == 0 && len(test.exp) == 0 {
continue

132
roaring/roaring_test.go
View file

@ -1059,32 +1059,34 @@ func TestBitmapBufIterator(t *testing.T) {
}
var benchmarkBitmapIntersectionCountData struct {
a, b *roaring.Bitmap
a, b, r *roaring.Bitmap
}
func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) {
func getBenchData() *struct{ a, b, r *roaring.Bitmap } {
data := &benchmarkBitmapIntersectionCountData
if data.a == nil {
const max = (1 << 24) / 64
// Build bitmap with array container.
data.a = roaring.NewBitmap()
for i, n := 0, rand.Intn(roaring.ArrayMaxSize); i < n; i++ {
for i, n := 0, 2*roaring.ArrayMaxSize/3; i < n; i++ {
data.a.Add(uint64(rand.Intn(max)))
}
// Build bitmap with bitmap container.
data.b = roaring.NewBitmap()
for i, n := 0, roaring.ArrayMaxSize*2; i < n; i++ {
for i, n := 0, MaxContainerVal/3; i < n; i++ {
data.b.Add(uint64(i * 3))
}
}
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a.IntersectionCount(data.b)
// build bitmap with run container
data.r = roaring.NewBitmap()
for i, n := 0, MaxContainerVal; i < n; i++ {
data.r.Add(uint64(i))
}
}
return data
}
// GenerateUint64Slice generates between [0, n) random uint64 numbers between min and max.
@ -1137,3 +1139,115 @@ func TestBitmap_Intersect(t *testing.T) {
t.Fatalf("Counts do not match %d %d", bm0.Count(), result.Count())
}
}
func BenchmarkBitmap_IntersectionCount_ArrayRun(b *testing.B) {
data := getBenchData()
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a.IntersectionCount(data.r)
}
}
func BenchmarkBitmap_IntersectionCount_BitmapRun(b *testing.B) {
data := getBenchData()
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.b.IntersectionCount(data.r)
}
}
func BenchmarkBitmap_IntersectionCount_ArrayBitmap(b *testing.B) {
data := getBenchData()
// Reset timer & benchmark.
b.ResetTimer()
for i := 0; i < b.N; i++ {
data.a.IntersectionCount(data.b)
}
}
const (
NumRows = uint64(10000)
NumColums = uint64(16)
MaxContainerVal = 0xffff
)
func BenchmarkContainerLinear(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for row := uint64(1); row < NumRows; row++ {
for col := uint64(1); col < NumColums; col++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
}
}
}
}
func BenchmarkContainerReverse(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for row := NumRows - 1; row >= 1; row-- {
for col := NumColums - 1; col >= 1; col-- {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
}
}
}
}
func BenchmarkContainerColumn(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < NumRows; row++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
}
}
}
}
func BenchmarkContainerOutsideIn(b *testing.B) {
middle := NumRows / uint64(2)
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row < middle; row++ {
bm.Add(row*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add((NumRows-row)*pilosa.SliceWidth + (col * MaxContainerVal))
}
}
}
}
func BenchmarkContainerInsideOut(b *testing.B) {
middle := NumRows / uint64(2)
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for col := uint64(1); col < NumColums; col++ {
for row := uint64(1); row <= middle; row++ {
bm.Add((middle+row)*pilosa.SliceWidth + (col * MaxContainerVal))
bm.Add((middle-row)*pilosa.SliceWidth + (col * MaxContainerVal))
}
}
}
}
func BenchmarkSliceAscending(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for col := uint64(0); col < pilosa.SliceWidth; col++ {
bm.Add(col)
}
}
}
func BenchmarkSliceDescending(b *testing.B) {
for n := 0; n < b.N; n++ {
bm := roaring.NewBitmap()
for col := uint64(pilosa.SliceWidth); col > uint64(0); col-- {
bm.Add(col)
}
}
}

63
server.go
View file

@ -88,6 +88,7 @@ type Server struct {
MaxWritesPerRequest int
LogOutput io.Writer
logger *log.Logger
defaultClient InternalClient
}
@ -111,9 +112,9 @@ func NewServer() *Server {
LogOutput: os.Stderr,
}
s.logger = log.New(s.LogOutput, "", log.LstdFlags)
s.Handler.Holder = s.Holder
return s
}
@ -313,7 +314,7 @@ func GetHTTPClient(t *tls.Config) *http.Client {
}
// Logger returns a logger that writes to LogOutput
func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
func (s *Server) Logger() *log.Logger { return s.logger }
func (s *Server) monitorAntiEntropy() {
ticker := time.NewTicker(s.AntiEntropyInterval)
@ -339,6 +340,7 @@ func (s *Server) monitorAntiEntropy() {
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.RemoteClient
syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
// Sync holders.
if err := syncer.SyncHolder(); err != nil {
@ -588,32 +590,17 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.Holder.NodeID)
s.diagnostics.Set("ClusterID", s.Cluster.ID)
s.diagnostics.EnrichWithOSInfo()
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
numFrames := 0
numSlices := uint64(0)
for _, index := range s.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
for _, f := range index.Frames() {
numFrames++
if f.rangeEnabled {
s.diagnostics.Set("BSIEnabled", true)
}
if f.timeQuantum != "" {
s.diagnostics.Set("TimeQuantumEnabled", true)
}
}
}
s.diagnostics.Set("NumIndexes", len(s.Holder.Indexes()))
s.diagnostics.Set("NumFrames", numFrames)
s.diagnostics.Set("NumSlices", numSlices)
enrichDiagnosticsWithSchemaProperties(s.diagnostics, s.Holder)
openFiles, err := CountOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
s.diagnostics.Set("GoRoutines", runtime.NumGoroutine())
s.diagnostics.EnrichWithMemoryInfo()
s.diagnostics.CheckVersion()
s.diagnostics.Flush()
}
@ -711,3 +698,39 @@ type StatusHandler interface {
ClusterStatus() (proto.Message, error)
HandleRemoteStatus(proto.Message) error
}
type diagnosticsFrameProperties struct {
BSIFieldCount int
TimeQuantumEnabled bool
}
func enrichDiagnosticsWithSchemaProperties(d *diagnostics.Diagnostics, holder *Holder) {
// NOTE: this function is not in the diagnostics package, since circular imports are not allowed.
var numSlices uint64
numFrames := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
for _, index := range holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if frame.rangeEnabled {
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
}
}
if frame.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)
}

38
server/server_test.go
View file

@ -277,24 +277,17 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
m0 := MustRunMain()
defer m0.Close()
m1 := MustRunMain()
defer m1.Close()
// Update cluster config.
m0.Server.Cluster.Nodes = []*pilosa.Node{
{URI: m0.Server.URI},
{URI: m1.Server.URI},
}
m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
// TODO: this test used to start a two node cluster, but there was a race
// condition with anti-entropy. We need some general code for starting up
// arbitrarily sized Pilosa clusters for testing, and then we should
// re-instate the multi-node nature of this test.
// Create frames.
client := m0.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
t.Fatal("create index:", err)
} else if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
t.Fatal("create frame:", err)
}
// Write data on first cluster.
@ -307,12 +300,12 @@ func TestMain_FrameRestore(t *testing.T) {
SetBit(rowID=1, frame="f", columnID=600000)
SetBit(rowID=1, frame="f", columnID=800000)
`); err != nil {
t.Fatal(err)
t.Fatal("setting bits:", err)
}
// Query row on first cluster.
if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal(err)
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -324,20 +317,20 @@ func TestMain_FrameRestore(t *testing.T) {
// Import from first cluster.
client, err := pilosa.NewInternalHTTPClient(m2.Server.URI.HostPort(), pilosa.GetHTTPClient(nil))
if err != nil {
t.Fatal(err)
t.Fatal("new client:", err)
} else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
t.Fatal("create new index:", err)
} else if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
t.Fatal("create new frame:", err)
} else if err := client.RestoreFrame(context.Background(), m0.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal(err)
t.Fatal("restore frame:", err)
}
// Query row on second cluster.
if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal(err)
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
t.Fatalf("2unexpected result: %s", res)
}
}
@ -419,6 +412,7 @@ func NewMain() *Main {
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Run(); err != nil {
panic(err)
}
@ -443,8 +437,6 @@ func (m *Main) Reopen() error {
m.Server.Network = *test.Network
m.Config = config
println("dbg/network", *test.Network)
// Run new program.
if err := m.Run(); err != nil {
return err

250
test/pilosa.go Normal file
View file

@ -0,0 +1,250 @@
package test
import (
"bytes"
"io"
"io/ioutil"
"net"
"os"
"strconv"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
////////////////////////////////////////////////////////////////////////////////////
// Main represents a test wrapper for main.Main.
type Main struct {
*server.Command
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain() *Main {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)}
m.Server.Network = *Network
m.Config.DataDir = path
m.Config.Bind = "localhost:0"
m.Config.Cluster.Type = "static"
m.Command.Stdin = &m.Stdin
m.Command.Stdout = &m.Stdout
m.Command.Stderr = &m.Stderr
if testing.Verbose() {
m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout)
m.Command.Stderr = io.MultiWriter(os.Stderr, m.Command.Stderr)
}
return m
}
/*
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Run(); err != nil {
panic(err)
}
return m
}
*/
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// RunWithTransport runs Main and returns the dynamically allocated gossip port.
func (m *Main) RunWithTransport(host string, bindPort int, joinSeed string, coordinator *pilosa.URI) (seed string, coord pilosa.URI, err error) {
defer close(m.Started)
m.Config.Cluster.Type = "gossip"
/*
TEST:
- SetupServer (just static settings from config)
- OpenListener (sets Server.Name to use in gossip)
- NewTransport (gossip)
- SetupNetworking (does the gossip or static stuff) - uses Server.Name
- Open server
PRODUCTION:
- SetupServer (just static settings from config)
- SetupNetworking (does the gossip or static stuff) - calls NewTransport
- Open server - calls OpenListener
*/
// SetupServer
err = m.SetupServer()
if err != nil {
return seed, coord, err
}
// Open server listener.
// This is used to set Server.Name, which is used as the node
// name for identifying a memberlist node.
err = m.Server.OpenListener()
if err != nil {
return seed, coord, err
}
// Open gossip transport to use in SetupServer.
transport, err := gossip.NewTransport(host, bindPort)
if err != nil {
return seed, coord, err
}
m.GossipTransport = transport
if joinSeed != "" {
m.Config.Gossip.Seed = joinSeed
} else {
m.Config.Gossip.Seed = transport.URI.String()
}
seed = m.Config.Gossip.Seed
// SetupNetworking
err = m.SetupNetworking()
if err != nil {
return seed, coord, err
}
if err = m.Server.BroadcastReceiver.Start(m.Server); err != nil {
return seed, coord, err
}
if coordinator != nil {
coord = *coordinator
} else {
coord = m.Server.URI
}
m.Server.Cluster.Coordinator = coord
m.Server.Cluster.Static = false
// Initialize server.
err = m.Server.Open()
if err != nil {
return seed, coord, err
}
return seed, coord, nil
}
////////////////////////////////////////////////////////////////////////////////////
func MustNewRunningServer(t *testing.T) *server.Command {
s, err := newServer()
if err != nil {
t.Fatalf("getting new server: %v", err)
}
err = s.Run()
if err != nil {
t.Fatalf("running new pilosa server: %v", err)
}
return s
}
func newServer() (*server.Command, error) {
s := server.NewCommand(&bytes.Buffer{}, ioutil.Discard, ioutil.Discard)
port, err := findPort()
if err != nil {
return nil, errors.Wrap(err, "getting port")
}
s.Config.Bind = "localhost:" + strconv.Itoa(port)
gport, err := findPort()
if err != nil {
return nil, errors.Wrap(err, "getting gossip port")
}
s.Config.GossipPort = strconv.Itoa(gport)
s.Config.GossipSeed = "localhost:" + s.Config.GossipPort
s.Config.Cluster.Type = "gossip"
s.Config.Metric.Diagnostics = false
td, err := ioutil.TempDir("", "")
if err != nil {
return nil, errors.Wrap(err, "temp dir")
}
s.Config.DataDir = td
return s, nil
}
func findPort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", ":0")
if err != nil {
return 0, errors.Wrap(err, "resolving new port addr")
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, errors.Wrap(err, "listening to get new port")
}
port := l.Addr().(*net.TCPAddr).Port
err = l.Close()
if err != nil {
return port, errors.Wrap(err, "closing listener")
}
return port, nil
}
func MustFindPort(t *testing.T) int {
port, err := findPort()
if err != nil {
t.Fatalf("allocating new port: %v", err)
}
return port
}
type Cluster struct {
Servers []*Main
}
func MustNewServerCluster(t *testing.T, size int) *Cluster {
cluster, err := NewServerCluster(size)
if err != nil {
t.Fatalf("new cluster: %v", err)
}
return cluster
}
func NewServerCluster(size int) (cluster *Cluster, err error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
cluster = &Cluster{
Servers: make([]*Main, size),
}
gossipHost := "localhost"
gossipPort := 0
var gossipSeed string
var coordinator pilosa.URI
for i := 0; i < size; i++ {
m := NewMain()
gossipSeed, coordinator, err = m.RunWithTransport(gossipHost, gossipPort, gossipSeed, &coordinator)
if err != nil {
return nil, errors.Wrap(err, "RunWithTransport")
}
cluster.Servers[i] = m
}
return cluster, nil
}

46
test/pilosa_test.go Normal file
View file

@ -0,0 +1,46 @@
package test_test
import (
"encoding/json"
"net/http"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestNewCluster(t *testing.T) {
cluster := test.MustNewServerCluster(t, 3)
response, err := http.Get("http://" + cluster.Servers[0].Server.Addr().String() + "/status")
if err != nil {
t.Fatalf("getting schema: %v", err)
}
dec := json.NewDecoder(response.Body)
body := struct {
State string
Nodes []struct {
Scheme string
Host string
Port int
}
}{}
err = dec.Decode(&body)
if err != nil {
t.Fatalf("decoding status response: %v", err)
}
bytes, err := json.MarshalIndent(body, "", " ")
if err != nil {
t.Fatalf("encoding: %v", err)
}
if len(body.Nodes) != 3 {
t.Fatalf("wrong number of nodes in status: %s", bytes)
}
if body.State != pilosa.ClusterStateNormal {
t.Fatalf("cluster state should be %s but is %s", pilosa.ClusterStateNormal, body.State)
}
}