From 8ceaef1eaaa96184dad0e84ed1860f0fcbd4cdfd Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 14:53:37 -0600 Subject: [PATCH 01/63] fragment cache size is set on frame creation with a default value of 50,000 --- fragment.go | 32 ++++++++++++++++++++++++++++---- fragment_test.go | 44 +++++++++++++++++++++++++++++++++++++++++--- frame.go | 47 ++++++++++++++++++++++++++++++++++++++++++----- view.go | 15 +++++++++------ view_test.go | 4 ++-- 5 files changed, 122 insertions(+), 20 deletions(-) diff --git a/fragment.go b/fragment.go index 063b7e17c..fe04682dc 100644 --- a/fragment.go +++ b/fragment.go @@ -70,6 +70,7 @@ type Fragment struct { // Cache for bitmap counts. cacheType string // passed in by frame cache Cache + cacheSize int // Cache containing full bitmaps (not just counts). bitmapCache BitmapCache @@ -93,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment { return &Fragment{ path: path, db: db, @@ -101,6 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64) *Fragment { view: view, slice: slice, cacheType: DefaultCacheType, + cacheSize: cacheSize, LogOutput: ioutil.Discard, MaxOpN: DefaultFragmentMaxOpN, @@ -217,21 +219,43 @@ func (f *Fragment) openStorage() error { } +// // It will be the one and only identifier after a package specifier. +// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`) + +// // Returns the name of the test function from the call stack. See +// // http://stackoverflow.com/q/35535635/149482 for another method. +// func GetTestName() string { +// pc := make([]uintptr, 32) +// n := runtime.Callers(0, pc) +// for i := 0; i < n; i++ { +// name := runtime.FuncForPC(pc[i]).Name() +// ms := testNameRegexp.FindStringSubmatch(name) +// if ms == nil { +// continue +// } +// return ms[1] +// } +// panic("test name could not be recovered") +// } + // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { case CacheTypeRanked: c := NewRankCache() - c.ThresholdLength = 50000 - c.ThresholdIndex = 45000 + c.ThresholdLength = f.cacheSize + c.ThresholdIndex = f.cacheSize - 500 f.cache = c case CacheTypeLRU: - f.cache = NewLRUCache(50000) + f.cache = NewLRUCache(f.cacheSize) default: return ErrInvalidCacheType } + // fmt.Println(GetTestName()) + // fmt.Printf("CACHE SIZE: %d\n", f.cacheSize) + // Read cache data from disk. path := f.CachePath() buf, err := ioutil.ReadFile(path) diff --git a/fragment_test.go b/fragment_test.go index b915bd449..ca6ffa3dc 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -277,6 +277,44 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { } } +// Ensure the fragment cache limit works +func TestFragment_TopN_CacheSize(t *testing.T) { + slice := uint64(0) + cacheLimit := 3 + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + f := &Fragment{ + Fragment: pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit), + BitmapAttrStore: MustOpenAttrStore(), + } + f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + if err := f.Open(); err != nil { + panic(err) + } + defer f.Close() + + // Set bits on various bitmaps. + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + f.MustSetBits(103, 8, 9, 10, 11, 12, 13) + f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.MustSetBits(105, 10, 11) + + // Retrieve top bitmaps. + if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + t.Fatal(err) + } else if len(pairs) != cacheLimit { + t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) + } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { + t.Fatalf("unexpected pair(0): %v", pairs[0]) + } +} + // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) @@ -526,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache) if err := f.Open(); err != nil { b.Fatal(err) } @@ -587,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -618,7 +656,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index c70a07b68..be949b19d 100644 --- a/frame.go +++ b/frame.go @@ -20,6 +20,9 @@ const ( DefaultRowLabel = "id" DefaultCacheType = CacheTypeLRU DefaultInverseEnabled = false + + // Default ranked frame cache + DefaultFrameCache = 50000 ) // Frame represents a container for views. @@ -42,6 +45,9 @@ type Frame struct { cacheType string inverseEnabled bool + // Cache size for ranked frames + rankedCacheSize int + LogOutput io.Writer } @@ -62,9 +68,10 @@ func NewFrame(path, db, name string) (*Frame, error) { stats: NopStatsClient, - rowLabel: DefaultRowLabel, - cacheType: DefaultCacheType, - inverseEnabled: DefaultInverseEnabled, + rowLabel: DefaultRowLabel, + cacheType: DefaultCacheType, + inverseEnabled: DefaultInverseEnabled, + rankedCacheSize: DefaultFrameCache, LogOutput: ioutil.Discard, }, nil @@ -149,13 +156,42 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } +// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultFrameCache 50000 +func (f *Frame) SetRankedCacheSize(v int) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == 0 || f.rankedCacheSize == v { + return nil + } + + // Persist meta data to disk on change. + f.rankedCacheSize = v + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + +// RankedCacheSize returns the ranked frame cache size. +func (f *Frame) RankedCacheSize() int { + f.mu.Lock() + v := f.rankedCacheSize + f.mu.Unlock() + return v +} + // Options returns all options for this frame. func (f *Frame) Options() FrameOptions { f.mu.Lock() opt := FrameOptions{ RowLabel: f.rowLabel, - CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, + CacheSize: f.rankedCacheSize, } f.mu.Unlock() return opt @@ -559,8 +595,9 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` - CacheType string `json:"cacheType,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize int `json:"cacheSize,omitempty"` } // importBitSet represents slices of row and column ids. diff --git a/view.go b/view.go index 54dda6f56..0f554de92 100644 --- a/view.go +++ b/view.go @@ -30,6 +30,8 @@ type View struct { frame string name string + cacheSize int + // Fragments by slice. cacheType string // passed in by frame fragments map[uint64]*Fragment @@ -41,12 +43,13 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string) *View { +func NewView(path, db, frame, name string, cacheSize int) *View { return &View{ - path: path, - db: db, - frame: frame, - name: name, + path: path, + db: db, + frame: frame, + name: name, + cacheSize: cacheSize, cacheType: DefaultCacheType, fragments: make(map[uint64]*Fragment), @@ -213,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize) frag.cacheType = v.cacheType frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) diff --git a/view_test.go b/view_test.go index 87b3e5826..d10c90936 100644 --- a/view_test.go +++ b/view_test.go @@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name), + View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), BitmapAttrStore: MustOpenAttrStore(), } v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore @@ -52,7 +52,7 @@ func (v *View) Reopen() error { return err } - v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name()) + v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore if err := v.Open(); err != nil { return err From 3b71ed1b2f1985b517c4a034b01997ba5cfff588 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 15:15:03 -0600 Subject: [PATCH 02/63] make the fragment cache ThresholdIndex a function at 90% of ThresholdLength --- fragment.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index fe04682dc..0b1734593 100644 --- a/fragment.go +++ b/fragment.go @@ -43,6 +43,9 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 + + // Percentage of the ThresholdLength size to resort the cache at + ThresholdBufferPct = 0.9 ) const ( @@ -245,7 +248,7 @@ func (f *Fragment) openCache() error { case CacheTypeRanked: c := NewRankCache() c.ThresholdLength = f.cacheSize - c.ThresholdIndex = f.cacheSize - 500 + c.ThresholdIndex = int(float64(f.cacheSize) * ThresholdBufferPct) f.cache = c case CacheTypeLRU: f.cache = NewLRUCache(f.cacheSize) From 591c35c95f5e2f124da35e81c93ff7bb8da2cea1 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 7 Mar 2017 15:34:01 -0600 Subject: [PATCH 03/63] set the cache size on frame creation --- db.go | 4 ++++ fragment.go | 22 ---------------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/db.go b/db.go index ef95a54bf..3abff2a9c 100644 --- a/db.go +++ b/db.go @@ -391,6 +391,10 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.RowLabel != "" { f.rowLabel = opt.RowLabel } + if opt.CacheSize != 0 { + f.rankedCacheSize = opt.CacheSize + } + f.inverseEnabled = opt.InverseEnabled if err := f.saveMeta(); err != nil { f.Close() diff --git a/fragment.go b/fragment.go index 0b1734593..4f6362193 100644 --- a/fragment.go +++ b/fragment.go @@ -222,25 +222,6 @@ func (f *Fragment) openStorage() error { } -// // It will be the one and only identifier after a package specifier. -// var testNameRegexp = regexp.MustCompile(`\.(Test[\p{L}_\p{N}]*)$`) - -// // Returns the name of the test function from the call stack. See -// // http://stackoverflow.com/q/35535635/149482 for another method. -// func GetTestName() string { -// pc := make([]uintptr, 32) -// n := runtime.Callers(0, pc) -// for i := 0; i < n; i++ { -// name := runtime.FuncForPC(pc[i]).Name() -// ms := testNameRegexp.FindStringSubmatch(name) -// if ms == nil { -// continue -// } -// return ms[1] -// } -// panic("test name could not be recovered") -// } - // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. @@ -256,9 +237,6 @@ func (f *Fragment) openCache() error { return ErrInvalidCacheType } - // fmt.Println(GetTestName()) - // fmt.Printf("CACHE SIZE: %d\n", f.cacheSize) - // Read cache data from disk. path := f.CachePath() buf, err := ioutil.ReadFile(path) From 4e7c2af5722da2f8b80c22d417e6f3e712d699f9 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 17 Mar 2017 11:28:17 -0500 Subject: [PATCH 04/63] fixed test to use ranked frame type --- fragment_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index ca6ffa3dc..56e2381e2 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -288,7 +288,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f", slice, cacheLimit), + Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", slice, cacheLimit), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -305,13 +305,24 @@ func TestFragment_TopN_CacheSize(t *testing.T) { f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) f.MustSetBits(105, 10, 11) + f.RecalculateCache() + + p := []pilosa.Pair{ + {Key: 104, Count: 7}, + {Key: 103, Count: 6}, + {Key: 102, Count: 5}} + // {Key: 101, Count: 4}, + // {Key: 100, Count: 3}} + // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) != cacheLimit { t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { - t.Fatalf("unexpected pair(0): %v", pairs[0]) + t.Fatalf("unexpected pair(0): %v", pairs) + } else if !reflect.DeepEqual(pairs, p) { + t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) } } From 9321637d18327a21329ee5319fee1f83e6d531b4 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 17 Mar 2017 13:45:30 -0500 Subject: [PATCH 05/63] trim the ranked cache set on the 1st recalculate call --- cache.go | 1 + fragment_test.go | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cache.go b/cache.go index 3504806c6..23bc52c30 100644 --- a/cache.go +++ b/cache.go @@ -211,6 +211,7 @@ func (c *RankCache) recalculate() { c.rankings = rankings if len(c.rankings) > c.ThresholdIndex { c.ThresholdValue = rankings[c.ThresholdIndex].Count + c.rankings = c.rankings[0:c.ThresholdIndex] } else { c.ThresholdValue = 1 } diff --git a/fragment_test.go b/fragment_test.go index 56e2381e2..7f3d35add 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -309,16 +309,13 @@ func TestFragment_TopN_CacheSize(t *testing.T) { p := []pilosa.Pair{ {Key: 104, Count: 7}, - {Key: 103, Count: 6}, - {Key: 102, Count: 5}} - // {Key: 101, Count: 4}, - // {Key: 100, Count: 3}} + {Key: 103, Count: 6}} // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) != cacheLimit { - t.Fatalf("TopN count cannot exceed cache size: %d", len(pairs)) + } else if len(pairs) > cacheLimit { + t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { From 7c2f6a9bae2f1de825a9ecbcc9d448d2e9488b48 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 7 Dec 2016 14:09:08 -0600 Subject: [PATCH 06/63] adds a basic gossip implementation using hashicorp/memberlist Changes Gossiper to NodeSet. Adds StaticNodeSet (for testing) and GossipNodeSet (for memberlist) implementations. Changes NodeSet interface to return a pilosa-specific generic instead of `NodeSet interface`. Removes NumMembers from interface (which is specific to memberlist). Implements a Messenger interface with which to send inter-node messages via NodeSet. The Pilosa implementation occurs in the GossipNodeSet. Implements the Messenger as an object on Server, Handler, and Index. Uses HealthStatus constants. Removes commented-out code. for gossip, make sure to bind to both host and port, and advertise those as well adjust messenger to work with the db schema logic add dependencies: hashicorp/memberlist, golang.org/x/sync add dependency: golang.org/x/net Uses `errgroup` to handle errors from broadcast messages. Marshals message one time instead of once for every node. Adds error handling for some errors that were being swallowed. --- cluster.go | 184 ++++++++++++++++++++++++++++++++++++++++++++++- cluster_test.go | 40 +++++++++++ config.go | 50 +++++++++++-- db.go | 5 +- executor_test.go | 4 +- frame.go | 21 +++++- glide.lock | 26 ++++++- glide.yaml | 5 ++ gossip.go | 165 ++++++++++++++++++++++++++++++++++++++++++ handler.go | 54 +++++++++++++- index.go | 28 +++++++- index_test.go | 27 +++++++ messenger.go | 73 +++++++++++++++++++ server.go | 27 +++---- 14 files changed, 682 insertions(+), 27 deletions(-) create mode 100644 gossip.go create mode 100644 messenger.go diff --git a/cluster.go b/cluster.go index beb33496b..8268fe508 100644 --- a/cluster.go +++ b/cluster.go @@ -1,8 +1,17 @@ package pilosa import ( + "bytes" "encoding/binary" + "fmt" "hash/fnv" + "io/ioutil" + "net/http" + "net/url" + + "golang.org/x/sync/errgroup" + + "github.com/gogo/protobuf/proto" ) const ( @@ -11,6 +20,10 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 + + // HealthStatus is the return value of the /health endpoint for a node in the cluster. + HealthStatusUp = "UP" + HealthStatusDown = "DOWN" ) // Node represents a node in the cluster. @@ -81,7 +94,8 @@ func (a Nodes) Clone() []*Node { // Cluster represents a collection of nodes. type Cluster struct { - Nodes []*Node + Nodes []*Node + NodeSet NodeSet // Hashing algorithm used to assign partitions to nodes. Hasher Hasher @@ -102,6 +116,33 @@ func NewCluster() *Cluster { } } +// NodeSetHosts returns the list of host strings for NodeSet members +func (c *Cluster) NodeSetHosts() []string { + if c.NodeSet == nil { + return []string{} + } + a := make([]string, 0, len(c.NodeSet.Nodes())) + for _, m := range c.NodeSet.Nodes() { + a = append(a, m.Host) + } + return a +} + +// Health returns a list of nodes in the cluster along with each node's state (UP/DOWN). +func (c *Cluster) Health() map[string]string { + h := make(map[string]string) + for _, n := range c.Nodes { + h[n.Host] = HealthStatusDown + } + // we are assuming that NodeSetHosts is a subset of c.Nodes + for _, m := range c.NodeSetHosts() { + if _, ok := h[m]; ok { + h[m] = HealthStatusUp + } + } + return h +} + // NodeByHost returns a node reference by host. func (c *Cluster) NodeByHost(host string) *Node { for _, n := range c.Nodes { @@ -157,6 +198,21 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } +// NodeSet represents an interface to maintaining Node state. +type NodeSet interface { + // Returns a list of all Nodes in the cluster + Nodes() []*Node + + // Attempts to join a cluster having `nodes` as its existing members + Join(nodes []*Node) (int, error) + + // Open starts any network activity implemented by the NodeSet + Open() error + + // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage + SetMessageHandler(f func(proto.Message) error) +} + // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). @@ -179,3 +235,129 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } + +// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. +type HTTPNodeSet struct { + nodes []*Node + messageHandler func(m proto.Message) error +} + +// NewHTTPNodeSet returns a new instance of HTTPNodeSet. +func NewHTTPNodeSet() *HTTPNodeSet { + return &HTTPNodeSet{} +} + +func (h *HTTPNodeSet) Nodes() []*Node { + return h.nodes +} + +func (h *HTTPNodeSet) Join(nodes []*Node) (int, error) { + h.nodes = nodes + return 0, nil +} + +func (h *HTTPNodeSet) Open() error { + return nil +} + +// SendMessage asyncronously broadcasts a protobuf message to all nodes. +func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { + + // Marshal the pb to []byte + buf, err := MarshalMessage(pb) + if err != nil { + return err + } + + var g errgroup.Group + for _, n := range h.nodes { + node := n + g.Go(func() error { + return h.sendNodeMessage(node, buf) + }) + } + return g.Wait() +} + +// ReceiveMessage is called when a node recieves a message. +func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { + return h.messageHandler(pb) +} + +func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { + var client *http.Client + client = http.DefaultClient + + // Create HTTP request. + req, err := http.NewRequest("POST", (&url.URL{ + Scheme: "http", + Host: node.Host, + Path: "/message", + }).String(), bytes.NewReader(msg)) + if err != nil { + return err + } + + // Require protobuf encoding. + req.Header.Set("Content-Type", "application/x-protobuf") + + // Send request to remote node. + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read response into buffer. + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + // Check status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) + } + + return nil +} + +// SetMessageHandler provides the Messenger with a function to handle incoming messages. +func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { + h.messageHandler = f +} + +// StaticNodeSet represents a basic NodeSet for testing +type StaticNodeSet struct { + Messenger + nodes []*Node +} + +func NewStaticNodeSet() *StaticNodeSet { + return &StaticNodeSet{} +} + +func (s *StaticNodeSet) Nodes() []*Node { + return s.nodes +} + +func (s *StaticNodeSet) Join(nodes []*Node) (int, error) { + s.nodes = nodes + return 0, nil +} + +func (s *StaticNodeSet) Open() error { + return nil +} + +func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { + return +} + +func (s *StaticNodeSet) SendMessage(pb proto.Message) error { + return nil +} +func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { + return nil +} diff --git a/cluster_test.go b/cluster_test.go index f5ff29edf..2e9b01d89 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -77,6 +77,46 @@ func TestHasher(t *testing.T) { } } +// Ensure that an empty cluster returns a valid (empty) NodeSet +func TestCluster_NodeSetHosts(t *testing.T) { + + c := pilosa.Cluster{} + + if h := c.NodeSetHosts(); !reflect.DeepEqual(h, []string{}) { + t.Fatalf("unexpected slice of hosts: %s", h) + } +} + +// Ensure cluster can compare its Nodes and Members +func TestCluster_Health(t *testing.T) { + c := pilosa.Cluster{ + Nodes: []*pilosa.Node{ + {Host: "serverA:1000"}, + {Host: "serverB:1000"}, + {Host: "serverC:1000"}, + }, + NodeSet: &pilosa.StaticNodeSet{}, + } + + j, err := c.NodeSet.Join([]*pilosa.Node{ + &pilosa.Node{Host: "serverA:1000"}, + &pilosa.Node{Host: "serverC:1000"}, + &pilosa.Node{Host: "serverD:1000"}, + }) + if err != nil { + t.Fatalf("unexpected gossiper nodes: %s", j) + } + + // Verify a DOWN node is reported, and extraneous nodes are ignored + if a := c.Health(); !reflect.DeepEqual(a, map[string]string{ + "serverA:1000": "UP", + "serverB:1000": "DOWN", + "serverC:1000": "UP", + }) { + t.Fatalf("unexpected health: %s", spew.Sdump(a)) + } +} + // NewCluster returns a cluster with n nodes and uses a mod-based hasher. func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() diff --git a/config.go b/config.go index 7d2efc5fe..4c0554ad4 100644 --- a/config.go +++ b/config.go @@ -1,11 +1,15 @@ package pilosa -import "time" +import ( + "net" + "time" +) const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultGossipPort = 14000 ) // Config represents the configuration for the command. @@ -14,9 +18,11 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` + ReplicaN int `toml:"replicas"` + MessengerType string `toml:"messenger-type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + Gossip *ConfigGossip `toml:"gossip"` } `toml:"cluster"` Plugins struct { @@ -30,6 +36,15 @@ type Config struct { LogPath string `toml:"log-path"` } +type ConfigNode struct { + Host string `toml:"host"` +} + +type ConfigGossip struct { + Port int `toml:"port"` + Seed string `toml:"seed"` +} + // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ @@ -61,6 +76,29 @@ func (c *Config) PilosaCluster() *Cluster { cluster.Nodes = append(cluster.Nodes, &Node{Host: hostport}) } + // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. + if c.Cluster.MessengerType == "broadcast" { + cluster.NodeSet = NewHTTPNodeSet() + cluster.NodeSet.Join(cluster.Nodes) + } else if (c.Cluster.MessengerType == "gossip") && (c.Cluster.Gossip != nil) { + gossipPort := DefaultGossipPort + gossipSeed := DefaultHost + if c.Cluster.Gossip.Port != 0 { + gossipPort = c.Cluster.Gossip.Port + } + if c.Cluster.Gossip.Seed != "" { + gossipSeed = c.Cluster.Gossip.Seed + } + // get the host portion of addr to use for binding + gossipHost, _, err := net.SplitHostPort(c.Host) + if err != nil { + gossipHost = c.Host + } + cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) + } else { + cluster.NodeSet = NewStaticNodeSet() + } + return cluster } diff --git a/db.go b/db.go index 3abff2a9c..ca03188a3 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,8 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - stats StatsClient + messenger Messenger + stats StatsClient LogOutput io.Writer } @@ -67,6 +68,7 @@ func NewDB(path, name string) (*DB, error) { columnLabel: DefaultColumnLabel, + messenger: NopMessenger, stats: NopStatsClient, LogOutput: ioutil.Discard, }, nil @@ -416,6 +418,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.messenger = db.messenger return f, nil } diff --git a/executor_test.go b/executor_test.go index be005e732..7a0c95622 100644 --- a/executor_test.go +++ b/executor_test.go @@ -470,7 +470,9 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0}) { //TODO: this is incorrect because the calling node doesn't know about slice 2 + // NOTE: while the following is technically incorrect (it should be {0, 2}) because the calling node doesn't know about slice 2 yet, + // we are ok with this and assuming that the calling node will become aware of slice 2 via inter-node messaging + } else if !reflect.DeepEqual(slices, []uint64{0}) { t.Fatalf("unexpected slices: %+v", slices) } diff --git a/frame.go b/frame.go index be949b19d..b299d2ac2 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,8 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - stats StatsClient + messenger Messenger + stats StatsClient // Frame settings. rowLabel string @@ -66,7 +67,8 @@ func NewFrame(path, db, name string) (*Frame, error) { views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), - stats: NopStatsClient, + messenger: NopMessenger, + stats: NopStatsClient, rowLabel: DefaultRowLabel, cacheType: DefaultCacheType, @@ -409,6 +411,21 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { view.BitmapAttrStore = f.bitmapAttrStore f.views[view.Name()] = view + // TODO: this needs to be refactored for views + /* + // Send a MaxSlice message + f.messenger.SendMessage( + &internal.CreateSliceMessage{ + DB: f.db, + Slice: slice, + }) + + frag.BitmapAttrStore = f.bitmapAttrStore + + // Save to lookup. + f.fragments[slice] = frag + */ + return view, nil } diff --git a/glide.lock b/glide.lock index 7fffeca5e..e4b6ce617 100644 --- a/glide.lock +++ b/glide.lock @@ -1,6 +1,10 @@ hash: 743e8f978eb4ad8f80a2ab71b05caebbf50b6769b71aa457bc4f144fef8c6595 updated: 2017-04-18T15:33:39.035615802-05:00 imports: +- name: github.com/armon/go-metrics + version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 +- name: github.com/aws/aws-sdk-go + version: 819b71cf8430e434c1eee7e7e8b0f2b8870be899 - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml @@ -24,13 +28,21 @@ imports: subpackages: - lru - name: github.com/golang/protobuf - version: 888eb0692c857ec880338addf316bd662d5e630e + version: 8ee79997227bf9b34611aee7946ae64735e6fd93 subpackages: - proto - name: github.com/gorilla/context version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 - name: github.com/gorilla/mux version: 392c28fe23e1c45ddba891b0320b3b5df220beea +- name: github.com/hashicorp/errwrap + version: 7554cd9344cec97297fa6649b055a8c98c2a1e55 +- name: github.com/hashicorp/go-msgpack + version: fa3f63826f7c23912c15263591e65d54d080b458 + subpackages: + - codec +- name: github.com/hashicorp/go-multierror + version: ed905158d87462226a13fe39ddf685ea65f1c11f - name: github.com/hashicorp/hcl version: 630949a3c5fa3c613328e1b8256052cbc2327c9b subpackages: @@ -42,10 +54,14 @@ imports: - json/parser - json/scanner - json/token +- name: github.com/hashicorp/memberlist + version: 9800c50ab79c002353852a9b1095e9591b161513 - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/magiconair/properties version: b3b15ef068fd0b17ddf408a23669f20811d194d2 +- name: github.com/miekg/dns + version: ca336a1f95a6b89be9c250df26c7a41742eb4a6f - name: github.com/mitchellh/mapstructure version: db1efb556f84b25a0a13a04aad883943538ad2e0 - name: github.com/pelletier/go-buffruneio @@ -68,6 +84,14 @@ imports: version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/spf13/viper version: 7538d73b4eb9511d85a9f1dfef202eeb8ac260f4 +- name: golang.org/x/net + version: 60c41d1de8da134c05b7b40154a9a82bf5b7edb9 + subpackages: + - context +- name: golang.org/x/sync + version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c + subpackages: + - errgroup - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: diff --git a/glide.yaml b/glide.yaml index d4615b0ba..f317a0a1b 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,3 +31,8 @@ import: - package: github.com/spf13/viper - package: github.com/gorilla/mux version: ^1.3.0 +- package: github.com/aws/aws-sdk-go + version: ^1.6.10 +- package: github.com/hashicorp/memberlist +- package: golang.org/x/sync +- package: golang.org/x/net diff --git a/gossip.go b/gossip.go new file mode 100644 index 000000000..79d301a97 --- /dev/null +++ b/gossip.go @@ -0,0 +1,165 @@ +package pilosa + +import ( + "io" + "log" + "os" + + "github.com/gogo/protobuf/proto" + "github.com/hashicorp/memberlist" +) + +// GossipNodeSet represents a gossip implementation of NodeSet using memberlist +// GossipNodeSet also represents an implementation of memberlist.Delegate +type GossipNodeSet struct { + Memberlist *memberlist.Memberlist + Broadcasts *memberlist.TransmitLimitedQueue + + config *GossipConfig + + messageHandler func(m proto.Message) error + + // The writer for any logging. + LogOutput io.Writer +} + +func (g *GossipNodeSet) Nodes() []*Node { + a := make([]*Node, 0, g.Memberlist.NumMembers()) + for _, n := range g.Memberlist.Members() { + a = append(a, &Node{Host: n.Name}) + } + return a +} + +func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { + return g.Memberlist.Join(Nodes(nodes).Hosts()) +} + +func (g *GossipNodeSet) Open() error { + ml, err := memberlist.Create(g.config.memberlistConfig) + if err != nil { + return err + } + g.Memberlist = ml + + // attach to gossip seed node + g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds + + g.Broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return g.Memberlist.NumMembers() + }, + RetransmitMult: 3, + } + return nil +} + +func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { + g.messageHandler = f +} + +// implementation of the messenger.Messenger interface +func (g *GossipNodeSet) SendMessage(pb proto.Message) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err + } + + b := &broadcast{ + msg: msg, + notify: nil, + } + g.Broadcasts.QueueBroadcast(b) + return nil +} + +func (g *GossipNodeSet) ReceiveMessage(pb proto.Message) error { + err := g.messageHandler(pb) + if err != nil { + return err + } + return nil +} + +// implementation of the memberlist.Delegate interface +func (g *GossipNodeSet) NodeMeta(limit int) []byte { + return []byte{} +} + +func (g *GossipNodeSet) NotifyMsg(b []byte) { + m, err := UnmarshalMessage(b) + if err != nil { + g.logger().Printf("unmarshal message error: %s", err) + return + } + if err := g.ReceiveMessage(m); err != nil { + g.logger().Printf("receive message error: %s", err) + return + } +} + +func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { + return g.Broadcasts.GetBroadcasts(overhead, limit) +} + +func (g *GossipNodeSet) LocalState(join bool) []byte { + return []byte{} +} + +func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { + return +} + +// logger returns a logger for the GossipNodeSet. +func (g *GossipNodeSet) logger() *log.Logger { + return log.New(g.LogOutput, "", log.LstdFlags) +} + +// broadcast represents an implementation of memberlist.Broadcast +type broadcast struct { + msg []byte + notify chan<- struct{} +} + +func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { + return false +} + +func (b *broadcast) Message() []byte { + return b.msg +} + +func (b *broadcast) Finished() { + if b.notify != nil { + close(b.notify) + } +} + +//////////////////////////////////////////////////////////////// + +type GossipConfig struct { + gossipSeed string + memberlistConfig *memberlist.Config +} + +// NewGossipNodeSet returns a new instance of GossipNodeSet. +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string) *GossipNodeSet { + g := &GossipNodeSet{ + LogOutput: os.Stderr, + } + + //TODO: pull memberlist config from pilosa.cfg file + g.config = &GossipConfig{ + memberlistConfig: memberlist.DefaultLocalConfig(), + gossipSeed: gossipSeed, + } + g.config.memberlistConfig.Name = name + g.config.memberlistConfig.BindAddr = gossipHost + g.config.memberlistConfig.BindPort = gossipPort + g.config.memberlistConfig.AdvertiseAddr = gossipHost + g.config.memberlistConfig.AdvertisePort = gossipPort + g.config.memberlistConfig.GossipNodes = 1 + g.config.memberlistConfig.Delegate = g + + return g +} diff --git a/handler.go b/handler.go index 0e25c3f2f..d3c5a28cc 100644 --- a/handler.go +++ b/handler.go @@ -25,7 +25,8 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Index *Index + Index *Index + Messenger Messenger // Local hostname & cluster configuration. Host string @@ -49,6 +50,7 @@ type Handler struct { func NewHandler() *Handler { handler := &Handler{ LogOutput: os.Stderr, + Messenger: NopMessenger, } handler.Router = NewRouter(handler) return handler @@ -111,10 +113,23 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getStatusResponse{ + Health: h.Cluster.Health(), + }); err != nil { + h.logger().Printf("write status response error: %s", err) + } +} + type getSchemaResponse struct { DBs []*DBInfo `json:"dbs"` } +type getStatusResponse struct { + Health map[string]string `json:"health"` +} + // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { dbName := mux.Vars(r)["db"] @@ -177,6 +192,36 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } +// handlePostMessage handles /message requests. +func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.Messenger.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + return +} + func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { @@ -324,6 +369,13 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + // NOTE: this calls a second DeleteDB on the local node + h.Messenger.SendMessage( + &internal.DeleteDBMessage{ + DB: req.DB, + }) + // Encode response. if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/index.go b/index.go index 7597984ba..01409a277 100644 --- a/index.go +++ b/index.go @@ -11,6 +11,9 @@ import ( "sort" "sync" "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. @@ -23,6 +26,8 @@ type Index struct { // Databases by name. dbs map[string]*DB + Messenger Messenger + // Close management wg sync.WaitGroup closing chan struct{} @@ -45,7 +50,8 @@ func NewIndex() *Index { dbs: make(map[string]*DB), closing: make(chan struct{}, 0), - Stats: NopStatsClient, + Messenger: NopMessenger, + Stats: NopStatsClient, CacheFlushInterval: DefaultCacheFlushInterval, @@ -243,6 +249,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) + db.messenger = i.Messenger return db, nil } @@ -338,6 +345,25 @@ func (i *Index) flushCaches() { } } +// HandleMessage handles protobuf Messages broadcasted to nodes in the +// cluster from the Cluster's NodeSet. +func (i *Index) HandleMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := i.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.DeleteDBMessage: + err := i.DeleteDB(obj.DB) + if err != nil { + return err + } + } + return nil +} + func (i *Index) logger() *log.Logger { return log.New(i.LogOutput, "", log.LstdFlags) } // IndexSyncer is an active anti-entropy tool that compares the local index diff --git a/index_test.go b/index_test.go index 4a1be4cf2..50ce1979e 100644 --- a/index_test.go +++ b/index_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -162,6 +163,32 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } +// Ensure index can handle Messenger messages. +func TestIndex_HandleMessage(t *testing.T) { + // Create a local index. + idx0 := MustOpenIndex() + defer idx0.Close() + + idx0.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + + msg0 := &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + } + idx0.HandleMessage(msg0) + if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{"d": 8}) { + t.Fatalf("unexpected max slice: %s", ms) + } + + msg1 := &internal.DeleteDBMessage{ + DB: "d", + } + idx0.HandleMessage(msg1) + if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{}) { + t.Fatalf("unexpected delete db: %s", ms) + } +} + // Index is a test wrapper for pilosa.Index. type Index struct { *pilosa.Index diff --git a/messenger.go b/messenger.go new file mode 100644 index 000000000..115627ae4 --- /dev/null +++ b/messenger.go @@ -0,0 +1,73 @@ +package pilosa + +import ( + "fmt" + "reflect" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +func init() { + NopMessenger = &nopMessenger{} +} + +var NopMessenger Messenger + +// nopMessenger represents a Messenger that doesn't do anything. +type nopMessenger struct{} + +func (c *nopMessenger) SendMessage(pb proto.Message) error { + fmt.Println("NOPMessenger: Send") + return nil +} +func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { + fmt.Println("NOPMessenger: Receive") + return nil +} + +type Messenger interface { + SendMessage(pb proto.Message) error + ReceiveMessage(pb proto.Message) error +} + +const ( + MessageTypeCreateSlice = 1 + MessageTypeDeleteDB = 2 +) + +func MarshalMessage(m proto.Message) ([]byte, error) { + var typ uint8 + switch obj := m.(type) { + case *internal.CreateSliceMessage: + typ = MessageTypeCreateSlice + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + default: + return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) + } + buf, err := proto.Marshal(m) + if err != nil { + return nil, err + } + return append([]byte{typ}, buf...), nil +} + +func UnmarshalMessage(buf []byte) (proto.Message, error) { + typ, buf := buf[0], buf[1:] + + var m proto.Message + switch typ { + case MessageTypeCreateSlice: + m = &internal.CreateSliceMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + default: + return nil, fmt.Errorf("invalid message type: %d", typ) + } + + if err := proto.Unmarshal(buf, m); err != nil { + return nil, err + } + return m, nil +} diff --git a/server.go b/server.go index 55702acf9..f4cef42f6 100644 --- a/server.go +++ b/server.go @@ -32,8 +32,9 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler + Index *Index + Handler *Handler + Messenger Messenger // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -52,8 +53,9 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), + Index: NewIndex(), + Handler: NewHandler(), + Messenger: NopMessenger, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -96,6 +98,11 @@ func (s *Server) Open() error { return err } + // Open NodeSet communication + if err := s.Cluster.NodeSet.Open(); err != nil { + return err + } + // Create executor for executing queries. e := NewExecutor() e.Index = s.Index @@ -202,21 +209,15 @@ func (s *Server) monitorMaxSlices() { if s.Host != node.Host { maxSlices, _ := checkMaxSlices(node.Host) for db, newmax := range maxSlices { - // if we don't know about a db locally, create it - // so that the /schema endpoint can report it + // if we don't know about a db locally, log an error because + // db's should be created and synced prior to slice creation if localdb := s.Index.DB(db); localdb != nil { if newmax > oldmaxslices[db] { oldmaxslices[db] = newmax localdb.SetRemoteMaxSlice(newmax) } } else { - d := s.Index.DB(db) - if d == nil { - s.logger().Printf("Local DB not found: %s", db) - return - } - oldmaxslices[db] = newmax - d.SetRemoteMaxSlice(newmax) + s.logger().Printf("Local DB not found: %s", db) } } } From 2b35df879cb3c7e0b474ac9c47116db789fabe8e Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 14 Mar 2017 17:44:23 -0500 Subject: [PATCH 07/63] WIP: first pass at sending `CreateDB()` through Messenger --- client.go | 1 + cluster.go | 4 ++++ index.go | 17 ++++++++++++++++- messenger.go | 5 +++++ server.go | 8 +++++--- server/server.go | 9 +++++++++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/client.go b/client.go index e4401a69a..d340ce640 100644 --- a/client.go +++ b/client.go @@ -159,6 +159,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { case http.StatusOK: return nil // ok case http.StatusConflict: + fmt.Println("ErrDatabaseExists: 1") return ErrDatabaseExists default: return errors.New(string(body)) diff --git a/cluster.go b/cluster.go index 8268fe508..061f76c62 100644 --- a/cluster.go +++ b/cluster.go @@ -285,6 +285,7 @@ func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { + fmt.Println("sendNodeMessage:", node.Host) var client *http.Client client = http.DefaultClient @@ -302,7 +303,9 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { req.Header.Set("Content-Type", "application/x-protobuf") // Send request to remote node. + fmt.Println("Send") resp, err := client.Do(req) + fmt.Println("Got back") if err != nil { return err } @@ -314,6 +317,7 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { if err != nil { return err } + fmt.Println("code:", resp.StatusCode) // Check status code. if resp.StatusCode != http.StatusOK { diff --git a/index.go b/index.go index 01409a277..f7269d670 100644 --- a/index.go +++ b/index.go @@ -187,12 +187,14 @@ func (i *Index) DBs() []*DB { } // CreateDB creates a database. +// An error is returned if the database already exists. func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { i.mu.Lock() defer i.mu.Unlock() // Ensure db doesn't already exist. if i.dbs[name] != nil { + fmt.Println("ErrDatabaseExists: 2") return nil, ErrDatabaseExists } return i.createDB(name, opt) @@ -204,7 +206,7 @@ func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { i.mu.Lock() defer i.mu.Unlock() - // Find frame in cache first. + // Find database in cache first. if db := i.dbs[name]; db != nil { return db, nil } @@ -239,6 +241,13 @@ func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { i.Stats.Count("dbN", 1) + // Send a CreateDB message + i.Messenger.SendMessage( + &internal.CreateDBMessage{ + DB: name, + ColumnLabel: opt.ColumnLabel, + }) + return db, nil } @@ -360,6 +369,12 @@ func (i *Index) HandleMessage(pb proto.Message) error { if err != nil { return err } + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.ColumnLabel} + _, err := i.CreateDB(obj.DB, opt) + if err != nil { + return err + } } return nil } diff --git a/messenger.go b/messenger.go index 115627ae4..5e2d128e2 100644 --- a/messenger.go +++ b/messenger.go @@ -34,6 +34,7 @@ type Messenger interface { const ( MessageTypeCreateSlice = 1 MessageTypeDeleteDB = 2 + MessageTypeCreateDB = 3 ) func MarshalMessage(m proto.Message) ([]byte, error) { @@ -43,6 +44,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateSlice case *internal.DeleteDBMessage: typ = MessageTypeDeleteDB + case *internal.CreateDBMessage: + typ = MessageTypeCreateDB default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -62,6 +65,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateSliceMessage{} case MessageTypeDeleteDB: m = &internal.DeleteDBMessage{} + case MessageTypeCreateDB: + m = &internal.CreateDBMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/server.go b/server.go index f4cef42f6..cde64c3ca 100644 --- a/server.go +++ b/server.go @@ -120,9 +120,11 @@ func (s *Server) Open() error { go func() { http.Serve(ln, s.Handler) }() // Start background monitoring. - s.wg.Add(2) - go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + /* + s.wg.Add(2) + go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + */ return nil } diff --git a/server/server.go b/server/server.go index 251a054aa..da83da52c 100644 --- a/server/server.go +++ b/server/server.go @@ -94,6 +94,15 @@ func (m *Command) Run(args ...string) (err error) { } m.Server.Cluster = m.Config.PilosaCluster() + // Setup Messenger. + fmt.Fprintf(m.Stderr, "Using Messenger type: %s\n", m.Config.Cluster.MessengerType) + m.Server.Messenger = m.Server.Cluster.NodeSet.(pilosa.Messenger) + m.Server.Handler.Messenger = m.Server.Messenger + m.Server.Index.Messenger = m.Server.Messenger + + // Set message handler. + m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) + // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) From 8fa45649b678745e6eb5e8d9262415426c3500d1 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Mar 2017 17:19:42 -0500 Subject: [PATCH 08/63] WIP: Added support for NodeState. Modified the gossip implementation to optionally send messages directly (instead of over gossip). Added support for local and remote NodeState, as well as function to merge remote into local state. Refactored Protobuf models to include `DBMeta` and `FrameMeta` as well as support for NodeState. --- cluster.go | 38 +++++++++++++++---- db.go | 36 +++++++++++++++--- frame.go | 48 ++++++++++++++++-------- gossip.go | 95 +++++++++++++++++++++++++++++++++++++++--------- handler.go | 32 ++++++++++++++-- index.go | 30 ++++++++------- messenger.go | 26 +++++++++---- server.go | 52 ++++++++++++++++++++++++++ server/server.go | 4 +- 9 files changed, 291 insertions(+), 70 deletions(-) diff --git a/cluster.go b/cluster.go index 061f76c62..bbdc8f51f 100644 --- a/cluster.go +++ b/cluster.go @@ -211,6 +211,12 @@ type NodeSet interface { // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage SetMessageHandler(f func(proto.Message) error) + + // SetRemoteStateHandler provides the function to call on MergeRemoteState + SetRemoteStateHandler(f func(proto.Message) error) + + // SetLocalStateSource provides the function to get the current node's local state. + SetLocalStateSource(f func() (proto.Message, error)) } // Hasher represents an interface to hash integers into buckets. @@ -240,6 +246,8 @@ func (h *jmphasher) Hash(key uint64, n int) int { type HTTPNodeSet struct { nodes []*Node messageHandler func(m proto.Message) error + // remoteStateHandler func(m proto.Message) error + // localStateSource func() (proto.Message, error) } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -261,7 +269,7 @@ func (h *HTTPNodeSet) Open() error { } // SendMessage asyncronously broadcasts a protobuf message to all nodes. -func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { +func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) @@ -279,13 +287,12 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message) error { return g.Wait() } -// ReceiveMessage is called when a node recieves a message. +// ReceiveMessage is called when a node receives a message. func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { return h.messageHandler(pb) } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { - fmt.Println("sendNodeMessage:", node.Host) var client *http.Client client = http.DefaultClient @@ -303,9 +310,7 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { req.Header.Set("Content-Type", "application/x-protobuf") // Send request to remote node. - fmt.Println("Send") resp, err := client.Do(req) - fmt.Println("Got back") if err != nil { return err } @@ -317,7 +322,6 @@ func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { if err != nil { return err } - fmt.Println("code:", resp.StatusCode) // Check status code. if resp.StatusCode != http.StatusOK { @@ -332,6 +336,18 @@ func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { h.messageHandler = f } +// SetRemoteStateHandler provides the Messenger with a function to merge remote state. +func (h *HTTPNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + // not implemented + // h.remoteStateHandler = f +} + +// SetLocalStateSource currently no-ops. +func (h *HTTPNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + // not implemented + // h.localStateSource = f +} + // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { Messenger @@ -359,7 +375,15 @@ func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { return } -func (s *StaticNodeSet) SendMessage(pb proto.Message) error { +func (s *StaticNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + return +} + +func (s *StaticNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + return +} + +func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { return nil } func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { diff --git a/db.go b/db.go index ca03188a3..a7f6f8ff1 100644 --- a/db.go +++ b/db.go @@ -173,7 +173,7 @@ func (db *DB) openFrames() error { // loadMeta reads meta data for the database, if any. func (db *DB) loadMeta() error { - var pb internal.DB + var pb internal.DBMeta // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(db.path, ".meta")) @@ -199,7 +199,7 @@ func (db *DB) loadMeta() error { // saveMeta writes meta data for the database. func (db *DB) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.DB{ + buf, err := proto.Marshal(&internal.DBMeta{ TimeQuantum: string(db.timeQuantum), ColumnLabel: db.columnLabel, }) @@ -251,10 +251,10 @@ func (db *DB) MaxSlice() uint64 { return max } -func (db *DB) SetRemoteMaxSlice(v uint64) { +func (db *DB) SetRemoteMaxSlice(newmax uint64) { db.mu.Lock() defer db.mu.Unlock() - db.remoteMaxSlice = v + db.remoteMaxSlice = newmax } // MaxInverseSlice returns the max inverse slice in the database according to this node. @@ -396,6 +396,9 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { if opt.CacheSize != 0 { f.rankedCacheSize = opt.CacheSize } + if opt.TimeQuantum.Valid() { + f.timeQuantum = opt.TimeQuantum + } f.inverseEnabled = opt.InverseEnabled if err := f.saveMeta(); err != nil { @@ -509,9 +512,32 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { return dbs } +// encodeDBs converts a into its internal representation. +func encodeDBs(a []*DB) []*internal.DB { + other := make([]*internal.DB, len(a)) + for i := range a { + other[i] = encodeDB(a[i]) + } + return other +} + +// encodeDB converts d into its internal representation. +func encodeDB(d *DB) *internal.DB { + return &internal.DB{ + Name: d.name, + Meta: &internal.DBMeta{ + ColumnLabel: d.columnLabel, + TimeQuantum: string(d.timeQuantum), + }, + MaxSlice: d.remoteMaxSlice, + Frames: encodeFrames(d.Frames()), + } +} + // DBOptions represents options to set when initializing a db. type DBOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` + ColumnLabel string `json:"columnLabel,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // hasTime returns true if a contains a non-nil time. diff --git a/frame.go b/frame.go index b299d2ac2..51cb2c41b 100644 --- a/frame.go +++ b/frame.go @@ -264,7 +264,7 @@ func (f *Frame) openViews() error { // loadMeta reads meta data for the frame, if any. func (f *Frame) loadMeta() error { - var pb internal.Frame + var pb internal.FrameMeta // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) @@ -413,17 +413,12 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // TODO: this needs to be refactored for views /* - // Send a MaxSlice message - f.messenger.SendMessage( - &internal.CreateSliceMessage{ - DB: f.db, - Slice: slice, - }) - - frag.BitmapAttrStore = f.bitmapAttrStore - - // Save to lookup. - f.fragments[slice] = frag + // Send a MaxSlice message + f.messenger.SendMessage( + &internal.CreateSliceMessage{ + DB: f.db, + Slice: slice, + }, "gossip") */ return view, nil @@ -591,6 +586,26 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) return nil } +// encodeFrames converts a into its internal representation. +func encodeFrames(a []*Frame) []*internal.Frame { + other := make([]*internal.Frame, len(a)) + for i := range a { + other[i] = encodeFrame(a[i]) + } + return other +} + +// encodeFrame converts f into its internal representation. +func encodeFrame(f *Frame) *internal.Frame { + return &internal.Frame{ + Name: f.name, + Meta: &internal.FrameMeta{ + TimeQuantum: string(f.timeQuantum), + RowLabel: f.rowLabel, + }, + } +} + type frameSlice []*Frame func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -611,10 +626,11 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - RowLabel string `json:"rowLabel,omitempty"` - InverseEnabled bool `json:"inverseEnabled,omitempty"` - CacheType string `json:"cacheType,omitempty"` - CacheSize int `json:"cacheSize,omitempty"` + RowLabel string `json:"rowLabel,omitempty"` + InverseEnabled bool `json:"inverseEnabled,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize int `json:"cacheSize,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } // importBitSet represents slices of row and column ids. diff --git a/gossip.go b/gossip.go index 79d301a97..e2b3ddcde 100644 --- a/gossip.go +++ b/gossip.go @@ -1,38 +1,44 @@ package pilosa import ( + "fmt" "io" "log" "os" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" + "github.com/pilosa/pilosa/internal" ) // GossipNodeSet represents a gossip implementation of NodeSet using memberlist // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { - Memberlist *memberlist.Memberlist - Broadcasts *memberlist.TransmitLimitedQueue + memberlist *memberlist.Memberlist + broadcasts *memberlist.TransmitLimitedQueue config *GossipConfig - messageHandler func(m proto.Message) error + messageHandler func(m proto.Message) error + remoteStateHandler func(m proto.Message) error + localStateSource func() (proto.Message, error) // The writer for any logging. LogOutput io.Writer } func (g *GossipNodeSet) Nodes() []*Node { - a := make([]*Node, 0, g.Memberlist.NumMembers()) - for _, n := range g.Memberlist.Members() { + a := make([]*Node, 0, g.memberlist.NumMembers()) + for _, n := range g.memberlist.Members() { a = append(a, &Node{Host: n.Name}) } return a } func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { - return g.Memberlist.Join(Nodes(nodes).Hosts()) + return g.memberlist.Join(Nodes(nodes).Hosts()) } func (g *GossipNodeSet) Open() error { @@ -40,14 +46,14 @@ func (g *GossipNodeSet) Open() error { if err != nil { return err } - g.Memberlist = ml + g.memberlist = ml // attach to gossip seed node g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds - g.Broadcasts = &memberlist.TransmitLimitedQueue{ + g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return g.Memberlist.NumMembers() + return g.memberlist.NumMembers() }, RetransmitMult: 3, } @@ -58,18 +64,49 @@ func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { g.messageHandler = f } +func (g *GossipNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { + g.remoteStateHandler = f +} + +func (g *GossipNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { + g.localStateSource = f +} + // implementation of the messenger.Messenger interface -func (g *GossipNodeSet) SendMessage(pb proto.Message) error { +func (g *GossipNodeSet) SendMessage(pb proto.Message, method string) error { msg, err := MarshalMessage(pb) if err != nil { return err } - b := &broadcast{ - msg: msg, - notify: nil, + // Broadcast asyncronously sends the message directly to each node. + // An error from any node raises an error on the entire operation. + // This is a blocking operation. + // + // Gossip uses the gossip protocol to eventually deliver the message + // to every node. + switch method { + case "broadcast": + var eg errgroup.Group + for _, n := range g.memberlist.Members() { + // Don't send the message to the local node. + if n == g.memberlist.LocalNode() { + continue + } + node := n + eg.Go(func() error { + return g.memberlist.SendToTCP(node, msg) + }) + } + return eg.Wait() + case "gossip": + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) } - g.Broadcasts.QueueBroadcast(b) + return nil } @@ -87,6 +124,8 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { + loc := g.memberlist.LocalNode() + fmt.Println("Received Msg:", loc) m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -99,14 +138,37 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { } func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { - return g.Broadcasts.GetBroadcasts(overhead, limit) + return g.broadcasts.GetBroadcasts(overhead, limit) } func (g *GossipNodeSet) LocalState(join bool) []byte { - return []byte{} + + pb, err := g.localStateSource() + if err != nil { + g.logger().Printf("error getting local state, err=%s", err) + return []byte{} + } + + // Marshal nodestate data to bytes. + buf, err := proto.Marshal(pb) + if err != nil { + g.logger().Printf("error marshaling nodestate data, err=%s", err) + return []byte{} + } + return buf } func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { + // Unmarshal nodestate data. + var pb internal.NodeState + if err := proto.Unmarshal(buf, &pb); err != nil { + g.logger().Printf("error unmarshaling nodestate data, err=%s", err) + return + } + err := g.remoteStateHandler(&pb) + if err != nil { + g.logger().Printf("merge state error: %s", err) + } return } @@ -158,7 +220,6 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort - g.config.memberlistConfig.GossipNodes = 1 g.config.memberlistConfig.Delegate = g return g diff --git a/handler.go b/handler.go index d3c5a28cc..ad337b2bc 100644 --- a/handler.go +++ b/handler.go @@ -370,11 +370,13 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - // NOTE: this calls a second DeleteDB on the local node - h.Messenger.SendMessage( + err := h.Messenger.SendMessage( &internal.DeleteDBMessage{ DB: req.DB, - }) + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending DeleteDB message: %s", err) + } // Encode response. if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { @@ -511,6 +513,20 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } + // Send the create message to all nodes. + err = h.Messenger.SendMessage( + &internal.CreateFrameMessage{ + DB: req.DB, + Frame: req.Frame, + Meta: &internal.FrameMeta{ + RowLabel: req.Options.RowLabel, + TimeQuantum: string(req.Options.TimeQuantum), + }, + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending CreateFrame message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -569,6 +585,16 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + err := h.Messenger.SendMessage( + &internal.DeleteFrameMessage{ + DB: req.DB, + Frame: req.Frame, + }, "broadcast") + if err != nil { + h.logger().Printf("problem sending DeleteFrame message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) diff --git a/index.go b/index.go index f7269d670..954629eea 100644 --- a/index.go +++ b/index.go @@ -194,7 +194,6 @@ func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { // Ensure db doesn't already exist. if i.dbs[name] != nil { - fmt.Println("ErrDatabaseExists: 2") return nil, ErrDatabaseExists } return i.createDB(name, opt) @@ -236,18 +235,12 @@ func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { // Update options. db.SetColumnLabel(opt.ColumnLabel) + db.SetTimeQuantum(opt.TimeQuantum) i.dbs[db.Name()] = db i.Stats.Count("dbN", 1) - // Send a CreateDB message - i.Messenger.SendMessage( - &internal.CreateDBMessage{ - DB: name, - ColumnLabel: opt.ColumnLabel, - }) - return db, nil } @@ -364,17 +357,28 @@ func (i *Index) HandleMessage(pb proto.Message) error { return fmt.Errorf("Local DB not found: %s", obj.DB) } d.SetRemoteMaxSlice(obj.Slice) - case *internal.DeleteDBMessage: - err := i.DeleteDB(obj.DB) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := i.CreateDB(obj.DB, opt) if err != nil { return err } - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.ColumnLabel} - _, err := i.CreateDB(obj.DB, opt) + case *internal.DeleteDBMessage: + if err := i.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := i.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) if err != nil { return err } + case *internal.DeleteFrameMessage: + db := i.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } } return nil } diff --git a/messenger.go b/messenger.go index 5e2d128e2..f28eb7ddb 100644 --- a/messenger.go +++ b/messenger.go @@ -17,7 +17,7 @@ var NopMessenger Messenger // nopMessenger represents a Messenger that doesn't do anything. type nopMessenger struct{} -func (c *nopMessenger) SendMessage(pb proto.Message) error { +func (c *nopMessenger) SendMessage(pb proto.Message, method string) error { fmt.Println("NOPMessenger: Send") return nil } @@ -27,14 +27,16 @@ func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { } type Messenger interface { - SendMessage(pb proto.Message) error + SendMessage(pb proto.Message, method string) error ReceiveMessage(pb proto.Message) error } const ( MessageTypeCreateSlice = 1 - MessageTypeDeleteDB = 2 - MessageTypeCreateDB = 3 + MessageTypeCreateDB = 2 + MessageTypeDeleteDB = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 ) func MarshalMessage(m proto.Message) ([]byte, error) { @@ -42,10 +44,14 @@ func MarshalMessage(m proto.Message) ([]byte, error) { switch obj := m.(type) { case *internal.CreateSliceMessage: typ = MessageTypeCreateSlice - case *internal.DeleteDBMessage: - typ = MessageTypeDeleteDB case *internal.CreateDBMessage: typ = MessageTypeCreateDB + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + case *internal.CreateFrameMessage: + typ = MessageTypeCreateFrame + case *internal.DeleteFrameMessage: + typ = MessageTypeDeleteFrame default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -63,10 +69,14 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { switch typ { case MessageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeDeleteDB: - m = &internal.DeleteDBMessage{} case MessageTypeCreateDB: m = &internal.CreateDBMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + case MessageTypeCreateFrame: + m = &internal.CreateFrameMessage{} + case MessageTypeDeleteFrame: + m = &internal.DeleteFrameMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/server.go b/server.go index cde64c3ca..0df1182ba 100644 --- a/server.go +++ b/server.go @@ -153,6 +153,49 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +func (s *Server) LocalState() (proto.Message, error) { + // TODO: are there errors to handle? + pb := encodeLocalState(s) + return pb, nil +} + +// HandleRemoteState provides the current, local state. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) HandleRemoteState(pb proto.Message) error { + return s.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (s *Server) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := s.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { @@ -189,6 +232,15 @@ func (s *Server) monitorAntiEntropy() { } } +// encodeLocalState converts s into its internal representation. +func encodeLocalState(s *Server) *internal.NodeState { + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from cluster.Node + DBs: encodeDBs(s.Index.DBs()), + } +} + // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. func (s *Server) monitorMaxSlices() { // Ignore if only one node in the cluster. diff --git a/server/server.go b/server/server.go index da83da52c..3a62c5c60 100644 --- a/server/server.go +++ b/server/server.go @@ -100,8 +100,10 @@ func (m *Command) Run(args ...string) (err error) { m.Server.Handler.Messenger = m.Server.Messenger m.Server.Index.Messenger = m.Server.Messenger - // Set message handler. + // Set message and state handlers. m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) + m.Server.Cluster.NodeSet.SetRemoteStateHandler(m.Server.HandleRemoteState) + m.Server.Cluster.NodeSet.SetLocalStateSource(m.Server.LocalState) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) From 74b81eeac1ae915134551b3e834c47fba71a4156 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 17 Mar 2017 19:05:40 -0500 Subject: [PATCH 09/63] adjustments to support the new Config structure --- cmd/server.go | 5 ++++- config.go | 16 ++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 82363f849..d25066764 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -75,7 +75,7 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") - flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number hosts each piece of data should be stored on.") + flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") @@ -83,6 +83,9 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") + flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "", "Type of Messenger to use for inter-host messaging.") + flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") + flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") return serveCmd } diff --git a/config.go b/config.go index 4c0554ad4..17edd639d 100644 --- a/config.go +++ b/config.go @@ -18,11 +18,11 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - MessengerType string `toml:"messenger-type"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` - Gossip *ConfigGossip `toml:"gossip"` + ReplicaN int `toml:"replicas"` + MessengerType string `toml:"messenger-type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + Gossip ConfigGossip `toml:"gossip"` } `toml:"cluster"` Plugins struct { @@ -36,10 +36,6 @@ type Config struct { LogPath string `toml:"log-path"` } -type ConfigNode struct { - Host string `toml:"host"` -} - type ConfigGossip struct { Port int `toml:"port"` Seed string `toml:"seed"` @@ -80,7 +76,7 @@ func (c *Config) PilosaCluster() *Cluster { if c.Cluster.MessengerType == "broadcast" { cluster.NodeSet = NewHTTPNodeSet() cluster.NodeSet.Join(cluster.Nodes) - } else if (c.Cluster.MessengerType == "gossip") && (c.Cluster.Gossip != nil) { + } else if c.Cluster.MessengerType == "gossip" { gossipPort := DefaultGossipPort gossipSeed := DefaultHost if c.Cluster.Gossip.Port != 0 { From 8e2592e70696318799e2d1924aab4c13f7d1b6bb Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 11:07:53 -0500 Subject: [PATCH 10/63] store cache size in Frame metadata --- frame.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame.go b/frame.go index 51cb2c41b..5fea71e3d 100644 --- a/frame.go +++ b/frame.go @@ -273,6 +273,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled + f.rankedCacheSize = DefaultFrameCache return nil } else if err != nil { return err @@ -286,6 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled + f.rankedCacheSize = int(pb.CacheSize) // Copy cache type. f.cacheType = pb.CacheType @@ -304,6 +306,7 @@ func (f *Frame) saveMeta() error { RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheSize: int64(f.rankedCacheSize), }) if err != nil { return err @@ -602,6 +605,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, + CacheSize: int64(f.rankedCacheSize), }, } } From 9a1a631dd4dac53beaf07de6ec5c22608000fc30 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 11:19:02 -0500 Subject: [PATCH 11/63] test storing frame cache size --- fragment.go | 2 +- frame_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 4f6362193..e83f76ed2 100644 --- a/fragment.go +++ b/fragment.go @@ -44,7 +44,7 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - // Percentage of the ThresholdLength size to resort the cache at + // ThresholdBufferPct is the percentage of the ThresholdLength size to resort the cache at ThresholdBufferPct = 0.9 ) diff --git a/frame_test.go b/frame_test.go index 5b659f826..16b935937 100644 --- a/frame_test.go +++ b/frame_test.go @@ -126,3 +126,24 @@ func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time } return changed } + +// Ensure frame can set its cache +func TestFrame_SetCacheSize(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + cacheSize := 100 + + // Set & retrieve frame cache size. + if err := f.SetRankedCacheSize(cacheSize); err != nil { + t.Fatal(err) + } else if q := f.RankedCacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size: %d", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.RankedCacheSize(); q != cacheSize { + t.Fatalf("unexpected frame cache size (reopen): %d", q) + } +} From ce1e8e2e75253f7cc489eaab47f2a931a877e69f Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 20 Mar 2017 15:10:26 -0500 Subject: [PATCH 12/63] changed the frame cache name --- fragment.go | 2 +- frame.go | 38 +++++++++++++++++++------------------- frame_test.go | 6 +++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/fragment.go b/fragment.go index e83f76ed2..de188d0e8 100644 --- a/fragment.go +++ b/fragment.go @@ -44,7 +44,7 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - // ThresholdBufferPct is the percentage of the ThresholdLength size to resort the cache at + // ThresholdBufferPct is the percentage of the ThresholdLength for which to maintain a sorted, ranked list. ThresholdBufferPct = 0.9 ) diff --git a/frame.go b/frame.go index 5fea71e3d..ebfe15ab0 100644 --- a/frame.go +++ b/frame.go @@ -22,7 +22,7 @@ const ( DefaultInverseEnabled = false // Default ranked frame cache - DefaultFrameCache = 50000 + DefaultCacheSize = 50000 ) // Frame represents a container for views. @@ -47,7 +47,7 @@ type Frame struct { inverseEnabled bool // Cache size for ranked frames - rankedCacheSize int + cacheSize int LogOutput io.Writer } @@ -70,10 +70,10 @@ func NewFrame(path, db, name string) (*Frame, error) { messenger: NopMessenger, stats: NopStatsClient, - rowLabel: DefaultRowLabel, - cacheType: DefaultCacheType, - inverseEnabled: DefaultInverseEnabled, - rankedCacheSize: DefaultFrameCache, + rowLabel: DefaultRowLabel, + inverseEnabled: DefaultInverseEnabled, + cacheType: DefaultCacheType, + cacheSize: DefaultCacheSize, LogOutput: ioutil.Discard, }, nil @@ -158,19 +158,19 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } -// SetRankedCacheSize sets the cache size for ranked fames. Persists to meta file on update. -// defaults to DefaultFrameCache 50000 -func (f *Frame) SetRankedCacheSize(v int) error { +// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultCacheSize 50000 +func (f *Frame) SetCacheSize(v int) error { f.mu.Lock() defer f.mu.Unlock() // Ignore if no change occurred. - if v == 0 || f.rankedCacheSize == v { + if v == 0 || f.cacheSize == v { return nil } // Persist meta data to disk on change. - f.rankedCacheSize = v + f.cacheSize = v if err := f.saveMeta(); err != nil { return err } @@ -178,10 +178,10 @@ func (f *Frame) SetRankedCacheSize(v int) error { return nil } -// RankedCacheSize returns the ranked frame cache size. -func (f *Frame) RankedCacheSize() int { +// CacheSize returns the ranked frame cache size. +func (f *Frame) CacheSize() int { f.mu.Lock() - v := f.rankedCacheSize + v := f.cacheSize f.mu.Unlock() return v } @@ -193,7 +193,7 @@ func (f *Frame) Options() FrameOptions { RowLabel: f.rowLabel, InverseEnabled: f.inverseEnabled, CacheType: f.cacheType, - CacheSize: f.rankedCacheSize, + CacheSize: f.cacheSize, } f.mu.Unlock() return opt @@ -273,7 +273,7 @@ func (f *Frame) loadMeta() error { f.rowLabel = DefaultRowLabel f.cacheType = DefaultCacheType f.inverseEnabled = DefaultInverseEnabled - f.rankedCacheSize = DefaultFrameCache + f.cacheSize = DefaultCacheSize return nil } else if err != nil { return err @@ -287,7 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.rankedCacheSize = int(pb.CacheSize) + f.cacheSize = int(pb.CacheSize) // Copy cache type. f.cacheType = pb.CacheType @@ -306,7 +306,7 @@ func (f *Frame) saveMeta() error { RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, - CacheSize: int64(f.rankedCacheSize), + CacheSize: int64(f.cacheSize), }) if err != nil { return err @@ -605,7 +605,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheSize: int64(f.rankedCacheSize), + CacheSize: int64(f.cacheSize), }, } } diff --git a/frame_test.go b/frame_test.go index 16b935937..5ac6589bf 100644 --- a/frame_test.go +++ b/frame_test.go @@ -134,16 +134,16 @@ func TestFrame_SetCacheSize(t *testing.T) { cacheSize := 100 // Set & retrieve frame cache size. - if err := f.SetRankedCacheSize(cacheSize); err != nil { + if err := f.SetCacheSize(cacheSize); err != nil { t.Fatal(err) - } else if q := f.RankedCacheSize(); q != cacheSize { + } else if q := f.CacheSize(); q != cacheSize { t.Fatalf("unexpected frame cache size: %d", q) } // Reload frame and verify that it is persisted. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if q := f.RankedCacheSize(); q != cacheSize { + } else if q := f.CacheSize(); q != cacheSize { t.Fatalf("unexpected frame cache size (reopen): %d", q) } } From 3abfea140b953faa5cf6fe020217ce10efc2da37 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 21 Mar 2017 08:50:15 -0500 Subject: [PATCH 13/63] placeholder for `localNode` in HTTPNodeSet --- cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster.go b/cluster.go index bbdc8f51f..44cbe90cd 100644 --- a/cluster.go +++ b/cluster.go @@ -245,6 +245,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { nodes []*Node + localNode *Node // TODO: this needs to be set somewhere messageHandler func(m proto.Message) error // remoteStateHandler func(m proto.Message) error // localStateSource func() (proto.Message, error) @@ -279,6 +280,10 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { var g errgroup.Group for _, n := range h.nodes { + // Don't send the message to the local node. + if n == h.localNode { + continue + } node := n g.Go(func() error { return h.sendNodeMessage(node, buf) From 13e08ac3a546edce3da167bd394cfc246a712624 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 21 Mar 2017 16:06:22 -0500 Subject: [PATCH 14/63] add tests for `HTTPNodeSet` --- cluster.go | 6 +++- cluster_test.go | 6 ++-- gossip.go | 3 -- handler_test.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++ messenger_test.go | 37 +++++++++++++++++++ 5 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 messenger_test.go diff --git a/cluster.go b/cluster.go index 44cbe90cd..5d7502ed8 100644 --- a/cluster.go +++ b/cluster.go @@ -294,7 +294,11 @@ func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { // ReceiveMessage is called when a node receives a message. func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { - return h.messageHandler(pb) + if h.messageHandler != nil { + return h.messageHandler(pb) + } + // The messageHandler has not been set. + return nil } func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { diff --git a/cluster_test.go b/cluster_test.go index 2e9b01d89..4c3ec4134 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -109,9 +109,9 @@ func TestCluster_Health(t *testing.T) { // Verify a DOWN node is reported, and extraneous nodes are ignored if a := c.Health(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": "UP", - "serverB:1000": "DOWN", - "serverC:1000": "UP", + "serverA:1000": pilosa.HealthStatusUp, + "serverB:1000": pilosa.HealthStatusDown, + "serverC:1000": pilosa.HealthStatusUp, }) { t.Fatalf("unexpected health: %s", spew.Sdump(a)) } diff --git a/gossip.go b/gossip.go index e2b3ddcde..115918533 100644 --- a/gossip.go +++ b/gossip.go @@ -1,7 +1,6 @@ package pilosa import ( - "fmt" "io" "log" "os" @@ -124,8 +123,6 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { - loc := g.memberlist.LocalNode() - fmt.Println("Received Msg:", loc) m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) diff --git a/handler_test.go b/handler_test.go index c19a83a93..ab724b1f8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -868,3 +868,93 @@ func MustReadAll(r io.Reader) []byte { } return buf } + +type MessageBin struct { + Cluster *pilosa.Cluster + messageReceived proto.Message +} + +func NewMessageBin() *MessageBin { + return &MessageBin{} +} + +func (m *MessageBin) messageHandler(pb proto.Message) error { + m.messageReceived = pb + return nil +} + +func NewHTTPMessageBin(s *Server, nodes []*pilosa.Node) (*MessageBin, error) { + ns := pilosa.NewHTTPNodeSet() + mb := NewMessageBin() + ns.SetMessageHandler(mb.messageHandler) + c := pilosa.Cluster{ + Nodes: nodes, + NodeSet: ns, + } + mb.Cluster = &c + s.Handler.Cluster = &c + s.Handler.Messenger = ns + + i, err := c.NodeSet.Join(c.Nodes) + if i != int(0) { + return nil, err + } + if err != nil { + return nil, err + } + + return mb, nil +} + +// Ensure that an HTTP message sent to the cluster reaches all nodes. +func TestHTTPNodeSet_Base(t *testing.T) { + + // servers + s1 := NewServer() + s2 := NewServer() + s3 := NewServer() + nodes := []*pilosa.Node{ + {Host: s1.Host()}, + {Host: s2.Host()}, + {Host: s3.Host()}, + } + + // node 1 + mb1, err := NewHTTPMessageBin(s1, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // node2 + mb2, err := NewHTTPMessageBin(s2, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // node3 + mb3, err := NewHTTPMessageBin(s3, nodes) + if err != nil { + t.Fatalf("unable to create message bin: %s", err) + } + + // message + msg := &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + } + + // send message + if err := mb1.Cluster.NodeSet.(pilosa.Messenger).SendMessage(msg, ""); err != nil { + t.Fatalf("failure sending message: %s", err) + } + + if !reflect.DeepEqual(mb1.messageReceived, msg) { + t.Fatalf("unexpected message received by node1: %s", mb1.messageReceived) + } + if !reflect.DeepEqual(mb2.messageReceived, msg) { + t.Fatalf("unexpected message received by node2: %s", mb2.messageReceived) + } + if !reflect.DeepEqual(mb3.messageReceived, msg) { + t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) + } +} diff --git a/messenger_test.go b/messenger_test.go new file mode 100644 index 000000000..ee2d720a9 --- /dev/null +++ b/messenger_test.go @@ -0,0 +1,37 @@ +package pilosa_test + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" +) + +// Ensure a message can be marshaled and unmarshaled. +func TestMessage_Marshal(t *testing.T) { + + testMessageMarshal(t, &internal.CreateSliceMessage{ + DB: "d", + Slice: 8, + }) + + testMessageMarshal(t, &internal.DeleteDBMessage{ + DB: "d", + }) +} + +func testMessageMarshal(t *testing.T, m proto.Message) { + marshalled, err := pilosa.MarshalMessage(m) + if err != nil { + t.Fatal(err) + } + unmarshalled, err := pilosa.UnmarshalMessage(marshalled) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(unmarshalled, m) { + t.Fatalf("unexpected message marshalling: %s", unmarshalled) + } +} From cebca697b13b2bda1311e85f2dceb36fb48ea343 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 6 Apr 2017 14:28:45 -0500 Subject: [PATCH 15/63] fixing tests that were affected by the messenger/view merge --- cmd/server_test.go | 2 ++ fragment_test.go | 14 +++++++------- frame.go | 2 +- server.go | 8 +++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 927ce8b66..f1310abc2 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -106,6 +106,8 @@ data-dir = "` + actualDataDir + `" // confirm log file was written info, err := logFile.Stat() if err != nil || info.Size() == 0 { + // NOTE: this test assumes that something is being written to the log + // currently, that is relying on log: "index sync monitor initializing" return errors.New("Log file was not written!") } return nil diff --git a/fragment_test.go b/fragment_test.go index 7f3d35add..83f9b42be 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -288,7 +288,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", slice, cacheLimit), + Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", pilosa.ViewStandard, slice, cacheLimit), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -308,15 +308,15 @@ func TestFragment_TopN_CacheSize(t *testing.T) { f.RecalculateCache() p := []pilosa.Pair{ - {Key: 104, Count: 7}, - {Key: 103, Count: 6}} + {ID: 104, Count: 7}, + {ID: 103, Count: 6}} // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > cacheLimit { t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) - } else if pairs[0] != (pilosa.Pair{Key: 104, Count: 7}) { + } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) @@ -572,7 +572,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultFrameCache) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultCacheSize) if err := f.Open(); err != nil { b.Fatal(err) } @@ -633,7 +633,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultFrameCache), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultCacheSize), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -664,7 +664,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultFrameCache) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultCacheSize) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/frame.go b/frame.go index ebfe15ab0..a15b6ee63 100644 --- a/frame.go +++ b/frame.go @@ -428,7 +428,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { } func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.db, f.name, name) + view := NewView(path, f.db, f.name, name, f.cacheSize) view.cacheType = f.cacheType view.LogOutput = f.LogOutput view.BitmapAttrStore = f.bitmapAttrStore diff --git a/server.go b/server.go index 0df1182ba..5008e2d5c 100644 --- a/server.go +++ b/server.go @@ -120,11 +120,9 @@ func (s *Server) Open() error { go func() { http.Serve(ln, s.Handler) }() // Start background monitoring. - /* - s.wg.Add(2) - go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlices() }() - */ + s.wg.Add(2) + go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() return nil } From d99edc2eefa2ac4c262dafb3f38f54be0990a65b Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 6 Apr 2017 23:22:31 -0500 Subject: [PATCH 16/63] remove errant debugging line --- client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client.go b/client.go index d340ce640..e4401a69a 100644 --- a/client.go +++ b/client.go @@ -159,7 +159,6 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { case http.StatusOK: return nil // ok case http.StatusConflict: - fmt.Println("ErrDatabaseExists: 1") return ErrDatabaseExists default: return errors.New(string(body)) From e864a45d5715c407e9b5537710a75edb940e6aff Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Apr 2017 11:12:51 -0500 Subject: [PATCH 17/63] cleaned up the comments and code for the cache threshold. Also made the max size the set value of the cache rather than the calculated index. This ensures the user will not receive less cached values than they expect --- cache.go | 39 ++++++++++++++++++++++++++------------- fragment.go | 8 +------- fragment_test.go | 4 +++- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/cache.go b/cache.go index 23bc52c30..2fe4ea567 100644 --- a/cache.go +++ b/cache.go @@ -12,6 +12,11 @@ import ( "github.com/pilosa/pilosa/internal" ) +const ( + // ThresholdFactor is used to calculate the threshold for new items entering the cache + ThresholdFactor = 1.1 +) + // Cache represents a cache for bitmap counts. type Cache interface { Add(bitmapID uint64, n uint64) @@ -111,15 +116,23 @@ type RankCache struct { updateN int updateTime time.Time - ThresholdLength int - ThresholdIndex int - ThresholdValue uint64 + // CacheLimit is the user defined size of the cache + maxEntries int + + // thresholdBuffer is used the calculate the lowest cached threshold value + // This threshold determines what new items are added to the cache + thresholdBuffer int + + // thresholdValue is the value of the last item in the cache + thresholdValue uint64 } // NewRankCache returns a new instance of RankCache. -func NewRankCache() *RankCache { +func NewRankCache(maxEntries int) *RankCache { return &RankCache{ - entries: make(map[uint64]uint64), + maxEntries: maxEntries, + thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + entries: make(map[uint64]uint64), } } @@ -128,7 +141,7 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() // Ignore if the bit count on the bitmap is below the threshold. - if n < c.ThresholdValue { + if n < c.thresholdValue { return } @@ -141,7 +154,7 @@ func (c *RankCache) Add(bitmapID uint64, n uint64) { func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - if n < c.ThresholdValue { + if n < c.thresholdValue { return } @@ -209,20 +222,20 @@ func (c *RankCache) recalculate() { // Store the count of the item at the threshold index. c.rankings = rankings - if len(c.rankings) > c.ThresholdIndex { - c.ThresholdValue = rankings[c.ThresholdIndex].Count - c.rankings = c.rankings[0:c.ThresholdIndex] + if len(c.rankings) > c.maxEntries { + c.thresholdValue = rankings[c.maxEntries].Count + c.rankings = c.rankings[0:c.maxEntries] } else { - c.ThresholdValue = 1 + c.thresholdValue = 1 } // Reset counters. c.updateTime, c.updateN = time.Now(), 0 // If size is larger than the threshold then trim it. - if len(c.entries) > c.ThresholdLength { + if len(c.entries) > c.thresholdBuffer { for id, cnt := range c.entries { - if cnt <= c.ThresholdValue { + if cnt <= c.thresholdValue { delete(c.entries, id) } } diff --git a/fragment.go b/fragment.go index de188d0e8..1f4a03b92 100644 --- a/fragment.go +++ b/fragment.go @@ -43,9 +43,6 @@ const ( // HashBlockSize is the number of bitmaps in a merkle hash block. HashBlockSize = 100 - - // ThresholdBufferPct is the percentage of the ThresholdLength for which to maintain a sorted, ranked list. - ThresholdBufferPct = 0.9 ) const ( @@ -227,10 +224,7 @@ func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { case CacheTypeRanked: - c := NewRankCache() - c.ThresholdLength = f.cacheSize - c.ThresholdIndex = int(float64(f.cacheSize) * ThresholdBufferPct) - f.cache = c + f.cache = NewRankCache(f.cacheSize) case CacheTypeLRU: f.cache = NewLRUCache(f.cacheSize) default: diff --git a/fragment_test.go b/fragment_test.go index 83f9b42be..89bc660d3 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -309,7 +309,9 @@ func TestFragment_TopN_CacheSize(t *testing.T) { p := []pilosa.Pair{ {ID: 104, Count: 7}, - {ID: 103, Count: 6}} + {ID: 103, Count: 6}, + {ID: 102, Count: 5}, + } // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { From 8e2e9254f72d76e454d27a19c8e1e3ecbce01338 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Fri, 7 Apr 2017 13:41:40 -0500 Subject: [PATCH 18/63] fixed comment --- cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache.go b/cache.go index 2fe4ea567..19866eea1 100644 --- a/cache.go +++ b/cache.go @@ -116,7 +116,7 @@ type RankCache struct { updateN int updateTime time.Time - // CacheLimit is the user defined size of the cache + // maxEntries is the user defined size of the cache maxEntries int // thresholdBuffer is used the calculate the lowest cached threshold value From b0f1fc7523f2364613562922731fbc55dc15ba3f Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 12 Apr 2017 17:45:48 -0500 Subject: [PATCH 19/63] Makes Messenger a first-class object under Server (with pointers in Handler and Index). Primary message interface is the MessageBroker which is an attribute of the Messenger. MessageBroker implementations: - Gossip (memberlist) - Broadcast (uses HTTP, received by existing Handler) - Static (no-ops) Changes CacheSize from `int` to `uint32` for consistency with protobuf. Removes unnecessary dependencies in glide: - `github.com/aws/aws-sdk-go` - `golang.org/x/net` (although this gets included by memberlist) TODO: - [ ] Add tests around the Messenger and MessageBroker objects. - [ ] Refactor CreateSliceMessage to work with views. - [ ] Support propogation of meta data on PATCH calls. fixing some issues from last rebase --- cache.go | 10 +- cluster.go | 124 +------------------ cluster_test.go | 6 +- cmd/server.go | 2 +- config.go | 57 +++++++-- db.go | 8 +- fragment.go | 4 +- fragment_test.go | 4 +- frame.go | 32 ++--- frame_test.go | 2 +- glide.lock | 2 - glide.yaml | 1 - gossip.go | 305 +++++++++++++++++++++++++---------------------- handler.go | 11 +- handler_test.go | 81 ++++--------- index.go | 44 +------ index_test.go | 3 +- messenger.go | 262 ++++++++++++++++++++++++++++++++++++++-- server.go | 67 ++--------- server/server.go | 13 +- view.go | 4 +- 21 files changed, 544 insertions(+), 498 deletions(-) diff --git a/cache.go b/cache.go index 19866eea1..7da28eb74 100644 --- a/cache.go +++ b/cache.go @@ -44,9 +44,9 @@ type LRUCache struct { } // NewLRUCache returns a new instance of LRUCache. -func NewLRUCache(maxEntries int) *LRUCache { +func NewLRUCache(maxEntries uint32) *LRUCache { c := &LRUCache{ - cache: lru.New(maxEntries), + cache: lru.New(int(maxEntries)), counts: make(map[uint64]uint64), } c.cache.OnEvicted = c.onEvicted @@ -117,7 +117,7 @@ type RankCache struct { updateTime time.Time // maxEntries is the user defined size of the cache - maxEntries int + maxEntries uint32 // thresholdBuffer is used the calculate the lowest cached threshold value // This threshold determines what new items are added to the cache @@ -128,7 +128,7 @@ type RankCache struct { } // NewRankCache returns a new instance of RankCache. -func NewRankCache(maxEntries int) *RankCache { +func NewRankCache(maxEntries uint32) *RankCache { return &RankCache{ maxEntries: maxEntries, thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), @@ -222,7 +222,7 @@ func (c *RankCache) recalculate() { // Store the count of the item at the threshold index. c.rankings = rankings - if len(c.rankings) > c.maxEntries { + if len(c.rankings) > int(c.maxEntries) { c.thresholdValue = rankings[c.maxEntries].Count c.rankings = c.rankings[0:c.maxEntries] } else { diff --git a/cluster.go b/cluster.go index 5d7502ed8..a3678ad00 100644 --- a/cluster.go +++ b/cluster.go @@ -1,15 +1,8 @@ package pilosa import ( - "bytes" "encoding/binary" - "fmt" "hash/fnv" - "io/ioutil" - "net/http" - "net/url" - - "golang.org/x/sync/errgroup" "github.com/gogo/protobuf/proto" ) @@ -198,25 +191,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// NodeSet represents an interface to maintaining Node state. +// NodeSet represents an interface for Node membership and inter-node communication. type NodeSet interface { // Returns a list of all Nodes in the cluster Nodes() []*Node - // Attempts to join a cluster having `nodes` as its existing members - Join(nodes []*Node) (int, error) - // Open starts any network activity implemented by the NodeSet Open() error - - // SetMessageHandler provides the NodeSet with a function to call on ReceiveMessage - SetMessageHandler(f func(proto.Message) error) - - // SetRemoteStateHandler provides the function to call on MergeRemoteState - SetRemoteStateHandler(f func(proto.Message) error) - - // SetLocalStateSource provides the function to get the current node's local state. - SetLocalStateSource(f func() (proto.Message, error)) } // Hasher represents an interface to hash integers into buckets. @@ -244,11 +225,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { - nodes []*Node - localNode *Node // TODO: this needs to be set somewhere - messageHandler func(m proto.Message) error - // remoteStateHandler func(m proto.Message) error - // localStateSource func() (proto.Message, error) + nodes []*Node } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -260,103 +237,15 @@ func (h *HTTPNodeSet) Nodes() []*Node { return h.nodes } -func (h *HTTPNodeSet) Join(nodes []*Node) (int, error) { - h.nodes = nodes - return 0, nil -} - func (h *HTTPNodeSet) Open() error { return nil } -// SendMessage asyncronously broadcasts a protobuf message to all nodes. -func (h *HTTPNodeSet) SendMessage(pb proto.Message, method string) error { - - // Marshal the pb to []byte - buf, err := MarshalMessage(pb) - if err != nil { - return err - } - - var g errgroup.Group - for _, n := range h.nodes { - // Don't send the message to the local node. - if n == h.localNode { - continue - } - node := n - g.Go(func() error { - return h.sendNodeMessage(node, buf) - }) - } - return g.Wait() -} - -// ReceiveMessage is called when a node receives a message. -func (h *HTTPNodeSet) ReceiveMessage(pb proto.Message) error { - if h.messageHandler != nil { - return h.messageHandler(pb) - } - // The messageHandler has not been set. +func (h *HTTPNodeSet) Join(nodes []*Node) error { + h.nodes = nodes return nil } -func (h *HTTPNodeSet) sendNodeMessage(node *Node, msg []byte) error { - var client *http.Client - client = http.DefaultClient - - // Create HTTP request. - req, err := http.NewRequest("POST", (&url.URL{ - Scheme: "http", - Host: node.Host, - Path: "/message", - }).String(), bytes.NewReader(msg)) - if err != nil { - return err - } - - // Require protobuf encoding. - req.Header.Set("Content-Type", "application/x-protobuf") - - // Send request to remote node. - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - // Read response into buffer. - body, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return err - } - - // Check status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) - } - - return nil -} - -// SetMessageHandler provides the Messenger with a function to handle incoming messages. -func (h *HTTPNodeSet) SetMessageHandler(f func(proto.Message) error) { - h.messageHandler = f -} - -// SetRemoteStateHandler provides the Messenger with a function to merge remote state. -func (h *HTTPNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - // not implemented - // h.remoteStateHandler = f -} - -// SetLocalStateSource currently no-ops. -func (h *HTTPNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - // not implemented - // h.localStateSource = f -} - // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { Messenger @@ -371,11 +260,6 @@ func (s *StaticNodeSet) Nodes() []*Node { return s.nodes } -func (s *StaticNodeSet) Join(nodes []*Node) (int, error) { - s.nodes = nodes - return 0, nil -} - func (s *StaticNodeSet) Open() error { return nil } diff --git a/cluster_test.go b/cluster_test.go index 4c3ec4134..1dd6bb486 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -95,16 +95,16 @@ func TestCluster_Health(t *testing.T) { {Host: "serverB:1000"}, {Host: "serverC:1000"}, }, - NodeSet: &pilosa.StaticNodeSet{}, + NodeSet: &pilosa.HTTPNodeSet{}, } - j, err := c.NodeSet.Join([]*pilosa.Node{ + err := c.NodeSet.(*pilosa.HTTPNodeSet).Join([]*pilosa.Node{ &pilosa.Node{Host: "serverA:1000"}, &pilosa.Node{Host: "serverC:1000"}, &pilosa.Node{Host: "serverD:1000"}, }) if err != nil { - t.Fatalf("unexpected gossiper nodes: %s", j) + t.Fatalf("unexpected gossiper nodes: %s", err) } // Verify a DOWN node is reported, and extraneous nodes are ignored diff --git a/cmd/server.go b/cmd/server.go index d25066764..27c238eb6 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "", "Type of Messenger to use for inter-host messaging.") + flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "static", "Type of Messenger to use for inter-host messaging. Choose from [static, broadcast, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 17edd639d..b74125ce7 100644 --- a/config.go +++ b/config.go @@ -2,14 +2,16 @@ package pilosa import ( "net" + "strconv" "time" ) const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultGossipPort = 14000 + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultMessengerType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -47,6 +49,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN + c.Cluster.MessengerType = DefaultMessengerType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) @@ -63,6 +66,24 @@ func NewConfigForHosts(hosts []string) *Config { return conf } +// PilosaMessenger returns a new instance of Messenger based on the config. +func (c *Config) PilosaMessenger() *Messenger { + messenger := NewMessenger() + switch c.Cluster.MessengerType { + case "broadcast": + n := NewHTTPMessageBroker() + n.messenger = messenger + messenger.Broker = n + case "gossip": + n := NewGossipMessageBroker() + n.messenger = messenger + messenger.Broker = n + case "static": + // nop + } + return messenger +} + // PilosaCluster returns a new instance of Cluster based on the config. func (c *Config) PilosaCluster() *Cluster { cluster := NewCluster() @@ -73,11 +94,16 @@ func (c *Config) PilosaCluster() *Cluster { } // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - if c.Cluster.MessengerType == "broadcast" { + switch c.Cluster.MessengerType { + case "broadcast": cluster.NodeSet = NewHTTPNodeSet() - cluster.NodeSet.Join(cluster.Nodes) - } else if c.Cluster.MessengerType == "gossip" { - gossipPort := DefaultGossipPort + cluster.NodeSet.(*HTTPNodeSet).Join(cluster.Nodes) + case "gossip": + gport, err := strconv.Atoi(DefaultGossipPort) + if err != nil { + // what? + } + gossipPort := gport gossipSeed := DefaultHost if c.Cluster.Gossip.Port != 0 { gossipPort = c.Cluster.Gossip.Port @@ -91,13 +117,28 @@ func (c *Config) PilosaCluster() *Cluster { gossipHost = c.Host } cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - } else { + case "static": + cluster.NodeSet = NewStaticNodeSet() + default: cluster.NodeSet = NewStaticNodeSet() } return cluster } +// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// after cluster configuration. +func (c *Config) AssociateMessageBroker(s *Server) { + switch c.Cluster.MessengerType { + case "broadcast": + // nop + case "gossip": + s.Cluster.NodeSet.(*GossipNodeSet).config.memberlistConfig.Delegate = s.Messenger.Broker.(*GossipMessageBroker) + case "static": + // nop + } +} + // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/db.go b/db.go index a7f6f8ff1..cc27e18c9 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,7 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - messenger Messenger + messenger *Messenger stats StatsClient LogOutput io.Writer @@ -68,7 +68,6 @@ func NewDB(path, name string) (*DB, error) { columnLabel: DefaultColumnLabel, - messenger: NopMessenger, stats: NopStatsClient, LogOutput: ioutil.Discard, }, nil @@ -394,10 +393,7 @@ func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { f.rowLabel = opt.RowLabel } if opt.CacheSize != 0 { - f.rankedCacheSize = opt.CacheSize - } - if opt.TimeQuantum.Valid() { - f.timeQuantum = opt.TimeQuantum + f.cacheSize = opt.CacheSize } f.inverseEnabled = opt.InverseEnabled diff --git a/fragment.go b/fragment.go index 1f4a03b92..5704e774d 100644 --- a/fragment.go +++ b/fragment.go @@ -70,7 +70,7 @@ type Fragment struct { // Cache for bitmap counts. cacheType string // passed in by frame cache Cache - cacheSize int + cacheSize uint32 // Cache containing full bitmaps (not just counts). bitmapCache BitmapCache @@ -94,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64, cacheSize int) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) *Fragment { return &Fragment{ path: path, db: db, diff --git a/fragment_test.go b/fragment_test.go index 89bc660d3..e54b11f82 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -280,7 +280,7 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) - cacheLimit := 3 + cacheLimit := uint32(3) file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -316,7 +316,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) > cacheLimit { + } else if len(pairs) > int(cacheLimit) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) diff --git a/frame.go b/frame.go index a15b6ee63..c5a40b1d1 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,7 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - messenger Messenger + messenger *Messenger stats StatsClient // Frame settings. @@ -47,7 +47,7 @@ type Frame struct { inverseEnabled bool // Cache size for ranked frames - cacheSize int + cacheSize uint32 LogOutput io.Writer } @@ -67,8 +67,7 @@ func NewFrame(path, db, name string) (*Frame, error) { views: make(map[string]*View), bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), - messenger: NopMessenger, - stats: NopStatsClient, + stats: NopStatsClient, rowLabel: DefaultRowLabel, inverseEnabled: DefaultInverseEnabled, @@ -160,7 +159,7 @@ func (f *Frame) InverseEnabled() bool { // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 -func (f *Frame) SetCacheSize(v int) error { +func (f *Frame) SetCacheSize(v uint32) error { f.mu.Lock() defer f.mu.Unlock() @@ -179,7 +178,7 @@ func (f *Frame) SetCacheSize(v int) error { } // CacheSize returns the ranked frame cache size. -func (f *Frame) CacheSize() int { +func (f *Frame) CacheSize() uint32 { f.mu.Lock() v := f.cacheSize f.mu.Unlock() @@ -194,6 +193,7 @@ func (f *Frame) Options() FrameOptions { InverseEnabled: f.inverseEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, + TimeQuantum: f.timeQuantum, } f.mu.Unlock() return opt @@ -287,7 +287,7 @@ func (f *Frame) loadMeta() error { f.timeQuantum = TimeQuantum(pb.TimeQuantum) f.rowLabel = pb.RowLabel f.inverseEnabled = pb.InverseEnabled - f.cacheSize = int(pb.CacheSize) + f.cacheSize = pb.CacheSize // Copy cache type. f.cacheType = pb.CacheType @@ -301,12 +301,12 @@ func (f *Frame) loadMeta() error { // saveMeta writes meta data for the frame. func (f *Frame) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.Frame{ + buf, err := proto.Marshal(&internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, - CacheSize: int64(f.cacheSize), + CacheSize: f.cacheSize, }) if err != nil { return err @@ -414,16 +414,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { view.BitmapAttrStore = f.bitmapAttrStore f.views[view.Name()] = view - // TODO: this needs to be refactored for views - /* - // Send a MaxSlice message - f.messenger.SendMessage( - &internal.CreateSliceMessage{ - DB: f.db, - Slice: slice, - }, "gossip") - */ - return view, nil } @@ -605,7 +595,7 @@ func encodeFrame(f *Frame) *internal.Frame { Meta: &internal.FrameMeta{ TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheSize: int64(f.cacheSize), + CacheSize: f.cacheSize, }, } } @@ -633,7 +623,7 @@ type FrameOptions struct { RowLabel string `json:"rowLabel,omitempty"` InverseEnabled bool `json:"inverseEnabled,omitempty"` CacheType string `json:"cacheType,omitempty"` - CacheSize int `json:"cacheSize,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } diff --git a/frame_test.go b/frame_test.go index 5ac6589bf..4814ca67e 100644 --- a/frame_test.go +++ b/frame_test.go @@ -131,7 +131,7 @@ func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time func TestFrame_SetCacheSize(t *testing.T) { f := MustOpenFrame() defer f.Close() - cacheSize := 100 + cacheSize := uint32(100) // Set & retrieve frame cache size. if err := f.SetCacheSize(cacheSize); err != nil { diff --git a/glide.lock b/glide.lock index e4b6ce617..ae504264c 100644 --- a/glide.lock +++ b/glide.lock @@ -3,8 +3,6 @@ updated: 2017-04-18T15:33:39.035615802-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 -- name: github.com/aws/aws-sdk-go - version: 819b71cf8430e434c1eee7e7e8b0f2b8870be899 - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml diff --git a/glide.yaml b/glide.yaml index f317a0a1b..d81cbaf28 100644 --- a/glide.yaml +++ b/glide.yaml @@ -35,4 +35,3 @@ import: version: ^1.6.10 - package: github.com/hashicorp/memberlist - package: golang.org/x/sync -- package: golang.org/x/net diff --git a/gossip.go b/gossip.go index 115918533..b25b634dd 100644 --- a/gossip.go +++ b/gossip.go @@ -16,14 +16,9 @@ import ( // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist - broadcasts *memberlist.TransmitLimitedQueue config *GossipConfig - messageHandler func(m proto.Message) error - remoteStateHandler func(m proto.Message) error - localStateSource func() (proto.Message, error) - // The writer for any logging. LogOutput io.Writer } @@ -36,10 +31,6 @@ func (g *GossipNodeSet) Nodes() []*Node { return a } -func (g *GossipNodeSet) Join(nodes []*Node) (int, error) { - return g.memberlist.Join(Nodes(nodes).Hosts()) -} - func (g *GossipNodeSet) Open() error { ml, err := memberlist.Create(g.config.memberlistConfig) if err != nil { @@ -48,152 +39,19 @@ func (g *GossipNodeSet) Open() error { g.memberlist = ml // attach to gossip seed node - g.Join([]*Node{&Node{Host: g.config.gossipSeed}}) //TODO: support a list of seeds - - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return g.memberlist.NumMembers() - }, - RetransmitMult: 3, - } - return nil -} - -func (g *GossipNodeSet) SetMessageHandler(f func(proto.Message) error) { - g.messageHandler = f -} - -func (g *GossipNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - g.remoteStateHandler = f -} - -func (g *GossipNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - g.localStateSource = f -} - -// implementation of the messenger.Messenger interface -func (g *GossipNodeSet) SendMessage(pb proto.Message, method string) error { - msg, err := MarshalMessage(pb) - if err != nil { - return err - } - - // Broadcast asyncronously sends the message directly to each node. - // An error from any node raises an error on the entire operation. - // This is a blocking operation. - // - // Gossip uses the gossip protocol to eventually deliver the message - // to every node. - switch method { - case "broadcast": - var eg errgroup.Group - for _, n := range g.memberlist.Members() { - // Don't send the message to the local node. - if n == g.memberlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return g.memberlist.SendToTCP(node, msg) - }) - } - return eg.Wait() - case "gossip": - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) - } - - return nil -} - -func (g *GossipNodeSet) ReceiveMessage(pb proto.Message) error { - err := g.messageHandler(pb) + nodes := []*Node{&Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds + _, err = g.memberlist.Join(Nodes(nodes).Hosts()) if err != nil { return err } return nil } -// implementation of the memberlist.Delegate interface -func (g *GossipNodeSet) NodeMeta(limit int) []byte { - return []byte{} -} - -func (g *GossipNodeSet) NotifyMsg(b []byte) { - m, err := UnmarshalMessage(b) - if err != nil { - g.logger().Printf("unmarshal message error: %s", err) - return - } - if err := g.ReceiveMessage(m); err != nil { - g.logger().Printf("receive message error: %s", err) - return - } -} - -func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { - return g.broadcasts.GetBroadcasts(overhead, limit) -} - -func (g *GossipNodeSet) LocalState(join bool) []byte { - - pb, err := g.localStateSource() - if err != nil { - g.logger().Printf("error getting local state, err=%s", err) - return []byte{} - } - - // Marshal nodestate data to bytes. - buf, err := proto.Marshal(pb) - if err != nil { - g.logger().Printf("error marshaling nodestate data, err=%s", err) - return []byte{} - } - return buf -} - -func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { - // Unmarshal nodestate data. - var pb internal.NodeState - if err := proto.Unmarshal(buf, &pb); err != nil { - g.logger().Printf("error unmarshaling nodestate data, err=%s", err) - return - } - err := g.remoteStateHandler(&pb) - if err != nil { - g.logger().Printf("merge state error: %s", err) - } - return -} - // logger returns a logger for the GossipNodeSet. func (g *GossipNodeSet) logger() *log.Logger { return log.New(g.LogOutput, "", log.LstdFlags) } -// broadcast represents an implementation of memberlist.Broadcast -type broadcast struct { - msg []byte - notify chan<- struct{} -} - -func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { - return false -} - -func (b *broadcast) Message() []byte { - return b.msg -} - -func (b *broadcast) Finished() { - if b.notify != nil { - close(b.notify) - } -} - //////////////////////////////////////////////////////////////// type GossipConfig struct { @@ -217,7 +75,164 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort - g.config.memberlistConfig.Delegate = g return g } + +//////////////////////////////////////////////////////////////// + +// GossipMessageBroker represents a gossip implementation of pilosa.MessageBroker +// GossipMessageBroker also represents an implementation of memberlist.Delegate +type GossipMessageBroker struct { + broadcasts *memberlist.TransmitLimitedQueue + + messenger *Messenger + + // The writer for any logging. + LogOutput io.Writer +} + +// implementation of the messenger.Messenger interface +func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err + } + + mlist := g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist + + // Direct sends the message directly to every node. + // An error from any node raises an error on the entire operation. + // + // Gossip uses the gossip protocol to eventually deliver the message + // to every node. + switch method { + case "direct": + var eg errgroup.Group + for _, n := range mlist.Members() { + // Don't send the message to the local node. + if n == mlist.LocalNode() { + continue + } + node := n + eg.Go(func() error { + return mlist.SendToTCP(node, msg) + }) + } + return eg.Wait() + case "gossip": + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) + } + + return nil +} + +func (g *GossipMessageBroker) Receive(pb proto.Message) error { + if err := g.messenger.ReceiveMessage(pb); err != nil { + return err + } + return nil +} + +func (g *GossipMessageBroker) SetMessenger(m *Messenger) { + g.messenger = m +} + +// implementation of the memberlist.Delegate interface +func (g *GossipMessageBroker) NodeMeta(limit int) []byte { + return []byte{} +} + +func (g *GossipMessageBroker) NotifyMsg(b []byte) { + m, err := UnmarshalMessage(b) + if err != nil { + g.logger().Printf("unmarshal message error: %s", err) + return + } + if err := g.Receive(m); err != nil { + g.logger().Printf("receive message error: %s", err) + return + } +} + +func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { + return g.broadcasts.GetBroadcasts(overhead, limit) +} + +func (g *GossipMessageBroker) LocalState(join bool) []byte { + pb, err := g.messenger.LocalState() + if err != nil { + g.logger().Printf("error getting local state, err=%s", err) + return []byte{} + } + + // Marshal nodestate data to bytes. + buf, err := proto.Marshal(pb) + if err != nil { + g.logger().Printf("error marshalling nodestate data, err=%s", err) + return []byte{} + } + return buf +} + +func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { + // Unmarshal nodestate data. + var pb internal.NodeState + if err := proto.Unmarshal(buf, &pb); err != nil { + g.logger().Printf("error unmarshalling nodestate data, err=%s", err) + return + } + err := g.messenger.HandleRemoteState(&pb) + if err != nil { + g.logger().Printf("merge state error: %s", err) + } +} + +// logger returns a logger for the GossipMessageBroker. +func (g *GossipMessageBroker) logger() *log.Logger { + return log.New(g.LogOutput, "", log.LstdFlags) +} + +//////////////////////////////////////////////////////////////// + +// NewGossipMessageBroker returns a new instance of GossipMessageBroker. +func NewGossipMessageBroker() *GossipMessageBroker { + g := &GossipMessageBroker{ + LogOutput: os.Stderr, + } + + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() + }, + RetransmitMult: 3, + } + + return g +} + +//////////////////////////////////////////////////////////////// + +// broadcast represents an implementation of memberlist.Broadcast +type broadcast struct { + msg []byte + notify chan<- struct{} +} + +func (b *broadcast) Invalidates(other memberlist.Broadcast) bool { + return false +} + +func (b *broadcast) Message() []byte { + return b.msg +} + +func (b *broadcast) Finished() { + if b.notify != nil { + close(b.notify) + } +} diff --git a/handler.go b/handler.go index ad337b2bc..0a7cb278d 100644 --- a/handler.go +++ b/handler.go @@ -26,7 +26,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { Index *Index - Messenger Messenger + Messenger *Messenger // Local hostname & cluster configuration. Host string @@ -50,7 +50,6 @@ type Handler struct { func NewHandler() *Handler { handler := &Handler{ LogOutput: os.Stderr, - Messenger: NopMessenger, } handler.Router = NewRouter(handler) return handler @@ -214,7 +213,7 @@ func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { return } - if err := h.Messenger.ReceiveMessage(m); err != nil { + if err := h.Messenger.Broker.Receive(m); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -373,7 +372,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { err := h.Messenger.SendMessage( &internal.DeleteDBMessage{ DB: req.DB, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) } @@ -522,7 +521,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), }, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) } @@ -590,7 +589,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, - }, "broadcast") + }, "direct") if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) } diff --git a/handler_test.go b/handler_test.go index ab724b1f8..92bcc7601 100644 --- a/handler_test.go +++ b/handler_test.go @@ -792,6 +792,10 @@ func NewHandler() *Handler { } h.Handler.Executor = &h.Executor h.Handler.LogOutput = ioutil.Discard + + // Handler test messages can no-op. + h.Messenger = pilosa.NewMessenger() + return h } @@ -823,6 +827,9 @@ func NewServer() *Server { // Update handler to use hostname. s.Handler.Host = s.Host() + // Handler test messages can no-op. + s.Handler.Messenger = pilosa.NewMessenger() + // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() @@ -869,74 +876,37 @@ func MustReadAll(r io.Reader) []byte { return buf } -type MessageBin struct { - Cluster *pilosa.Cluster - messageReceived proto.Message -} - -func NewMessageBin() *MessageBin { - return &MessageBin{} -} - -func (m *MessageBin) messageHandler(pb proto.Message) error { - m.messageReceived = pb - return nil -} - -func NewHTTPMessageBin(s *Server, nodes []*pilosa.Node) (*MessageBin, error) { - ns := pilosa.NewHTTPNodeSet() - mb := NewMessageBin() - ns.SetMessageHandler(mb.messageHandler) - c := pilosa.Cluster{ - Nodes: nodes, - NodeSet: ns, - } - mb.Cluster = &c - s.Handler.Cluster = &c - s.Handler.Messenger = ns - - i, err := c.NodeSet.Join(c.Nodes) - if i != int(0) { - return nil, err - } - if err != nil { - return nil, err - } - - return mb, nil -} +/* +// TODO: move this test to messenger.go (with NewServer()) // Ensure that an HTTP message sent to the cluster reaches all nodes. func TestHTTPNodeSet_Base(t *testing.T) { // servers s1 := NewServer() + s1.Messenger = pilosa.NewMessenger() + n1 := NewHTTPMessageBroker() + n1.messenger = s1.Messenger + s1.Messenger.Broker = n1 + s2 := NewServer() + s2.Messenger = pilosa.NewMessenger() + n2 := NewHTTPMessageBroker() + n2.messenger = s2.Messenger + s2.Messenger.Broker = n2 + s3 := NewServer() + s3.Messenger = pilosa.NewMessenger() + n3 := NewHTTPMessageBroker() + n3.messenger = s3.Messenger + s3.Messenger.Broker = n3 + nodes := []*pilosa.Node{ {Host: s1.Host()}, {Host: s2.Host()}, {Host: s3.Host()}, } - // node 1 - mb1, err := NewHTTPMessageBin(s1, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - - // node2 - mb2, err := NewHTTPMessageBin(s2, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - - // node3 - mb3, err := NewHTTPMessageBin(s3, nodes) - if err != nil { - t.Fatalf("unable to create message bin: %s", err) - } - // message msg := &internal.CreateSliceMessage{ DB: "d", @@ -944,7 +914,7 @@ func TestHTTPNodeSet_Base(t *testing.T) { } // send message - if err := mb1.Cluster.NodeSet.(pilosa.Messenger).SendMessage(msg, ""); err != nil { + if err := s1.Messenger.SendMessage(msg, ""); err != nil { t.Fatalf("failure sending message: %s", err) } @@ -958,3 +928,4 @@ func TestHTTPNodeSet_Base(t *testing.T) { t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) } } +*/ diff --git a/index.go b/index.go index 954629eea..5e9f5187e 100644 --- a/index.go +++ b/index.go @@ -11,9 +11,6 @@ import ( "sort" "sync" "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. @@ -26,7 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - Messenger Messenger + Messenger *Messenger // Close management wg sync.WaitGroup @@ -50,8 +47,7 @@ func NewIndex() *Index { dbs: make(map[string]*DB), closing: make(chan struct{}, 0), - Messenger: NopMessenger, - Stats: NopStatsClient, + Stats: NopStatsClient, CacheFlushInterval: DefaultCacheFlushInterval, @@ -347,42 +343,6 @@ func (i *Index) flushCaches() { } } -// HandleMessage handles protobuf Messages broadcasted to nodes in the -// cluster from the Cluster's NodeSet. -func (i *Index) HandleMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateSliceMessage: - d := i.DB(obj.DB) - if d == nil { - return fmt.Errorf("Local DB not found: %s", obj.DB) - } - d.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := i.CreateDB(obj.DB, opt) - if err != nil { - return err - } - case *internal.DeleteDBMessage: - if err := i.DeleteDB(obj.DB); err != nil { - return err - } - case *internal.CreateFrameMessage: - db := i.DB(obj.DB) - opt := FrameOptions{RowLabel: obj.Meta.RowLabel} - _, err := db.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } - case *internal.DeleteFrameMessage: - db := i.DB(obj.DB) - if err := db.DeleteFrame(obj.Frame); err != nil { - return err - } - } - return nil -} - func (i *Index) logger() *log.Logger { return log.New(i.LogOutput, "", log.LstdFlags) } // IndexSyncer is an active anti-entropy tool that compares the local index diff --git a/index_test.go b/index_test.go index 50ce1979e..c09fe69f5 100644 --- a/index_test.go +++ b/index_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -163,6 +162,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } +/* TODO: move this to messenger.go // Ensure index can handle Messenger messages. func TestIndex_HandleMessage(t *testing.T) { // Create a local index. @@ -188,6 +188,7 @@ func TestIndex_HandleMessage(t *testing.T) { t.Fatalf("unexpected delete db: %s", ms) } } +*/ // Index is a test wrapper for pilosa.Index. type Index struct { diff --git a/messenger.go b/messenger.go index f28eb7ddb..03e15ba0e 100644 --- a/messenger.go +++ b/messenger.go @@ -1,36 +1,276 @@ package pilosa import ( + "bytes" + "errors" "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" "reflect" + "golang.org/x/sync/errgroup" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) +// Messenger represents an internal message handler. +type Messenger struct { + + // Broker handles Send/Receive Messages. + Broker MessageBroker + + Index *Index + + // Local hostname & cluster configuration. + Host string + Cluster *Cluster + + // The writer for any logging. + LogOutput io.Writer +} + +// NewMessenger returns a new instance of Messenger with a default logger. +func NewMessenger() *Messenger { + return &Messenger{ + Broker: NopMessageBroker, + LogOutput: os.Stderr, + } +} + +func (m *Messenger) SendMessage(pb proto.Message, method string) error { + if m.Broker == nil { + return errors.New("Messenger.Broker is not defined.") + } + return m.Broker.Send(pb, method) +} +func (m *Messenger) ReceiveMessage(pb proto.Message) error { + return m.handleMessage(pb) +} + +// handleMessage handles protobuf Messages sent to nodes in the cluster. +func (m *Messenger) handleMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := m.Index.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := m.Index.CreateDB(obj.DB, opt) + if err != nil { + return err + } + case *internal.DeleteDBMessage: + fmt.Println("DELETE:", obj.DB) + if err := m.Index.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := m.Index.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) + if err != nil { + return err + } + case *internal.DeleteFrameMessage: + db := m.Index.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } + } + return nil +} + +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +// It seems odd to have this as part of Messenger, but with the +// exception of Server, it's currenntly the only object with access +// to the necessary information (Host, Index, Cluster). +func (m *Messenger) LocalState() (proto.Message, error) { + if m.Index == nil { + return nil, errors.New("Messenger.Index is nil.") + } + return &internal.NodeState{ + Host: m.Host, + State: "OK", // TODO: make this work, pull from m.Cluster.Node + DBs: encodeDBs(m.Index.DBs()), + }, nil +} + +// HandleRemoteState receives incoming NodeState from remote nodes. +func (m *Messenger) HandleRemoteState(pb proto.Message) error { + return m.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := m.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + CacheSize: f.Meta.CacheSize, + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + +////////////////////////////////////////////////////////////////// + +// MessageBroker is an interface for handling incoming/outgoing messages. +type MessageBroker interface { + Send(pb proto.Message, method string) error + Receive(pb proto.Message) error + SetMessenger(m *Messenger) +} + +////////////////////////////////////////////////////////////////// + func init() { - NopMessenger = &nopMessenger{} + NopMessageBroker = &nopMessageBroker{} } -var NopMessenger Messenger +var NopMessageBroker MessageBroker -// nopMessenger represents a Messenger that doesn't do anything. -type nopMessenger struct{} +// nopMessageBroker represents a MessageBroker that doesn't do anything. +type nopMessageBroker struct{} -func (c *nopMessenger) SendMessage(pb proto.Message, method string) error { - fmt.Println("NOPMessenger: Send") +func (c *nopMessageBroker) Send(pb proto.Message, method string) error { + fmt.Println("NOPMessageBroker: Send") return nil } -func (c *nopMessenger) ReceiveMessage(pb proto.Message) error { - fmt.Println("NOPMessenger: Receive") +func (c *nopMessageBroker) Receive(pb proto.Message) error { + fmt.Println("NOPMessageBroker: Receive") + return nil +} +func (c *nopMessageBroker) SetMessenger(m *Messenger) {} + +////////////////////////////////////////////////////////////////// + +// HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. +type HTTPMessageBroker struct { + messenger *Messenger +} + +// NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. +func NewHTTPMessageBroker() *HTTPMessageBroker { + return &HTTPMessageBroker{} +} + +// Send sends a protobuf message to all nodes simultaneously. +// It waits for all nodes to respond before the function returns (and returns any errors). +func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { + // Marshal the pb to []byte + buf, err := MarshalMessage(pb) + if err != nil { + return err + } + + nodes, err := h.nodes() + if err != nil { + return err + } + + var g errgroup.Group + for _, n := range nodes { + // Don't send the message to the local node. + if n.Host == h.messenger.Host { + continue + } + node := n + g.Go(func() error { + return h.sendNodeMessage(node, buf) + }) + } + return g.Wait() +} + +// Receive is called when a node receives a message. +func (h *HTTPMessageBroker) Receive(pb proto.Message) error { + if err := h.messenger.ReceiveMessage(pb); err != nil { + return err + } return nil } -type Messenger interface { - SendMessage(pb proto.Message, method string) error - ReceiveMessage(pb proto.Message) error +func (h *HTTPMessageBroker) SetMessenger(m *Messenger) {} + +func (h *HTTPMessageBroker) nodes() ([]*Node, error) { + if h.messenger == nil { + return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") + } + nodeset, ok := h.messenger.Cluster.NodeSet.(*HTTPNodeSet) + if !ok { + return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet.") + } + return nodeset.Nodes(), nil } +func (h *HTTPMessageBroker) sendNodeMessage(node *Node, msg []byte) error { + var client *http.Client + client = http.DefaultClient + + // Create HTTP request. + req, err := http.NewRequest("POST", (&url.URL{ + Scheme: "http", + Host: node.Host, + Path: "/message", + }).String(), bytes.NewReader(msg)) + if err != nil { + return err + } + + // Require protobuf encoding. + req.Header.Set("Content-Type", "application/x-protobuf") + + // Send request to remote node. + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read response into buffer. + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + // Check status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) + } + + return nil +} + +////////////////////////////////////////////////////////////////// + const ( MessageTypeCreateSlice = 1 MessageTypeCreateDB = 2 diff --git a/server.go b/server.go index 5008e2d5c..a93d29f57 100644 --- a/server.go +++ b/server.go @@ -34,7 +34,7 @@ type Server struct { // Data storage and HTTP interface. Index *Index Handler *Handler - Messenger Messenger + Messenger *Messenger // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -55,7 +55,7 @@ func NewServer() *Server { Index: NewIndex(), Handler: NewHandler(), - Messenger: NopMessenger, + Messenger: NewMessenger(), AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -64,6 +64,7 @@ func NewServer() *Server { } s.Handler.Index = s.Index + s.Messenger.Index = s.Index return s } @@ -109,11 +110,21 @@ func (s *Server) Open() error { e.Host = s.Host e.Cluster = s.Cluster + // Initialize Messenger. + s.Messenger.Index = s.Index + s.Messenger.Host = s.Host + s.Messenger.Cluster = s.Cluster + s.Messenger.LogOutput = s.LogOutput + // Initialize HTTP handler. + s.Handler.Messenger = s.Messenger s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput + + // Initialize Index. + s.Index.Messenger = s.Messenger s.Index.LogOutput = s.LogOutput // Serve HTTP. @@ -151,49 +162,6 @@ func (s *Server) Addr() net.Addr { return s.ln.Addr() } -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -func (s *Server) LocalState() (proto.Message, error) { - // TODO: are there errors to handle? - pb := encodeLocalState(s) - return pb, nil -} - -// HandleRemoteState provides the current, local state. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) HandleRemoteState(pb proto.Message) error { - return s.mergeRemoteState(pb.(*internal.NodeState)) -} - -func (s *Server) mergeRemoteState(ns *internal.NodeState) error { - // TODO: update some node state value in the cluster (it should be in cluster.node i guess) - - // Create databases that don't exist. - for _, db := range ns.DBs { - opt := DBOptions{ - ColumnLabel: db.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), - } - d, err := s.Index.CreateDBIfNotExists(db.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range db.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - } - _, err := d.CreateFrameIfNotExists(f.Name, opt) - if err != nil { - return err - } - } - } - - return nil -} - func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) } func (s *Server) monitorAntiEntropy() { @@ -230,15 +198,6 @@ func (s *Server) monitorAntiEntropy() { } } -// encodeLocalState converts s into its internal representation. -func encodeLocalState(s *Server) *internal.NodeState { - return &internal.NodeState{ - Host: s.Host, - State: "OK", // TODO: make this work, pull from cluster.Node - DBs: encodeDBs(s.Index.DBs()), - } -} - // monitorMaxSlices periodically pulls the highest slice from each node in the cluster. func (s *Server) monitorMaxSlices() { // Ignore if only one node in the cluster. diff --git a/server/server.go b/server/server.go index 3a62c5c60..ba41f3dd0 100644 --- a/server/server.go +++ b/server/server.go @@ -92,18 +92,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } + m.Server.Messenger = m.Config.PilosaMessenger() m.Server.Cluster = m.Config.PilosaCluster() - // Setup Messenger. - fmt.Fprintf(m.Stderr, "Using Messenger type: %s\n", m.Config.Cluster.MessengerType) - m.Server.Messenger = m.Server.Cluster.NodeSet.(pilosa.Messenger) - m.Server.Handler.Messenger = m.Server.Messenger - m.Server.Index.Messenger = m.Server.Messenger - - // Set message and state handlers. - m.Server.Cluster.NodeSet.SetMessageHandler(m.Server.Index.HandleMessage) - m.Server.Cluster.NodeSet.SetRemoteStateHandler(m.Server.HandleRemoteState) - m.Server.Cluster.NodeSet.SetLocalStateSource(m.Server.LocalState) + // Associate objects to the MessageBroker based on config. + m.Config.AssociateMessageBroker(m.Server) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) diff --git a/view.go b/view.go index 0f554de92..9ded867ce 100644 --- a/view.go +++ b/view.go @@ -30,7 +30,7 @@ type View struct { frame string name string - cacheSize int + cacheSize uint32 // Fragments by slice. cacheType string // passed in by frame @@ -43,7 +43,7 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string, cacheSize int) *View { +func NewView(path, db, frame, name string, cacheSize uint32) *View { return &View{ path: path, db: db, From 5cc4080ccfbd7985075bf3645bddb2336c14fab4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 14:43:22 -0500 Subject: [PATCH 20/63] remove unused SetMessenger method --- gossip.go | 4 ---- messenger.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/gossip.go b/gossip.go index b25b634dd..9b18b45dd 100644 --- a/gossip.go +++ b/gossip.go @@ -138,10 +138,6 @@ func (g *GossipMessageBroker) Receive(pb proto.Message) error { return nil } -func (g *GossipMessageBroker) SetMessenger(m *Messenger) { - g.messenger = m -} - // implementation of the memberlist.Delegate interface func (g *GossipMessageBroker) NodeMeta(limit int) []byte { return []byte{} diff --git a/messenger.go b/messenger.go index 03e15ba0e..03579bee6 100644 --- a/messenger.go +++ b/messenger.go @@ -145,7 +145,6 @@ func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { type MessageBroker interface { Send(pb proto.Message, method string) error Receive(pb proto.Message) error - SetMessenger(m *Messenger) } ////////////////////////////////////////////////////////////////// @@ -167,7 +166,6 @@ func (c *nopMessageBroker) Receive(pb proto.Message) error { fmt.Println("NOPMessageBroker: Receive") return nil } -func (c *nopMessageBroker) SetMessenger(m *Messenger) {} ////////////////////////////////////////////////////////////////// @@ -217,8 +215,6 @@ func (h *HTTPMessageBroker) Receive(pb proto.Message) error { return nil } -func (h *HTTPMessageBroker) SetMessenger(m *Messenger) {} - func (h *HTTPMessageBroker) nodes() ([]*Node, error) { if h.messenger == nil { return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") From 340cbc0cd05a5702f703724b581972846d5d5301 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 15:59:54 -0500 Subject: [PATCH 21/63] fix comment s/list/map --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index a3678ad00..e9c6bb2c1 100644 --- a/cluster.go +++ b/cluster.go @@ -121,7 +121,7 @@ func (c *Cluster) NodeSetHosts() []string { return a } -// Health returns a list of nodes in the cluster along with each node's state (UP/DOWN). +// Health returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. func (c *Cluster) Health() map[string]string { h := make(map[string]string) for _, n := range c.Nodes { From 10a6cead17445dba35573a1c26d408ff67491cde Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 16:14:05 -0500 Subject: [PATCH 22/63] remove unused methods on StaticNodeSet these used to implement the NodeSet interface, but they've since been removed from NodeSet --- cluster.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cluster.go b/cluster.go index e9c6bb2c1..13f620b88 100644 --- a/cluster.go +++ b/cluster.go @@ -264,18 +264,6 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) SetMessageHandler(f func(proto.Message) error) { - return -} - -func (s *StaticNodeSet) SetRemoteStateHandler(f func(proto.Message) error) { - return -} - -func (s *StaticNodeSet) SetLocalStateSource(f func() (proto.Message, error)) { - return -} - func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { return nil } From 25f3ac92bddfdbecbc9a10a3f4b445ec96cfc81c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Apr 2017 16:55:40 -0500 Subject: [PATCH 23/63] move complex, implementation-specific config bits out of config.go moved into pilosa/server/server.go with minimal changes for now - had to tweak a few things to be able to get at unexported fields. Next step will be to move the implementations of NodeSet and MessageBroker out of the pilosa package so that pilosa core knows nothing about them. pilosa/server will deal with any complexities involved in setting them up. --- config.go | 79 +----------------------------------------------- gossip.go | 7 ++++- messenger.go | 4 +-- server/server.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 85 insertions(+), 84 deletions(-) diff --git a/config.go b/config.go index b74125ce7..150bd1ec9 100644 --- a/config.go +++ b/config.go @@ -1,10 +1,6 @@ package pilosa -import ( - "net" - "strconv" - "time" -) +import "time" const ( // DefaultHost is the default hostname and port to use. @@ -66,79 +62,6 @@ func NewConfigForHosts(hosts []string) *Config { return conf } -// PilosaMessenger returns a new instance of Messenger based on the config. -func (c *Config) PilosaMessenger() *Messenger { - messenger := NewMessenger() - switch c.Cluster.MessengerType { - case "broadcast": - n := NewHTTPMessageBroker() - n.messenger = messenger - messenger.Broker = n - case "gossip": - n := NewGossipMessageBroker() - n.messenger = messenger - messenger.Broker = n - case "static": - // nop - } - return messenger -} - -// PilosaCluster returns a new instance of Cluster based on the config. -func (c *Config) PilosaCluster() *Cluster { - cluster := NewCluster() - cluster.ReplicaN = c.Cluster.ReplicaN - - for _, hostport := range c.Cluster.Nodes { - cluster.Nodes = append(cluster.Nodes, &Node{Host: hostport}) - } - - // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.MessengerType { - case "broadcast": - cluster.NodeSet = NewHTTPNodeSet() - cluster.NodeSet.(*HTTPNodeSet).Join(cluster.Nodes) - case "gossip": - gport, err := strconv.Atoi(DefaultGossipPort) - if err != nil { - // what? - } - gossipPort := gport - gossipSeed := DefaultHost - if c.Cluster.Gossip.Port != 0 { - gossipPort = c.Cluster.Gossip.Port - } - if c.Cluster.Gossip.Seed != "" { - gossipSeed = c.Cluster.Gossip.Seed - } - // get the host portion of addr to use for binding - gossipHost, _, err := net.SplitHostPort(c.Host) - if err != nil { - gossipHost = c.Host - } - cluster.NodeSet = NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - case "static": - cluster.NodeSet = NewStaticNodeSet() - default: - cluster.NodeSet = NewStaticNodeSet() - } - - return cluster -} - -// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker -// after cluster configuration. -func (c *Config) AssociateMessageBroker(s *Server) { - switch c.Cluster.MessengerType { - case "broadcast": - // nop - case "gossip": - s.Cluster.NodeSet.(*GossipNodeSet).config.memberlistConfig.Delegate = s.Messenger.Broker.(*GossipMessageBroker) - case "static": - // nop - } -} - // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/gossip.go b/gossip.go index 9b18b45dd..178fa999e 100644 --- a/gossip.go +++ b/gossip.go @@ -23,6 +23,10 @@ type GossipNodeSet struct { LogOutput io.Writer } +func (g *GossipNodeSet) AttachBroker(mb *GossipMessageBroker) { + g.config.memberlistConfig.Delegate = mb +} + func (g *GossipNodeSet) Nodes() []*Node { a := make([]*Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { @@ -196,9 +200,10 @@ func (g *GossipMessageBroker) logger() *log.Logger { //////////////////////////////////////////////////////////////// // NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker() *GossipMessageBroker { +func NewGossipMessageBroker(m *Messenger) *GossipMessageBroker { g := &GossipMessageBroker{ LogOutput: os.Stderr, + messenger: m, } g.broadcasts = &memberlist.TransmitLimitedQueue{ diff --git a/messenger.go b/messenger.go index 03579bee6..1ffbfa51e 100644 --- a/messenger.go +++ b/messenger.go @@ -175,8 +175,8 @@ type HTTPMessageBroker struct { } // NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker() *HTTPMessageBroker { - return &HTTPMessageBroker{} +func NewHTTPMessageBroker(m *Messenger) *HTTPMessageBroker { + return &HTTPMessageBroker{messenger: m} } // Send sends a protobuf message to all nodes simultaneously. diff --git a/server/server.go b/server/server.go index ba41f3dd0..11bb055a9 100644 --- a/server/server.go +++ b/server/server.go @@ -9,8 +9,10 @@ import ( "fmt" "io" "math/rand" + "net" "os" "path/filepath" + "strconv" "strings" "time" @@ -92,11 +94,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = m.Config.PilosaMessenger() - m.Server.Cluster = m.Config.PilosaCluster() + m.Server.Messenger = PilosaMessenger(m.Config) + m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. - m.Config.AssociateMessageBroker(m.Server) + AssociateMessageBroker(m.Server, m.Config) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) @@ -109,6 +111,77 @@ func (m *Command) Run(args ...string) (err error) { return nil } +// PilosaMessenger returns a new instance of Messenger based on the config. +func PilosaMessenger(c *pilosa.Config) *pilosa.Messenger { + messenger := pilosa.NewMessenger() + switch c.Cluster.MessengerType { + case "broadcast": + n := pilosa.NewHTTPMessageBroker(messenger) + messenger.Broker = n + case "gossip": + n := pilosa.NewGossipMessageBroker(messenger) + messenger.Broker = n + case "static": + // nop + } + return messenger +} + +// PilosaCluster returns a new instance of Cluster based on the config. +func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { + cluster := pilosa.NewCluster() + cluster.ReplicaN = c.Cluster.ReplicaN + + for _, hostport := range c.Cluster.Nodes { + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) + } + + // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. + switch c.Cluster.MessengerType { + case "broadcast": + cluster.NodeSet = pilosa.NewHTTPNodeSet() + cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) + case "gossip": + gport, err := strconv.Atoi(pilosa.DefaultGossipPort) + if err != nil { + // what? + } + gossipPort := gport + gossipSeed := pilosa.DefaultHost + if c.Cluster.Gossip.Port != 0 { + gossipPort = c.Cluster.Gossip.Port + } + if c.Cluster.Gossip.Seed != "" { + gossipSeed = c.Cluster.Gossip.Seed + } + // get the host portion of addr to use for binding + gossipHost, _, err := net.SplitHostPort(c.Host) + if err != nil { + gossipHost = c.Host + } + cluster.NodeSet = pilosa.NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) + case "static": + cluster.NodeSet = pilosa.NewStaticNodeSet() + default: + cluster.NodeSet = pilosa.NewStaticNodeSet() + } + + return cluster +} + +// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// after cluster configuration. +func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { + switch c.Cluster.MessengerType { + case "broadcast": + // nop + case "gossip": + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.Messenger.Broker.(*pilosa.GossipMessageBroker)) + case "static": + // nop + } +} + func normalizeHost(host string) (string, error) { if !strings.Contains(host, ":") { host = host + ":" From 1aca2ce1de6c10045301e41102110a0aa3efcdb7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 11:06:58 -0500 Subject: [PATCH 24/63] implement LocalState and internal message reception on Server greatly simplifies Messenger, to the point of making it basically shell around MessageBroker. Next stesp are to make the receiving of internal messages more well-defined and behind an interface, remove/merge Messenger and MessageBroker, and have the implementations of MessageBroker in separate packages. --- gossip.go | 16 +++--- handler.go | 3 +- messenger.go | 126 +++-------------------------------------------- server.go | 90 +++++++++++++++++++++++++++++++-- server/server.go | 10 ++-- 5 files changed, 108 insertions(+), 137 deletions(-) diff --git a/gossip.go b/gossip.go index 178fa999e..59abef70d 100644 --- a/gossip.go +++ b/gossip.go @@ -90,7 +90,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed type GossipMessageBroker struct { broadcasts *memberlist.TransmitLimitedQueue - messenger *Messenger + server *Server // The writer for any logging. LogOutput io.Writer @@ -103,7 +103,7 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { return err } - mlist := g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist + mlist := g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist // Direct sends the message directly to every node. // An error from any node raises an error on the entire operation. @@ -136,7 +136,7 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { } func (g *GossipMessageBroker) Receive(pb proto.Message) error { - if err := g.messenger.ReceiveMessage(pb); err != nil { + if err := g.server.ReceiveMessage(pb); err != nil { return err } return nil @@ -164,7 +164,7 @@ func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { } func (g *GossipMessageBroker) LocalState(join bool) []byte { - pb, err := g.messenger.LocalState() + pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) return []byte{} @@ -186,7 +186,7 @@ func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { g.logger().Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.messenger.HandleRemoteState(&pb) + err := g.server.HandleRemoteState(&pb) if err != nil { g.logger().Printf("merge state error: %s", err) } @@ -200,15 +200,15 @@ func (g *GossipMessageBroker) logger() *log.Logger { //////////////////////////////////////////////////////////////// // NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker(m *Messenger) *GossipMessageBroker { +func NewGossipMessageBroker(s *Server) *GossipMessageBroker { g := &GossipMessageBroker{ LogOutput: os.Stderr, - messenger: m, + server: s, } g.broadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { - return g.messenger.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() + return g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() }, RetransmitMult: 3, } diff --git a/handler.go b/handler.go index 0a7cb278d..7616fd2d8 100644 --- a/handler.go +++ b/handler.go @@ -27,6 +27,7 @@ import ( type Handler struct { Index *Index Messenger *Messenger + Server *Server // Local hostname & cluster configuration. Host string @@ -213,7 +214,7 @@ func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { return } - if err := h.Messenger.Broker.Receive(m); err != nil { + if err := h.Server.ReceiveMessage(m); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/messenger.go b/messenger.go index 1ffbfa51e..3672a407a 100644 --- a/messenger.go +++ b/messenger.go @@ -20,15 +20,9 @@ import ( // Messenger represents an internal message handler. type Messenger struct { - // Broker handles Send/Receive Messages. + // Broker handles Send Broker MessageBroker - Index *Index - - // Local hostname & cluster configuration. - Host string - Cluster *Cluster - // The writer for any logging. LogOutput io.Writer } @@ -47,104 +41,12 @@ func (m *Messenger) SendMessage(pb proto.Message, method string) error { } return m.Broker.Send(pb, method) } -func (m *Messenger) ReceiveMessage(pb proto.Message) error { - return m.handleMessage(pb) -} - -// handleMessage handles protobuf Messages sent to nodes in the cluster. -func (m *Messenger) handleMessage(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.CreateSliceMessage: - d := m.Index.DB(obj.DB) - if d == nil { - return fmt.Errorf("Local DB not found: %s", obj.DB) - } - d.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := m.Index.CreateDB(obj.DB, opt) - if err != nil { - return err - } - case *internal.DeleteDBMessage: - fmt.Println("DELETE:", obj.DB) - if err := m.Index.DeleteDB(obj.DB); err != nil { - return err - } - case *internal.CreateFrameMessage: - db := m.Index.DB(obj.DB) - opt := FrameOptions{RowLabel: obj.Meta.RowLabel} - _, err := db.CreateFrame(obj.Frame, opt) - if err != nil { - return err - } - case *internal.DeleteFrameMessage: - db := m.Index.DB(obj.DB) - if err := db.DeleteFrame(obj.Frame); err != nil { - return err - } - } - return nil -} - -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -// It seems odd to have this as part of Messenger, but with the -// exception of Server, it's currenntly the only object with access -// to the necessary information (Host, Index, Cluster). -func (m *Messenger) LocalState() (proto.Message, error) { - if m.Index == nil { - return nil, errors.New("Messenger.Index is nil.") - } - return &internal.NodeState{ - Host: m.Host, - State: "OK", // TODO: make this work, pull from m.Cluster.Node - DBs: encodeDBs(m.Index.DBs()), - }, nil -} - -// HandleRemoteState receives incoming NodeState from remote nodes. -func (m *Messenger) HandleRemoteState(pb proto.Message) error { - return m.mergeRemoteState(pb.(*internal.NodeState)) -} - -func (m *Messenger) mergeRemoteState(ns *internal.NodeState) error { - // TODO: update some node state value in the cluster (it should be in cluster.node i guess) - - // Create databases that don't exist. - for _, db := range ns.DBs { - opt := DBOptions{ - ColumnLabel: db.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), - } - d, err := m.Index.CreateDBIfNotExists(db.Name, opt) - if err != nil { - return err - } - // Create frames that don't exist. - for _, f := range db.Frames { - opt := FrameOptions{ - RowLabel: f.Meta.RowLabel, - TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), - CacheSize: f.Meta.CacheSize, - } - _, err := d.CreateFrameIfNotExists(f.Name, opt) - if err != nil { - return err - } - } - } - - return nil -} ////////////////////////////////////////////////////////////////// // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { Send(pb proto.Message, method string) error - Receive(pb proto.Message) error } ////////////////////////////////////////////////////////////////// @@ -162,21 +64,17 @@ func (c *nopMessageBroker) Send(pb proto.Message, method string) error { fmt.Println("NOPMessageBroker: Send") return nil } -func (c *nopMessageBroker) Receive(pb proto.Message) error { - fmt.Println("NOPMessageBroker: Receive") - return nil -} ////////////////////////////////////////////////////////////////// // HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. type HTTPMessageBroker struct { - messenger *Messenger + server *Server } // NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker(m *Messenger) *HTTPMessageBroker { - return &HTTPMessageBroker{messenger: m} +func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { + return &HTTPMessageBroker{server: s} } // Send sends a protobuf message to all nodes simultaneously. @@ -196,7 +94,7 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { var g errgroup.Group for _, n := range nodes { // Don't send the message to the local node. - if n.Host == h.messenger.Host { + if n.Host == h.server.Host { continue } node := n @@ -207,19 +105,11 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { return g.Wait() } -// Receive is called when a node receives a message. -func (h *HTTPMessageBroker) Receive(pb proto.Message) error { - if err := h.messenger.ReceiveMessage(pb); err != nil { - return err - } - return nil -} - func (h *HTTPMessageBroker) nodes() ([]*Node, error) { - if h.messenger == nil { - return nil, errors.New("HTTPMessageBroker has no reference to Messenger.") + if h.server == nil { + return nil, errors.New("HTTPMessageBroker has no reference to Server.") } - nodeset, ok := h.messenger.Cluster.NodeSet.(*HTTPNodeSet) + nodeset, ok := h.server.Cluster.NodeSet.(*HTTPNodeSet) if !ok { return nil, errors.New("NodeSet cannot be caste to HTTPNodeSet.") } diff --git a/server.go b/server.go index a93d29f57..7dd35fa8f 100644 --- a/server.go +++ b/server.go @@ -1,6 +1,7 @@ package pilosa import ( + "errors" "fmt" "io" "io/ioutil" @@ -64,7 +65,7 @@ func NewServer() *Server { } s.Handler.Index = s.Index - s.Messenger.Index = s.Index + s.Handler.Server = s // TODO remove return s } @@ -111,9 +112,6 @@ func (s *Server) Open() error { e.Cluster = s.Cluster // Initialize Messenger. - s.Messenger.Index = s.Index - s.Messenger.Host = s.Host - s.Messenger.Cluster = s.Cluster s.Messenger.LogOutput = s.LogOutput // Initialize HTTP handler. @@ -236,6 +234,90 @@ func (s *Server) monitorMaxSlices() { } } +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) LocalState() (proto.Message, error) { + if s.Index == nil { + return nil, errors.New("Messenger.Index is nil.") + } + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from s.Cluster.Node + DBs: encodeDBs(s.Index.DBs()), + }, nil +} + +func (s *Server) ReceiveMessage(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.CreateSliceMessage: + d := s.Index.DB(obj.DB) + if d == nil { + return fmt.Errorf("Local DB not found: %s", obj.DB) + } + d.SetRemoteMaxSlice(obj.Slice) + case *internal.CreateDBMessage: + opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := s.Index.CreateDB(obj.DB, opt) + if err != nil { + return err + } + case *internal.DeleteDBMessage: + fmt.Println("DELETE:", obj.DB) + if err := s.Index.DeleteDB(obj.DB); err != nil { + return err + } + case *internal.CreateFrameMessage: + db := s.Index.DB(obj.DB) + opt := FrameOptions{RowLabel: obj.Meta.RowLabel} + _, err := db.CreateFrame(obj.Frame, opt) + if err != nil { + return err + } + case *internal.DeleteFrameMessage: + db := s.Index.DB(obj.DB) + if err := db.DeleteFrame(obj.Frame); err != nil { + return err + } + } + return nil +} + +// HandleRemoteState receives incoming NodeState from remote nodes. +func (s *Server) HandleRemoteState(pb proto.Message) error { + return s.mergeRemoteState(pb.(*internal.NodeState)) +} + +func (s *Server) mergeRemoteState(ns *internal.NodeState) error { + // TODO: update some node state value in the cluster (it should be in cluster.node i guess) + + // Create databases that don't exist. + for _, db := range ns.DBs { + opt := DBOptions{ + ColumnLabel: db.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + } + d, err := s.Index.CreateDBIfNotExists(db.Name, opt) + if err != nil { + return err + } + // Create frames that don't exist. + for _, f := range db.Frames { + opt := FrameOptions{ + RowLabel: f.Meta.RowLabel, + TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), + CacheSize: f.Meta.CacheSize, + } + _, err := d.CreateFrameIfNotExists(f.Name, opt) + if err != nil { + return err + } + } + } + + return nil +} + func checkMaxSlices(hostport string) (map[string]uint64, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ diff --git a/server/server.go b/server/server.go index 11bb055a9..0b61bf405 100644 --- a/server/server.go +++ b/server/server.go @@ -94,7 +94,7 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = PilosaMessenger(m.Config) + m.Server.Messenger = PilosaMessenger(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. @@ -112,15 +112,13 @@ func (m *Command) Run(args ...string) (err error) { } // PilosaMessenger returns a new instance of Messenger based on the config. -func PilosaMessenger(c *pilosa.Config) *pilosa.Messenger { +func PilosaMessenger(c *pilosa.Config, server *pilosa.Server) *pilosa.Messenger { messenger := pilosa.NewMessenger() switch c.Cluster.MessengerType { case "broadcast": - n := pilosa.NewHTTPMessageBroker(messenger) - messenger.Broker = n + messenger.Broker = pilosa.NewHTTPMessageBroker(server) case "gossip": - n := pilosa.NewGossipMessageBroker(messenger) - messenger.Broker = n + messenger.Broker = pilosa.NewGossipMessageBroker(server) case "static": // nop } From 5079ee7a2286766aa9ff597baa2c2ef5bbc0fda5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:14:11 -0500 Subject: [PATCH 25/63] remove messenger "wrapper" around messagebroker all the functionality aside from the broker has been removed --- cluster.go | 6 +----- db.go | 4 ++-- frame.go | 2 +- gossip.go | 2 +- handler.go | 8 ++++---- handler_test.go | 5 ++--- index.go | 5 ++--- messenger.go | 33 +-------------------------------- server.go | 13 +++++-------- server/server.go | 17 ++++++++--------- 10 files changed, 27 insertions(+), 68 deletions(-) diff --git a/cluster.go b/cluster.go index 13f620b88..a435a20b7 100644 --- a/cluster.go +++ b/cluster.go @@ -248,7 +248,6 @@ func (h *HTTPNodeSet) Join(nodes []*Node) error { // StaticNodeSet represents a basic NodeSet for testing type StaticNodeSet struct { - Messenger nodes []*Node } @@ -264,9 +263,6 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) SendMessage(pb proto.Message, method string) error { - return nil -} -func (s *StaticNodeSet) ReceiveMessage(pb proto.Message) error { +func (s *StaticNodeSet) Send(pb proto.Message, method string) error { return nil } diff --git a/db.go b/db.go index cc27e18c9..4eb61707e 100644 --- a/db.go +++ b/db.go @@ -43,7 +43,7 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - messenger *Messenger + msgbroker MessageBroker stats StatsClient LogOutput io.Writer @@ -417,7 +417,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - f.messenger = db.messenger + f.msgbroker = db.msgbroker return f, nil } diff --git a/frame.go b/frame.go index c5a40b1d1..1da970efc 100644 --- a/frame.go +++ b/frame.go @@ -38,7 +38,7 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - messenger *Messenger + msgbroker MessageBroker stats StatsClient // Frame settings. diff --git a/gossip.go b/gossip.go index 59abef70d..37b8c3b97 100644 --- a/gossip.go +++ b/gossip.go @@ -96,7 +96,7 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// implementation of the messenger.Messenger interface +// implementation of the messenger.MessageBroker interface func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { msg, err := MarshalMessage(pb) if err != nil { diff --git a/handler.go b/handler.go index 7616fd2d8..5c015abde 100644 --- a/handler.go +++ b/handler.go @@ -26,7 +26,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { Index *Index - Messenger *Messenger + MsgBroker MessageBroker Server *Server // Local hostname & cluster configuration. @@ -370,7 +370,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Messenger.SendMessage( + err := h.MsgBroker.Send( &internal.DeleteDBMessage{ DB: req.DB, }, "direct") @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.Messenger.SendMessage( + err = h.MsgBroker.Send( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -586,7 +586,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Messenger.SendMessage( + err := h.MsgBroker.Send( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, diff --git a/handler_test.go b/handler_test.go index 92bcc7601..e95e5c0dd 100644 --- a/handler_test.go +++ b/handler_test.go @@ -794,7 +794,7 @@ func NewHandler() *Handler { h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. - h.Messenger = pilosa.NewMessenger() + h.MsgBroker = pilosa.NopMessageBroker return h } @@ -828,8 +828,7 @@ func NewServer() *Server { s.Handler.Host = s.Host() // Handler test messages can no-op. - s.Handler.Messenger = pilosa.NewMessenger() - + s.Handler.MsgBroker = pilosa.NopMessageBroker // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() diff --git a/index.go b/index.go index 5e9f5187e..ab659ada1 100644 --- a/index.go +++ b/index.go @@ -23,8 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - Messenger *Messenger - + MsgBroker MessageBroker // Close management wg sync.WaitGroup closing chan struct{} @@ -247,7 +246,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.messenger = i.Messenger + db.msgbroker = i.MsgBroker return db, nil } diff --git a/messenger.go b/messenger.go index 3672a407a..fc0c55d77 100644 --- a/messenger.go +++ b/messenger.go @@ -4,11 +4,9 @@ import ( "bytes" "errors" "fmt" - "io" "io/ioutil" "net/http" "net/url" - "os" "reflect" "golang.org/x/sync/errgroup" @@ -17,40 +15,11 @@ import ( "github.com/pilosa/pilosa/internal" ) -// Messenger represents an internal message handler. -type Messenger struct { - - // Broker handles Send - Broker MessageBroker - - // The writer for any logging. - LogOutput io.Writer -} - -// NewMessenger returns a new instance of Messenger with a default logger. -func NewMessenger() *Messenger { - return &Messenger{ - Broker: NopMessageBroker, - LogOutput: os.Stderr, - } -} - -func (m *Messenger) SendMessage(pb proto.Message, method string) error { - if m.Broker == nil { - return errors.New("Messenger.Broker is not defined.") - } - return m.Broker.Send(pb, method) -} - -////////////////////////////////////////////////////////////////// - // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { Send(pb proto.Message, method string) error } -////////////////////////////////////////////////////////////////// - func init() { NopMessageBroker = &nopMessageBroker{} } @@ -61,7 +30,7 @@ var NopMessageBroker MessageBroker type nopMessageBroker struct{} func (c *nopMessageBroker) Send(pb proto.Message, method string) error { - fmt.Println("NOPMessageBroker: Send") + fmt.Println("NOPMessageBroker: Send") // TODO remove or log properly? return nil } diff --git a/server.go b/server.go index 7dd35fa8f..68f9a6921 100644 --- a/server.go +++ b/server.go @@ -35,7 +35,7 @@ type Server struct { // Data storage and HTTP interface. Index *Index Handler *Handler - Messenger *Messenger + MsgBroker MessageBroker // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -56,7 +56,7 @@ func NewServer() *Server { Index: NewIndex(), Handler: NewHandler(), - Messenger: NewMessenger(), + MsgBroker: NopMessageBroker, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -111,18 +111,15 @@ func (s *Server) Open() error { e.Host = s.Host e.Cluster = s.Cluster - // Initialize Messenger. - s.Messenger.LogOutput = s.LogOutput - // Initialize HTTP handler. - s.Handler.Messenger = s.Messenger + s.Handler.MsgBroker = s.MsgBroker s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput // Initialize Index. - s.Index.Messenger = s.Messenger + s.Index.MsgBroker = s.MsgBroker s.Index.LogOutput = s.LogOutput // Serve HTTP. @@ -239,7 +236,7 @@ func (s *Server) monitorMaxSlices() { // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalState() (proto.Message, error) { if s.Index == nil { - return nil, errors.New("Messenger.Index is nil.") + return nil, errors.New("Server.Index is nil.") } return &internal.NodeState{ Host: s.Host, diff --git a/server/server.go b/server/server.go index 0b61bf405..69041ef3a 100644 --- a/server/server.go +++ b/server/server.go @@ -94,7 +94,7 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.Messenger = PilosaMessenger(m.Config, m.Server) + m.Server.MsgBroker = PilosaMessageBroker(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) // Associate objects to the MessageBroker based on config. @@ -111,18 +111,17 @@ func (m *Command) Run(args ...string) (err error) { return nil } -// PilosaMessenger returns a new instance of Messenger based on the config. -func PilosaMessenger(c *pilosa.Config, server *pilosa.Server) *pilosa.Messenger { - messenger := pilosa.NewMessenger() +// PilosaMessageBroker returns a new instance of MessageBroker based on the config. +func PilosaMessageBroker(c *pilosa.Config, server *pilosa.Server) (broker pilosa.MessageBroker) { switch c.Cluster.MessengerType { case "broadcast": - messenger.Broker = pilosa.NewHTTPMessageBroker(server) + broker = pilosa.NewHTTPMessageBroker(server) case "gossip": - messenger.Broker = pilosa.NewGossipMessageBroker(server) + broker = pilosa.NewGossipMessageBroker(server) case "static": - // nop + broker = pilosa.NopMessageBroker } - return messenger + return broker } // PilosaCluster returns a new instance of Cluster based on the config. @@ -174,7 +173,7 @@ func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { case "broadcast": // nop case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.Messenger.Broker.(*pilosa.GossipMessageBroker)) + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.MsgBroker.(*pilosa.GossipMessageBroker)) case "static": // nop } From 4b8c2630e3838039537cf9df5c7e47b4c8b911df Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:30:02 -0500 Subject: [PATCH 26/63] split MessageBroker.Send into SendSync and SendAsync --- cluster.go | 6 +++++- gossip.go | 46 +++++++++++++++++++++++++--------------------- handler.go | 12 ++++++------ messenger.go | 24 +++++++++++++++++++----- 4 files changed, 55 insertions(+), 33 deletions(-) diff --git a/cluster.go b/cluster.go index a435a20b7..ef67b7a4a 100644 --- a/cluster.go +++ b/cluster.go @@ -263,6 +263,10 @@ func (s *StaticNodeSet) Open() error { return nil } -func (s *StaticNodeSet) Send(pb proto.Message, method string) error { +func (s *StaticNodeSet) SendSync(pb proto.Message) error { + return nil +} + +func (s *StaticNodeSet) SendAsync(pb proto.Message) error { return nil } diff --git a/gossip.go b/gossip.go index 37b8c3b97..6482a333d 100644 --- a/gossip.go +++ b/gossip.go @@ -96,8 +96,8 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { +// SendSync implementation of the messenger.MessageBroker interface +func (g *GossipMessageBroker) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -110,28 +110,32 @@ func (g *GossipMessageBroker) Send(pb proto.Message, method string) error { // // Gossip uses the gossip protocol to eventually deliver the message // to every node. - switch method { - case "direct": - var eg errgroup.Group - for _, n := range mlist.Members() { - // Don't send the message to the local node. - if n == mlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return mlist.SendToTCP(node, msg) - }) + var eg errgroup.Group + for _, n := range mlist.Members() { + // Don't send the message to the local node. + if n == mlist.LocalNode() { + continue } - return eg.Wait() - case "gossip": - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) + node := n + eg.Go(func() error { + return mlist.SendToTCP(node, msg) + }) + } + return eg.Wait() +} + +// SendAsync implementation of the messenger.MessageBroker interface +func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { + msg, err := MarshalMessage(pb) + if err != nil { + return err } + b := &broadcast{ + msg: msg, + notify: nil, + } + g.broadcasts.QueueBroadcast(b) return nil } diff --git a/handler.go b/handler.go index 5c015abde..ed1418f9a 100644 --- a/handler.go +++ b/handler.go @@ -370,10 +370,10 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.Send( + err := h.MsgBroker.SendSync( &internal.DeleteDBMessage{ DB: req.DB, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) } @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.MsgBroker.Send( + err = h.MsgBroker.SendSync( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -522,7 +522,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), }, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) } @@ -586,11 +586,11 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.Send( + err := h.MsgBroker.SendSync( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, - }, "direct") + }) if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) } diff --git a/messenger.go b/messenger.go index fc0c55d77..22ba8d61f 100644 --- a/messenger.go +++ b/messenger.go @@ -17,7 +17,8 @@ import ( // MessageBroker is an interface for handling incoming/outgoing messages. type MessageBroker interface { - Send(pb proto.Message, method string) error + SendSync(pb proto.Message) error + SendAsync(pb proto.Message) error } func init() { @@ -29,8 +30,15 @@ var NopMessageBroker MessageBroker // nopMessageBroker represents a MessageBroker that doesn't do anything. type nopMessageBroker struct{} -func (c *nopMessageBroker) Send(pb proto.Message, method string) error { - fmt.Println("NOPMessageBroker: Send") // TODO remove or log properly? +// SendSync A no-op implemenetation of MessageBroker SendSync method. +func (c *nopMessageBroker) SendSync(pb proto.Message) error { + fmt.Println("NOPMessageBroker: SendSync") // TODO remove or log properly? + return nil +} + +// SendAsync A no-op implemenetation of MessageBroker SendAsync method. +func (c *nopMessageBroker) SendAsync(pb proto.Message) error { + fmt.Println("NOPMessageBroker: SendAsync") // TODO remove or log properly? return nil } @@ -46,9 +54,9 @@ func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { return &HTTPMessageBroker{server: s} } -// Send sends a protobuf message to all nodes simultaneously. +// SendSync sends a protobuf message to all nodes simultaneously. // It waits for all nodes to respond before the function returns (and returns any errors). -func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { +func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) if err != nil { @@ -74,6 +82,12 @@ func (h *HTTPMessageBroker) Send(pb proto.Message, method string) error { return g.Wait() } +// SendAsync exists to implement the MessageBroker interface, but just calls +// SendSync. +func (h *HTTPMessageBroker) SendAsync(pb proto.Message) error { + return h.SendSync(pb) +} + func (h *HTTPMessageBroker) nodes() ([]*Node, error) { if h.server == nil { return nil, errors.New("HTTPMessageBroker has no reference to Server.") From 32d07fc9e1c1fa06f02375e13311b8315c7e5921 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 12:47:26 -0500 Subject: [PATCH 27/63] rename MessageBroker -> Broadcaster --- cmd/server.go | 2 +- config.go | 12 ++++++------ db.go | 6 +++--- frame.go | 4 ++-- gossip.go | 38 +++++++++++++++++++------------------- handler.go | 12 ++++++------ handler_test.go | 4 ++-- index.go | 4 ++-- messenger.go | 46 +++++++++++++++++++++++----------------------- server.go | 16 ++++++++-------- server/server.go | 36 ++++++++++++++++++------------------ 11 files changed, 90 insertions(+), 90 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 27c238eb6..e9d7fcfdf 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.MessengerType, "cluster.messenger-type", "", "static", "Type of Messenger to use for inter-host messaging. Choose from [static, broadcast, gossip]") + flags.StringVarP(&Server.Config.Cluster.BroadcasterType, "cluster.broadcaster-type", "", "static", "Type of Broadcaster to use for inter-host messaging. Choose from [static, http, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 150bd1ec9..8b718f4e1 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultMessengerType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultBroadcasterType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -17,7 +17,7 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` - MessengerType string `toml:"messenger-type"` + BroadcasterType string `toml:"broadcaster-type"` Nodes []string `toml:"hosts"` PollingInterval Duration `toml:"polling-interval"` Gossip ConfigGossip `toml:"gossip"` @@ -45,7 +45,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.MessengerType = DefaultMessengerType + c.Cluster.BroadcasterType = DefaultBroadcasterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) diff --git a/db.go b/db.go index 4eb61707e..d2ceaf13a 100644 --- a/db.go +++ b/db.go @@ -43,8 +43,8 @@ type DB struct { // Profile attribute storage and cache profileAttrStore *AttrStore - msgbroker MessageBroker - stats StatsClient + broadcaster Broadcaster + stats StatsClient LogOutput io.Writer } @@ -417,7 +417,7 @@ func (db *DB) newFrame(path, name string) (*Frame, error) { } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - f.msgbroker = db.msgbroker + f.broadcaster = db.broadcaster return f, nil } diff --git a/frame.go b/frame.go index 1da970efc..a497e6906 100644 --- a/frame.go +++ b/frame.go @@ -38,8 +38,8 @@ type Frame struct { // Bitmap attribute storage and cache bitmapAttrStore *AttrStore - msgbroker MessageBroker - stats StatsClient + broadcaster Broadcaster + stats StatsClient // Frame settings. rowLabel string diff --git a/gossip.go b/gossip.go index 6482a333d..da9c847f5 100644 --- a/gossip.go +++ b/gossip.go @@ -23,7 +23,7 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) AttachBroker(mb *GossipMessageBroker) { +func (g *GossipNodeSet) AttachBroadcaster(mb *GossipBroadcaster) { g.config.memberlistConfig.Delegate = mb } @@ -85,9 +85,9 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed //////////////////////////////////////////////////////////////// -// GossipMessageBroker represents a gossip implementation of pilosa.MessageBroker -// GossipMessageBroker also represents an implementation of memberlist.Delegate -type GossipMessageBroker struct { +// GossipBroadcaster represents a gossip implementation of pilosa.Broadcaster +// GossipBroadcaster also represents an implementation of memberlist.Delegate +type GossipBroadcaster struct { broadcasts *memberlist.TransmitLimitedQueue server *Server @@ -96,8 +96,8 @@ type GossipMessageBroker struct { LogOutput io.Writer } -// SendSync implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) SendSync(pb proto.Message) error { +// SendSync implementation of the Broadcaster interface +func (g *GossipBroadcaster) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -124,8 +124,8 @@ func (g *GossipMessageBroker) SendSync(pb proto.Message) error { return eg.Wait() } -// SendAsync implementation of the messenger.MessageBroker interface -func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { +// SendAsync implementation of the Broadcaster interface +func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -139,7 +139,7 @@ func (g *GossipMessageBroker) SendAsync(pb proto.Message) error { return nil } -func (g *GossipMessageBroker) Receive(pb proto.Message) error { +func (g *GossipBroadcaster) Receive(pb proto.Message) error { if err := g.server.ReceiveMessage(pb); err != nil { return err } @@ -147,11 +147,11 @@ func (g *GossipMessageBroker) Receive(pb proto.Message) error { } // implementation of the memberlist.Delegate interface -func (g *GossipMessageBroker) NodeMeta(limit int) []byte { +func (g *GossipBroadcaster) NodeMeta(limit int) []byte { return []byte{} } -func (g *GossipMessageBroker) NotifyMsg(b []byte) { +func (g *GossipBroadcaster) NotifyMsg(b []byte) { m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -163,11 +163,11 @@ func (g *GossipMessageBroker) NotifyMsg(b []byte) { } } -func (g *GossipMessageBroker) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipBroadcaster) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } -func (g *GossipMessageBroker) LocalState(join bool) []byte { +func (g *GossipBroadcaster) LocalState(join bool) []byte { pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) @@ -183,7 +183,7 @@ func (g *GossipMessageBroker) LocalState(join bool) []byte { return buf } -func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { +func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeState if err := proto.Unmarshal(buf, &pb); err != nil { @@ -196,16 +196,16 @@ func (g *GossipMessageBroker) MergeRemoteState(buf []byte, join bool) { } } -// logger returns a logger for the GossipMessageBroker. -func (g *GossipMessageBroker) logger() *log.Logger { +// logger returns a logger for the GossipBroadcaster +func (g *GossipBroadcaster) logger() *log.Logger { return log.New(g.LogOutput, "", log.LstdFlags) } //////////////////////////////////////////////////////////////// -// NewGossipMessageBroker returns a new instance of GossipMessageBroker. -func NewGossipMessageBroker(s *Server) *GossipMessageBroker { - g := &GossipMessageBroker{ +// NewGossipBroadcaster returns a new instance of GossipBroadcaster. +func NewGossipBroadcaster(s *Server) *GossipBroadcaster { + g := &GossipBroadcaster{ LogOutput: os.Stderr, server: s, } diff --git a/handler.go b/handler.go index ed1418f9a..83f969d22 100644 --- a/handler.go +++ b/handler.go @@ -25,9 +25,9 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Index *Index - MsgBroker MessageBroker - Server *Server + Index *Index + Broadcaster Broadcaster + Server *Server // Local hostname & cluster configuration. Host string @@ -370,7 +370,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.SendSync( + err := h.Broadcaster.SendSync( &internal.DeleteDBMessage{ DB: req.DB, }) @@ -514,7 +514,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Send the create message to all nodes. - err = h.MsgBroker.SendSync( + err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ DB: req.DB, Frame: req.Frame, @@ -586,7 +586,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.MsgBroker.SendSync( + err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ DB: req.DB, Frame: req.Frame, diff --git a/handler_test.go b/handler_test.go index e95e5c0dd..51f48fef7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -794,7 +794,7 @@ func NewHandler() *Handler { h.Handler.LogOutput = ioutil.Discard // Handler test messages can no-op. - h.MsgBroker = pilosa.NopMessageBroker + h.Broadcaster = pilosa.NopBroadcaster return h } @@ -828,7 +828,7 @@ func NewServer() *Server { s.Handler.Host = s.Host() // Handler test messages can no-op. - s.Handler.MsgBroker = pilosa.NopMessageBroker + s.Handler.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() diff --git a/index.go b/index.go index ab659ada1..eb8f4b204 100644 --- a/index.go +++ b/index.go @@ -23,7 +23,7 @@ type Index struct { // Databases by name. dbs map[string]*DB - MsgBroker MessageBroker + Broadcaster Broadcaster // Close management wg sync.WaitGroup closing chan struct{} @@ -246,7 +246,7 @@ func (i *Index) newDB(path, name string) (*DB, error) { } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.msgbroker = i.MsgBroker + db.broadcaster = i.Broadcaster return db, nil } diff --git a/messenger.go b/messenger.go index 22ba8d61f..0549e35fc 100644 --- a/messenger.go +++ b/messenger.go @@ -15,48 +15,48 @@ import ( "github.com/pilosa/pilosa/internal" ) -// MessageBroker is an interface for handling incoming/outgoing messages. -type MessageBroker interface { +// Broadcaster is an interface for handling incoming/outgoing messages. +type Broadcaster interface { SendSync(pb proto.Message) error SendAsync(pb proto.Message) error } func init() { - NopMessageBroker = &nopMessageBroker{} + NopBroadcaster = &nopBroadcaster{} } -var NopMessageBroker MessageBroker +var NopBroadcaster Broadcaster -// nopMessageBroker represents a MessageBroker that doesn't do anything. -type nopMessageBroker struct{} +// nopBroadcaster represents a Broadcaster that doesn't do anything. +type nopBroadcaster struct{} -// SendSync A no-op implemenetation of MessageBroker SendSync method. -func (c *nopMessageBroker) SendSync(pb proto.Message) error { - fmt.Println("NOPMessageBroker: SendSync") // TODO remove or log properly? +// SendSync A no-op implemenetation of Broadcaster SendSync method. +func (c *nopBroadcaster) SendSync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? return nil } -// SendAsync A no-op implemenetation of MessageBroker SendAsync method. -func (c *nopMessageBroker) SendAsync(pb proto.Message) error { - fmt.Println("NOPMessageBroker: SendAsync") // TODO remove or log properly? +// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +func (c *nopBroadcaster) SendAsync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? return nil } ////////////////////////////////////////////////////////////////// -// HTTPMessageBroker represents a NodeSet that broadcasts messages over HTTP. -type HTTPMessageBroker struct { +// HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. +type HTTPBroadcaster struct { server *Server } -// NewHTTPMessageBroker returns a new instance of HTTPMessageBroker. -func NewHTTPMessageBroker(s *Server) *HTTPMessageBroker { - return &HTTPMessageBroker{server: s} +// NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. +func NewHTTPBroadcaster(s *Server) *HTTPBroadcaster { + return &HTTPBroadcaster{server: s} } // SendSync sends a protobuf message to all nodes simultaneously. // It waits for all nodes to respond before the function returns (and returns any errors). -func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { +func (h *HTTPBroadcaster) SendSync(pb proto.Message) error { // Marshal the pb to []byte buf, err := MarshalMessage(pb) if err != nil { @@ -82,15 +82,15 @@ func (h *HTTPMessageBroker) SendSync(pb proto.Message) error { return g.Wait() } -// SendAsync exists to implement the MessageBroker interface, but just calls +// SendAsync exists to implement the Broadcaster interface, but just calls // SendSync. -func (h *HTTPMessageBroker) SendAsync(pb proto.Message) error { +func (h *HTTPBroadcaster) SendAsync(pb proto.Message) error { return h.SendSync(pb) } -func (h *HTTPMessageBroker) nodes() ([]*Node, error) { +func (h *HTTPBroadcaster) nodes() ([]*Node, error) { if h.server == nil { - return nil, errors.New("HTTPMessageBroker has no reference to Server.") + return nil, errors.New("HTTPBroadcaster has no reference to Server.") } nodeset, ok := h.server.Cluster.NodeSet.(*HTTPNodeSet) if !ok { @@ -99,7 +99,7 @@ func (h *HTTPMessageBroker) nodes() ([]*Node, error) { return nodeset.Nodes(), nil } -func (h *HTTPMessageBroker) sendNodeMessage(node *Node, msg []byte) error { +func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { var client *http.Client client = http.DefaultClient diff --git a/server.go b/server.go index 68f9a6921..2f0ace9ce 100644 --- a/server.go +++ b/server.go @@ -33,9 +33,9 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler - MsgBroker MessageBroker + Index *Index + Handler *Handler + Broadcaster Broadcaster // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -54,9 +54,9 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), - MsgBroker: NopMessageBroker, + Index: NewIndex(), + Handler: NewHandler(), + Broadcaster: NopBroadcaster, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -112,14 +112,14 @@ func (s *Server) Open() error { e.Cluster = s.Cluster // Initialize HTTP handler. - s.Handler.MsgBroker = s.MsgBroker + s.Handler.Broadcaster = s.Broadcaster s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput // Initialize Index. - s.Index.MsgBroker = s.MsgBroker + s.Index.Broadcaster = s.Broadcaster s.Index.LogOutput = s.LogOutput // Serve HTTP. diff --git a/server/server.go b/server/server.go index 69041ef3a..e880a1362 100644 --- a/server/server.go +++ b/server/server.go @@ -94,11 +94,11 @@ func (m *Command) Run(args ...string) (err error) { if err != nil { return err } - m.Server.MsgBroker = PilosaMessageBroker(m.Config, m.Server) + m.Server.Broadcaster = PilosaBroadcaster(m.Config, m.Server) m.Server.Cluster = PilosaCluster(m.Config) - // Associate objects to the MessageBroker based on config. - AssociateMessageBroker(m.Server, m.Config) + // Associate objects to the Broadcaster based on config. + AssociateBroadcaster(m.Server, m.Config) // Set configuration options. m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) @@ -111,17 +111,17 @@ func (m *Command) Run(args ...string) (err error) { return nil } -// PilosaMessageBroker returns a new instance of MessageBroker based on the config. -func PilosaMessageBroker(c *pilosa.Config, server *pilosa.Server) (broker pilosa.MessageBroker) { - switch c.Cluster.MessengerType { - case "broadcast": - broker = pilosa.NewHTTPMessageBroker(server) +// PilosaBroadcaster returns a new instance of Broadcaster based on the config. +func PilosaBroadcaster(c *pilosa.Config, server *pilosa.Server) (broadcaster pilosa.Broadcaster) { + switch c.Cluster.BroadcasterType { + case "http": + broadcaster = pilosa.NewHTTPBroadcaster(server) case "gossip": - broker = pilosa.NewGossipMessageBroker(server) + broadcaster = pilosa.NewGossipBroadcaster(server) case "static": - broker = pilosa.NopMessageBroker + broadcaster = pilosa.NopBroadcaster } - return broker + return broadcaster } // PilosaCluster returns a new instance of Cluster based on the config. @@ -134,8 +134,8 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { } // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.MessengerType { - case "broadcast": + switch c.Cluster.BroadcasterType { + case "http": cluster.NodeSet = pilosa.NewHTTPNodeSet() cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) case "gossip": @@ -166,14 +166,14 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { return cluster } -// AssociateMessageBroker allows an implementation to associate objects to the MessageBroker +// AssociateBroadcaster allows an implementation to associate objects to the Broadcaster // after cluster configuration. -func AssociateMessageBroker(s *pilosa.Server, c *pilosa.Config) { - switch c.Cluster.MessengerType { - case "broadcast": +func AssociateBroadcaster(s *pilosa.Server, c *pilosa.Config) { + switch c.Cluster.BroadcasterType { + case "http": // nop case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroker(s.MsgBroker.(*pilosa.GossipMessageBroker)) + s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroadcaster(s.Broadcaster.(*pilosa.GossipBroadcaster)) case "static": // nop } From 4fc2494cc0ce6d55a4da5ec3a037777458429fc6 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 13:51:27 -0500 Subject: [PATCH 28/63] add panic if Atoi on DefaultGossipPort fails --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index e880a1362..251bf77a5 100644 --- a/server/server.go +++ b/server/server.go @@ -141,7 +141,7 @@ func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { case "gossip": gport, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { - // what? + panic(err) // Atoi on a compile-time constant should never fail. } gossipPort := gport gossipSeed := pilosa.DefaultHost From bc8f7f702f10b89cfb37b3d2072b1e857614127a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 17:09:52 -0500 Subject: [PATCH 29/63] add BroadcastReceiver and collapse Gossip structs into one GossipNodeSet and GossipBroadcaster had a lot of cross dependency - I made them the same object and had it implement all three interfaces (NodeSet, Broadcaster, BroadcastReceiver). I implmented an HTTPBroadcastReceiver which runs as as separate server. BroadcastReceiver is a separate entity on the pilosa.Server object and is started separately from broadcaster and nodeset. In the case of GossipNodeSet, the broadcast receiver must be started before Open()ing the Nodeset, because it doesn't actually start listening until Open() is called, but it needs the handler set up before then. server/server.go was heavily refactored to configure the new structs and interfaces on the pilosa.Server object. --- broadcast.go | 83 +++++++++++++++++++++++++++++++++ gossip.go | 87 ++++++++++++++--------------------- handler.go | 30 ------------ messenger.go | 12 +++-- server.go | 18 +++++--- server/server.go | 117 ++++++++++++++++++++--------------------------- 6 files changed, 186 insertions(+), 161 deletions(-) create mode 100644 broadcast.go diff --git a/broadcast.go b/broadcast.go new file mode 100644 index 000000000..feaf7f55f --- /dev/null +++ b/broadcast.go @@ -0,0 +1,83 @@ +package pilosa + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/gogo/protobuf/proto" +) + +// BroadcastHandler is the interface for the pilosa object which knows how to +// handle broadcast messages. (Hint: this is implemented by pilosa.Server) +type BroadcastHandler interface { + ReceiveMessage(pb proto.Message) error +} + +// BroadcastReceiver is the interface for the object which will listen for and +// decode broadcast messages before passing them to pilosa to handle. The +// implementation of this could be an http server which listens for messages, +// gets the protobuf payload, and then passes it to +// BroadcastHandler.ReceiveMessage. +type BroadcastReceiver interface { + // Start starts listening for broadcast messages - it should return + // immediately, spawning a goroutine if necessary. + Start(BroadcastHandler) error +} + +type nopBroadcastReceiver struct{} + +func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } + +var NopBroadcastReceiver = &nopBroadcastReceiver{} + +type HTTPBroadcastReceiver struct { + port string + handler BroadcastHandler + logOutput io.Writer +} + +func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { + return &HTTPBroadcastReceiver{ + port: port, + logOutput: logOutput, + } +} + +func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { + rec.handler = b + go func() { + err := http.ListenAndServe(":"+rec.port, rec) + if err != nil { + fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) + } + }() + return nil +} + +func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := rec.handler.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} diff --git a/gossip.go b/gossip.go index da9c847f5..1224cf69b 100644 --- a/gossip.go +++ b/gossip.go @@ -1,6 +1,7 @@ package pilosa import ( + "fmt" "io" "log" "os" @@ -13,9 +14,15 @@ import ( ) // GossipNodeSet represents a gossip implementation of NodeSet using memberlist +// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist + handler BroadcastHandler + + broadcasts *memberlist.TransmitLimitedQueue + + server *Server config *GossipConfig @@ -23,10 +30,6 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) AttachBroadcaster(mb *GossipBroadcaster) { - g.config.memberlistConfig.Delegate = mb -} - func (g *GossipNodeSet) Nodes() []*Node { a := make([]*Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { @@ -35,7 +38,15 @@ func (g *GossipNodeSet) Nodes() []*Node { return a } +func (g *GossipNodeSet) Start(h BroadcastHandler) error { + g.handler = h + return nil +} + func (g *GossipNodeSet) Open() error { + if g.handler == nil { + return fmt.Errorf("opening GossipNodeSet: you must call Start(pilosa.BroadcastHandler) before calling Open()") + } ml, err := memberlist.Create(g.config.memberlistConfig) if err != nil { return err @@ -48,6 +59,12 @@ func (g *GossipNodeSet) Open() error { if err != nil { return err } + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return ml.NumMembers() + }, + RetransmitMult: 3, + } return nil } @@ -64,7 +81,7 @@ type GossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, s *Server) *GossipNodeSet { g := &GossipNodeSet{ LogOutput: os.Stderr, } @@ -79,25 +96,15 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.BindPort = gossipPort g.config.memberlistConfig.AdvertiseAddr = gossipHost g.config.memberlistConfig.AdvertisePort = gossipPort + g.config.memberlistConfig.Delegate = g + + g.server = s return g } -//////////////////////////////////////////////////////////////// - -// GossipBroadcaster represents a gossip implementation of pilosa.Broadcaster -// GossipBroadcaster also represents an implementation of memberlist.Delegate -type GossipBroadcaster struct { - broadcasts *memberlist.TransmitLimitedQueue - - server *Server - - // The writer for any logging. - LogOutput io.Writer -} - // SendSync implementation of the Broadcaster interface -func (g *GossipBroadcaster) SendSync(pb proto.Message) error { +func (g *GossipNodeSet) SendSync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -125,7 +132,7 @@ func (g *GossipBroadcaster) SendSync(pb proto.Message) error { } // SendAsync implementation of the Broadcaster interface -func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { +func (g *GossipNodeSet) SendAsync(pb proto.Message) error { msg, err := MarshalMessage(pb) if err != nil { return err @@ -139,19 +146,19 @@ func (g *GossipBroadcaster) SendAsync(pb proto.Message) error { return nil } -func (g *GossipBroadcaster) Receive(pb proto.Message) error { - if err := g.server.ReceiveMessage(pb); err != nil { +func (g *GossipNodeSet) Receive(pb proto.Message) error { + if err := g.handler.ReceiveMessage(pb); err != nil { return err } return nil } // implementation of the memberlist.Delegate interface -func (g *GossipBroadcaster) NodeMeta(limit int) []byte { +func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} } -func (g *GossipBroadcaster) NotifyMsg(b []byte) { +func (g *GossipNodeSet) NotifyMsg(b []byte) { m, err := UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) @@ -163,11 +170,11 @@ func (g *GossipBroadcaster) NotifyMsg(b []byte) { } } -func (g *GossipBroadcaster) GetBroadcasts(overhead, limit int) [][]byte { +func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { return g.broadcasts.GetBroadcasts(overhead, limit) } -func (g *GossipBroadcaster) LocalState(join bool) []byte { +func (g *GossipNodeSet) LocalState(join bool) []byte { pb, err := g.server.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) @@ -183,7 +190,7 @@ func (g *GossipBroadcaster) LocalState(join bool) []byte { return buf } -func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { +func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. var pb internal.NodeState if err := proto.Unmarshal(buf, &pb); err != nil { @@ -196,32 +203,6 @@ func (g *GossipBroadcaster) MergeRemoteState(buf []byte, join bool) { } } -// logger returns a logger for the GossipBroadcaster -func (g *GossipBroadcaster) logger() *log.Logger { - return log.New(g.LogOutput, "", log.LstdFlags) -} - -//////////////////////////////////////////////////////////////// - -// NewGossipBroadcaster returns a new instance of GossipBroadcaster. -func NewGossipBroadcaster(s *Server) *GossipBroadcaster { - g := &GossipBroadcaster{ - LogOutput: os.Stderr, - server: s, - } - - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist.NumMembers() - }, - RetransmitMult: 3, - } - - return g -} - -//////////////////////////////////////////////////////////////// - // broadcast represents an implementation of memberlist.Broadcast type broadcast struct { msg []byte diff --git a/handler.go b/handler.go index 83f969d22..94b99bbe0 100644 --- a/handler.go +++ b/handler.go @@ -192,36 +192,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handlePostMessage handles /message requests. -func (h *Handler) handlePostMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.Server.ReceiveMessage(m); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - return -} - func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { diff --git a/messenger.go b/messenger.go index 0549e35fc..f0b1e76a6 100644 --- a/messenger.go +++ b/messenger.go @@ -11,6 +11,8 @@ import ( "golang.org/x/sync/errgroup" + "net" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -46,11 +48,12 @@ func (c *nopBroadcaster) SendAsync(pb proto.Message) error { // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { - server *Server + server *Server + internalPort string } // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. -func NewHTTPBroadcaster(s *Server) *HTTPBroadcaster { +func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { return &HTTPBroadcaster{server: s} } @@ -103,11 +106,12 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { var client *http.Client client = http.DefaultClient + host, _, err := net.SplitHostPort(node.Host) + // Create HTTP request. req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", - Host: node.Host, - Path: "/message", + Host: host + ":" + h.internalPort, }).String(), bytes.NewReader(msg)) if err != nil { return err diff --git a/server.go b/server.go index 2f0ace9ce..111e94ee7 100644 --- a/server.go +++ b/server.go @@ -33,9 +33,10 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index - Handler *Handler - Broadcaster Broadcaster + Index *Index + Handler *Handler + Broadcaster Broadcaster + BroadcastReceiver BroadcastReceiver // Cluster configuration. // Host is replaced with actual host after opening if port is ":0". @@ -54,9 +55,10 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), - Handler: NewHandler(), - Broadcaster: NopBroadcaster, + Index: NewIndex(), + Handler: NewHandler(), + Broadcaster: NopBroadcaster, + BroadcastReceiver: NopBroadcastReceiver, AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, @@ -100,6 +102,10 @@ func (s *Server) Open() error { return err } + if err := s.BroadcastReceiver.Start(s); err != nil { + return err + } + // Open NodeSet communication if err := s.Cluster.NodeSet.Open(); err != nil { return err diff --git a/server/server.go b/server/server.go index 251bf77a5..9d00eabfb 100644 --- a/server/server.go +++ b/server/server.go @@ -73,6 +73,29 @@ func (m *Command) Run(args ...string) (err error) { m.Config.DataDir = filepath.Join(HomeDir, strings.TrimPrefix(m.Config.DataDir, prefix)) } + // SetupServer + err = m.SetupServer() + if err != nil { + return err + } + + // Initialize server. + if err = m.Server.Open(); err != nil { + return fmt.Errorf("server.Open: %v", err) + } + fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) + return nil +} + +func (m *Command) SetupServer() error { + cluster := pilosa.NewCluster() + cluster.ReplicaN = m.Config.Cluster.ReplicaN + + for _, hostport := range m.Config.Cluster.Nodes { + cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) + } + m.Server.Cluster = cluster + // Setup logging output. if m.Config.LogPath == "" { m.Server.LogOutput = m.Stderr @@ -90,93 +113,51 @@ func (m *Command) Run(args ...string) (err error) { m.Server.Index.Stats = pilosa.NewExpvarStatsClient() // Build cluster from config file. + var err error m.Server.Host, err = normalizeHost(m.Config.Host) if err != nil { return err } - m.Server.Broadcaster = PilosaBroadcaster(m.Config, m.Server) - m.Server.Cluster = PilosaCluster(m.Config) - // Associate objects to the Broadcaster based on config. - AssociateBroadcaster(m.Server, m.Config) - - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - - // Initialize server. - if err = m.Server.Open(); err != nil { - return fmt.Errorf("server.Open: %v", err) - } - fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host) - return nil -} - -// PilosaBroadcaster returns a new instance of Broadcaster based on the config. -func PilosaBroadcaster(c *pilosa.Config, server *pilosa.Server) (broadcaster pilosa.Broadcaster) { - switch c.Cluster.BroadcasterType { + switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": - broadcaster = pilosa.NewHTTPBroadcaster(server) + port := strconv.Itoa(m.Config.Cluster.Gossip.Port) + m.Server.Broadcaster = pilosa.NewHTTPBroadcaster(m.Server, port) + m.Server.BroadcastReceiver = pilosa.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Cluster.NodeSet = pilosa.NewHTTPNodeSet() + m.Server.Cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(m.Server.Cluster.Nodes) case "gossip": - broadcaster = pilosa.NewGossipBroadcaster(server) - case "static": - broadcaster = pilosa.NopBroadcaster - } - return broadcaster -} - -// PilosaCluster returns a new instance of Cluster based on the config. -func PilosaCluster(c *pilosa.Config) *pilosa.Cluster { - cluster := pilosa.NewCluster() - cluster.ReplicaN = c.Cluster.ReplicaN - - for _, hostport := range c.Cluster.Nodes { - cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) - } - - // Setup a Broadcast (over HTTP) or Gossip NodeSet based on config. - switch c.Cluster.BroadcasterType { - case "http": - cluster.NodeSet = pilosa.NewHTTPNodeSet() - cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(cluster.Nodes) - case "gossip": - gport, err := strconv.Atoi(pilosa.DefaultGossipPort) + gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { panic(err) // Atoi on a compile-time constant should never fail. } - gossipPort := gport gossipSeed := pilosa.DefaultHost - if c.Cluster.Gossip.Port != 0 { - gossipPort = c.Cluster.Gossip.Port + if m.Config.Cluster.Gossip.Port != 0 { + gossipPort = m.Config.Cluster.Gossip.Port } - if c.Cluster.Gossip.Seed != "" { - gossipSeed = c.Cluster.Gossip.Seed + if m.Config.Cluster.Gossip.Seed != "" { + gossipSeed = m.Config.Cluster.Gossip.Seed } // get the host portion of addr to use for binding - gossipHost, _, err := net.SplitHostPort(c.Host) + gossipHost, _, err := net.SplitHostPort(m.Config.Host) if err != nil { - gossipHost = c.Host + gossipHost = m.Config.Host } - cluster.NodeSet = pilosa.NewGossipNodeSet(c.Host, gossipHost, gossipPort, gossipSeed) - case "static": - cluster.NodeSet = pilosa.NewStaticNodeSet() + gossipNodeSet := pilosa.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) + m.Server.Cluster.NodeSet = gossipNodeSet + m.Server.Broadcaster = gossipNodeSet + m.Server.BroadcastReceiver = gossipNodeSet + case "static", "": + m.Server.Broadcaster = pilosa.NopBroadcaster + m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() + m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver default: - cluster.NodeSet = pilosa.NewStaticNodeSet() + return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.BroadcasterType) } - return cluster -} - -// AssociateBroadcaster allows an implementation to associate objects to the Broadcaster -// after cluster configuration. -func AssociateBroadcaster(s *pilosa.Server, c *pilosa.Config) { - switch c.Cluster.BroadcasterType { - case "http": - // nop - case "gossip": - s.Cluster.NodeSet.(*pilosa.GossipNodeSet).AttachBroadcaster(s.Broadcaster.(*pilosa.GossipBroadcaster)) - case "static": - // nop - } + // Set configuration options. + m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) + return nil } func normalizeHost(host string) (string, error) { From 2d139d299c123426431ddec54b85eab4c5f55864 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 22:05:27 -0500 Subject: [PATCH 30/63] remove server from handler - not used anymore --- handler.go | 1 - server.go | 1 - 2 files changed, 2 deletions(-) diff --git a/handler.go b/handler.go index 94b99bbe0..c8dda53c2 100644 --- a/handler.go +++ b/handler.go @@ -27,7 +27,6 @@ import ( type Handler struct { Index *Index Broadcaster Broadcaster - Server *Server // Local hostname & cluster configuration. Host string diff --git a/server.go b/server.go index 111e94ee7..229214805 100644 --- a/server.go +++ b/server.go @@ -67,7 +67,6 @@ func NewServer() *Server { } s.Handler.Index = s.Index - s.Handler.Server = s // TODO remove return s } From 059403e6731d18d687d2ac099f721a6ab1219b2f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Apr 2017 22:31:32 -0500 Subject: [PATCH 31/63] add stateHandler interface to gossip now it doesn't need to directly reference pilosa.Server --- gossip.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gossip.go b/gossip.go index 1224cf69b..664b91aa7 100644 --- a/gossip.go +++ b/gossip.go @@ -13,6 +13,14 @@ import ( "github.com/pilosa/pilosa/internal" ) +// StateHandler specifies two methods which an object must implement to share +// state in the cluster. These are used by the GossipNodeSet to implement the +// LocalState and MergeRemoteState methods of memberlist.Delegate +type StateHandler interface { + LocalState() (proto.Message, error) + HandleRemoteState(proto.Message) error +} + // GossipNodeSet represents a gossip implementation of NodeSet using memberlist // GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster // GossipNodeSet also represents an implementation of memberlist.Delegate @@ -22,9 +30,8 @@ type GossipNodeSet struct { broadcasts *memberlist.TransmitLimitedQueue - server *Server - - config *GossipConfig + stateHandler StateHandler + config *GossipConfig // The writer for any logging. LogOutput io.Writer @@ -81,7 +88,7 @@ type GossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, s *Server) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh StateHandler) *GossipNodeSet { g := &GossipNodeSet{ LogOutput: os.Stderr, } @@ -98,7 +105,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.AdvertisePort = gossipPort g.config.memberlistConfig.Delegate = g - g.server = s + g.stateHandler = sh return g } @@ -110,7 +117,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { return err } - mlist := g.server.Cluster.NodeSet.(*GossipNodeSet).memberlist + mlist := g.memberlist // Direct sends the message directly to every node. // An error from any node raises an error on the entire operation. @@ -175,7 +182,7 @@ func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { } func (g *GossipNodeSet) LocalState(join bool) []byte { - pb, err := g.server.LocalState() + pb, err := g.stateHandler.LocalState() if err != nil { g.logger().Printf("error getting local state, err=%s", err) return []byte{} @@ -197,7 +204,7 @@ func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { g.logger().Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.server.HandleRemoteState(&pb) + err := g.stateHandler.HandleRemoteState(&pb) if err != nil { g.logger().Printf("merge state error: %s", err) } From c4d3bde50198802c3df44369c165eacd62b0b14a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 08:41:23 -0500 Subject: [PATCH 32/63] move gossip code to separate package --- gossip.go => gossip/gossip.go | 23 ++++++++++++----------- server/server.go | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) rename gossip.go => gossip/gossip.go (90%) diff --git a/gossip.go b/gossip/gossip.go similarity index 90% rename from gossip.go rename to gossip/gossip.go index 664b91aa7..050435441 100644 --- a/gossip.go +++ b/gossip/gossip.go @@ -1,4 +1,4 @@ -package pilosa +package gossip import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" + "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" ) @@ -26,7 +27,7 @@ type StateHandler interface { // GossipNodeSet also represents an implementation of memberlist.Delegate type GossipNodeSet struct { memberlist *memberlist.Memberlist - handler BroadcastHandler + handler pilosa.BroadcastHandler broadcasts *memberlist.TransmitLimitedQueue @@ -37,15 +38,15 @@ type GossipNodeSet struct { LogOutput io.Writer } -func (g *GossipNodeSet) Nodes() []*Node { - a := make([]*Node, 0, g.memberlist.NumMembers()) +func (g *GossipNodeSet) Nodes() []*pilosa.Node { + a := make([]*pilosa.Node, 0, g.memberlist.NumMembers()) for _, n := range g.memberlist.Members() { - a = append(a, &Node{Host: n.Name}) + a = append(a, &pilosa.Node{Host: n.Name}) } return a } -func (g *GossipNodeSet) Start(h BroadcastHandler) error { +func (g *GossipNodeSet) Start(h pilosa.BroadcastHandler) error { g.handler = h return nil } @@ -61,8 +62,8 @@ func (g *GossipNodeSet) Open() error { g.memberlist = ml // attach to gossip seed node - nodes := []*Node{&Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds - _, err = g.memberlist.Join(Nodes(nodes).Hosts()) + nodes := []*pilosa.Node{&pilosa.Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds + _, err = g.memberlist.Join(pilosa.Nodes(nodes).Hosts()) if err != nil { return err } @@ -112,7 +113,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed // SendSync implementation of the Broadcaster interface func (g *GossipNodeSet) SendSync(pb proto.Message) error { - msg, err := MarshalMessage(pb) + msg, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -140,7 +141,7 @@ func (g *GossipNodeSet) SendSync(pb proto.Message) error { // SendAsync implementation of the Broadcaster interface func (g *GossipNodeSet) SendAsync(pb proto.Message) error { - msg, err := MarshalMessage(pb) + msg, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -166,7 +167,7 @@ func (g *GossipNodeSet) NodeMeta(limit int) []byte { } func (g *GossipNodeSet) NotifyMsg(b []byte) { - m, err := UnmarshalMessage(b) + m, err := pilosa.UnmarshalMessage(b) if err != nil { g.logger().Printf("unmarshal message error: %s", err) return diff --git a/server/server.go b/server/server.go index 9d00eabfb..80c0457f8 100644 --- a/server/server.go +++ b/server/server.go @@ -17,6 +17,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/gossip" ) func init() { @@ -143,7 +144,7 @@ func (m *Command) SetupServer() error { if err != nil { gossipHost = m.Config.Host } - gossipNodeSet := pilosa.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) + gossipNodeSet := gossip.NewGossipNodeSet(m.Config.Host, gossipHost, gossipPort, gossipSeed, m.Server) m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet From 802fd32988c66927b43401f911ec090b474c1519 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 08:59:46 -0500 Subject: [PATCH 33/63] separate broadcast interfaces from implementations interfaces are in broadcast.go and http implementations in messenger.go (gossip is already in a separate package). Next changes will bring in the HTTP nodestate implementation, and separate the http implementation into a separate package. --- broadcast.go | 126 +++++++++++++++++++++++++++++++------------------- messenger.go | 128 ++++++++++++++++++--------------------------------- 2 files changed, 124 insertions(+), 130 deletions(-) diff --git a/broadcast.go b/broadcast.go index feaf7f55f..d56576b2c 100644 --- a/broadcast.go +++ b/broadcast.go @@ -2,13 +2,39 @@ package pilosa import ( "fmt" - "io" - "io/ioutil" - "net/http" + "reflect" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) +// Broadcaster is an interface for broadcasting messages. +type Broadcaster interface { + SendSync(pb proto.Message) error + SendAsync(pb proto.Message) error +} + +func init() { + NopBroadcaster = &nopBroadcaster{} +} + +var NopBroadcaster Broadcaster + +// nopBroadcaster represents a Broadcaster that doesn't do anything. +type nopBroadcaster struct{} + +// SendSync A no-op implemenetation of Broadcaster SendSync method. +func (c *nopBroadcaster) SendSync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? + return nil +} + +// SendAsync A no-op implemenetation of Broadcaster SendAsync method. +func (c *nopBroadcaster) SendAsync(pb proto.Message) error { + fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? + return nil +} + // BroadcastHandler is the interface for the pilosa object which knows how to // handle broadcast messages. (Hint: this is implemented by pilosa.Server) type BroadcastHandler interface { @@ -32,52 +58,58 @@ func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil } var NopBroadcastReceiver = &nopBroadcastReceiver{} -type HTTPBroadcastReceiver struct { - port string - handler BroadcastHandler - logOutput io.Writer -} +const ( + MessageTypeCreateSlice = 1 + MessageTypeCreateDB = 2 + MessageTypeDeleteDB = 3 + MessageTypeCreateFrame = 4 + MessageTypeDeleteFrame = 5 +) -func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { - return &HTTPBroadcastReceiver{ - port: port, - logOutput: logOutput, +func MarshalMessage(m proto.Message) ([]byte, error) { + var typ uint8 + switch obj := m.(type) { + case *internal.CreateSliceMessage: + typ = MessageTypeCreateSlice + case *internal.CreateDBMessage: + typ = MessageTypeCreateDB + case *internal.DeleteDBMessage: + typ = MessageTypeDeleteDB + case *internal.CreateFrameMessage: + typ = MessageTypeCreateFrame + case *internal.DeleteFrameMessage: + typ = MessageTypeDeleteFrame + default: + return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } -} - -func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { - rec.handler = b - go func() { - err := http.ListenAndServe(":"+rec.port, rec) - if err != nil { - fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) - } - }() - return nil -} - -func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) + buf, err := proto.Marshal(m) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := rec.handler.ReceiveMessage(m); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + return nil, err } + return append([]byte{typ}, buf...), nil +} + +func UnmarshalMessage(buf []byte) (proto.Message, error) { + typ, buf := buf[0], buf[1:] + + var m proto.Message + switch typ { + case MessageTypeCreateSlice: + m = &internal.CreateSliceMessage{} + case MessageTypeCreateDB: + m = &internal.CreateDBMessage{} + case MessageTypeDeleteDB: + m = &internal.DeleteDBMessage{} + case MessageTypeCreateFrame: + m = &internal.CreateFrameMessage{} + case MessageTypeDeleteFrame: + m = &internal.DeleteFrameMessage{} + default: + return nil, fmt.Errorf("invalid message type: %d", typ) + } + + if err := proto.Unmarshal(buf, m); err != nil { + return nil, err + } + return m, nil } diff --git a/messenger.go b/messenger.go index f0b1e76a6..78a1d1696 100644 --- a/messenger.go +++ b/messenger.go @@ -4,48 +4,18 @@ import ( "bytes" "errors" "fmt" + "io" "io/ioutil" "net/http" "net/url" - "reflect" "golang.org/x/sync/errgroup" "net" "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" ) -// Broadcaster is an interface for handling incoming/outgoing messages. -type Broadcaster interface { - SendSync(pb proto.Message) error - SendAsync(pb proto.Message) error -} - -func init() { - NopBroadcaster = &nopBroadcaster{} -} - -var NopBroadcaster Broadcaster - -// nopBroadcaster represents a Broadcaster that doesn't do anything. -type nopBroadcaster struct{} - -// SendSync A no-op implemenetation of Broadcaster SendSync method. -func (c *nopBroadcaster) SendSync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? - return nil -} - -// SendAsync A no-op implemenetation of Broadcaster SendAsync method. -func (c *nopBroadcaster) SendAsync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? - return nil -} - -////////////////////////////////////////////////////////////////// - // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { server *Server @@ -142,60 +112,52 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { return nil } -////////////////////////////////////////////////////////////////// +type HTTPBroadcastReceiver struct { + port string + handler BroadcastHandler + logOutput io.Writer +} -const ( - MessageTypeCreateSlice = 1 - MessageTypeCreateDB = 2 - MessageTypeDeleteDB = 3 - MessageTypeCreateFrame = 4 - MessageTypeDeleteFrame = 5 -) - -func MarshalMessage(m proto.Message) ([]byte, error) { - var typ uint8 - switch obj := m.(type) { - case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice - case *internal.CreateDBMessage: - typ = MessageTypeCreateDB - case *internal.DeleteDBMessage: - typ = MessageTypeDeleteDB - case *internal.CreateFrameMessage: - typ = MessageTypeCreateFrame - case *internal.DeleteFrameMessage: - typ = MessageTypeDeleteFrame - default: - return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) +func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastReceiver { + return &HTTPBroadcastReceiver{ + port: port, + logOutput: logOutput, } - buf, err := proto.Marshal(m) +} + +func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { + rec.handler = b + go func() { + err := http.ListenAndServe(":"+rec.port, rec) + if err != nil { + fmt.Fprintf(rec.logOutput, "Error listening on %v for HTTPBroadcastReceiver: %v\n", ":"+rec.port, err) + } + }() + return nil +} + +func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) if err != nil { - return nil, err + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Unmarshal message to specific proto type. + m, err := UnmarshalMessage(body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := rec.handler.ReceiveMessage(m); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return } - return append([]byte{typ}, buf...), nil -} - -func UnmarshalMessage(buf []byte) (proto.Message, error) { - typ, buf := buf[0], buf[1:] - - var m proto.Message - switch typ { - case MessageTypeCreateSlice: - m = &internal.CreateSliceMessage{} - case MessageTypeCreateDB: - m = &internal.CreateDBMessage{} - case MessageTypeDeleteDB: - m = &internal.DeleteDBMessage{} - case MessageTypeCreateFrame: - m = &internal.CreateFrameMessage{} - case MessageTypeDeleteFrame: - m = &internal.DeleteFrameMessage{} - default: - return nil, fmt.Errorf("invalid message type: %d", typ) - } - - if err := proto.Unmarshal(buf, m); err != nil { - return nil, err - } - return m, nil } From d2621709f13a1485c36b2edc752c244ecf9a2c09 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 09:19:34 -0500 Subject: [PATCH 34/63] move NodeSet out of cluster.go into broadcast and messenger HTTP implementation will be split into separate package. --- broadcast.go | 26 +++++++++++++++++++++++ cluster.go | 59 ---------------------------------------------------- messenger.go | 23 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 59 deletions(-) diff --git a/broadcast.go b/broadcast.go index d56576b2c..6eaf834cc 100644 --- a/broadcast.go +++ b/broadcast.go @@ -8,6 +8,32 @@ import ( "github.com/pilosa/pilosa/internal" ) +// NodeSet represents an interface for Node membership and inter-node communication. +type NodeSet interface { + // Returns a list of all Nodes in the cluster + Nodes() []*Node + + // Open starts any network activity implemented by the NodeSet + Open() error +} + +// StaticNodeSet represents a basic NodeSet for testing +type StaticNodeSet struct { + nodes []*Node +} + +func NewStaticNodeSet() *StaticNodeSet { + return &StaticNodeSet{} +} + +func (s *StaticNodeSet) Nodes() []*Node { + return s.nodes +} + +func (s *StaticNodeSet) Open() error { + return nil +} + // Broadcaster is an interface for broadcasting messages. type Broadcaster interface { SendSync(pb proto.Message) error diff --git a/cluster.go b/cluster.go index ef67b7a4a..d8f4401e3 100644 --- a/cluster.go +++ b/cluster.go @@ -3,8 +3,6 @@ package pilosa import ( "encoding/binary" "hash/fnv" - - "github.com/gogo/protobuf/proto" ) const ( @@ -191,15 +189,6 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// NodeSet represents an interface for Node membership and inter-node communication. -type NodeSet interface { - // Returns a list of all Nodes in the cluster - Nodes() []*Node - - // Open starts any network activity implemented by the NodeSet - Open() error -} - // Hasher represents an interface to hash integers into buckets. type Hasher interface { // Hashes the key into a number between [0,N). @@ -222,51 +211,3 @@ func (h *jmphasher) Hash(key uint64, n int) int { } return int(b) } - -// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. -type HTTPNodeSet struct { - nodes []*Node -} - -// NewHTTPNodeSet returns a new instance of HTTPNodeSet. -func NewHTTPNodeSet() *HTTPNodeSet { - return &HTTPNodeSet{} -} - -func (h *HTTPNodeSet) Nodes() []*Node { - return h.nodes -} - -func (h *HTTPNodeSet) Open() error { - return nil -} - -func (h *HTTPNodeSet) Join(nodes []*Node) error { - h.nodes = nodes - return nil -} - -// StaticNodeSet represents a basic NodeSet for testing -type StaticNodeSet struct { - nodes []*Node -} - -func NewStaticNodeSet() *StaticNodeSet { - return &StaticNodeSet{} -} - -func (s *StaticNodeSet) Nodes() []*Node { - return s.nodes -} - -func (s *StaticNodeSet) Open() error { - return nil -} - -func (s *StaticNodeSet) SendSync(pb proto.Message) error { - return nil -} - -func (s *StaticNodeSet) SendAsync(pb proto.Message) error { - return nil -} diff --git a/messenger.go b/messenger.go index 78a1d1696..37be45c0c 100644 --- a/messenger.go +++ b/messenger.go @@ -161,3 +161,26 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque return } } + +// HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. +type HTTPNodeSet struct { + nodes []*Node +} + +// NewHTTPNodeSet returns a new instance of HTTPNodeSet. +func NewHTTPNodeSet() *HTTPNodeSet { + return &HTTPNodeSet{} +} + +func (h *HTTPNodeSet) Nodes() []*Node { + return h.nodes +} + +func (h *HTTPNodeSet) Open() error { + return nil +} + +func (h *HTTPNodeSet) Join(nodes []*Node) error { + h.nodes = nodes + return nil +} From 6e8bf5dcc16f78fb6a68ac9a401d7d6659d51209 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 09:43:50 -0500 Subject: [PATCH 35/63] move http messenger stuff to separate package --- cluster_test.go | 5 +++-- messenger.go => httpbroadcast/messenger.go | 25 +++++++++++----------- server/server.go | 12 +++++++---- 3 files changed, 24 insertions(+), 18 deletions(-) rename messenger.go => httpbroadcast/messenger.go (85%) diff --git a/cluster_test.go b/cluster_test.go index 1dd6bb486..9dbaa420d 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -9,6 +9,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/httpbroadcast" ) // Ensure the cluster can fairly distribute partitions across the nodes. @@ -95,10 +96,10 @@ func TestCluster_Health(t *testing.T) { {Host: "serverB:1000"}, {Host: "serverC:1000"}, }, - NodeSet: &pilosa.HTTPNodeSet{}, + NodeSet: &httpbroadcast.HTTPNodeSet{}, } - err := c.NodeSet.(*pilosa.HTTPNodeSet).Join([]*pilosa.Node{ + err := c.NodeSet.(*httpbroadcast.HTTPNodeSet).Join([]*pilosa.Node{ &pilosa.Node{Host: "serverA:1000"}, &pilosa.Node{Host: "serverC:1000"}, &pilosa.Node{Host: "serverD:1000"}, diff --git a/messenger.go b/httpbroadcast/messenger.go similarity index 85% rename from messenger.go rename to httpbroadcast/messenger.go index 37be45c0c..1545e3328 100644 --- a/messenger.go +++ b/httpbroadcast/messenger.go @@ -1,4 +1,4 @@ -package pilosa +package httpbroadcast import ( "bytes" @@ -14,16 +14,17 @@ import ( "net" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" ) // HTTPBroadcaster represents a NodeSet that broadcasts messages over HTTP. type HTTPBroadcaster struct { - server *Server + server *pilosa.Server internalPort string } // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. -func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { +func NewHTTPBroadcaster(s *pilosa.Server, internalPort string) *HTTPBroadcaster { return &HTTPBroadcaster{server: s} } @@ -31,7 +32,7 @@ func NewHTTPBroadcaster(s *Server, internalPort string) *HTTPBroadcaster { // It waits for all nodes to respond before the function returns (and returns any errors). func (h *HTTPBroadcaster) SendSync(pb proto.Message) error { // Marshal the pb to []byte - buf, err := MarshalMessage(pb) + buf, err := pilosa.MarshalMessage(pb) if err != nil { return err } @@ -61,7 +62,7 @@ func (h *HTTPBroadcaster) SendAsync(pb proto.Message) error { return h.SendSync(pb) } -func (h *HTTPBroadcaster) nodes() ([]*Node, error) { +func (h *HTTPBroadcaster) nodes() ([]*pilosa.Node, error) { if h.server == nil { return nil, errors.New("HTTPBroadcaster has no reference to Server.") } @@ -72,7 +73,7 @@ func (h *HTTPBroadcaster) nodes() ([]*Node, error) { return nodeset.Nodes(), nil } -func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { +func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { var client *http.Client client = http.DefaultClient @@ -114,7 +115,7 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *Node, msg []byte) error { type HTTPBroadcastReceiver struct { port string - handler BroadcastHandler + handler pilosa.BroadcastHandler logOutput io.Writer } @@ -125,7 +126,7 @@ func NewHTTPBroadcastReceiver(port string, logOutput io.Writer) *HTTPBroadcastRe } } -func (rec *HTTPBroadcastReceiver) Start(b BroadcastHandler) error { +func (rec *HTTPBroadcastReceiver) Start(b pilosa.BroadcastHandler) error { rec.handler = b go func() { err := http.ListenAndServe(":"+rec.port, rec) @@ -150,7 +151,7 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque } // Unmarshal message to specific proto type. - m, err := UnmarshalMessage(body) + m, err := pilosa.UnmarshalMessage(body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -164,7 +165,7 @@ func (rec *HTTPBroadcastReceiver) ServeHTTP(w http.ResponseWriter, r *http.Reque // HTTPNodeSet represents a NodeSet that broadcasts messages over HTTP. type HTTPNodeSet struct { - nodes []*Node + nodes []*pilosa.Node } // NewHTTPNodeSet returns a new instance of HTTPNodeSet. @@ -172,7 +173,7 @@ func NewHTTPNodeSet() *HTTPNodeSet { return &HTTPNodeSet{} } -func (h *HTTPNodeSet) Nodes() []*Node { +func (h *HTTPNodeSet) Nodes() []*pilosa.Node { return h.nodes } @@ -180,7 +181,7 @@ func (h *HTTPNodeSet) Open() error { return nil } -func (h *HTTPNodeSet) Join(nodes []*Node) error { +func (h *HTTPNodeSet) Join(nodes []*pilosa.Node) error { h.nodes = nodes return nil } diff --git a/server/server.go b/server/server.go index 80c0457f8..3571dcf3c 100644 --- a/server/server.go +++ b/server/server.go @@ -18,6 +18,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/httpbroadcast" ) func init() { @@ -123,10 +124,13 @@ func (m *Command) SetupServer() error { switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": port := strconv.Itoa(m.Config.Cluster.Gossip.Port) - m.Server.Broadcaster = pilosa.NewHTTPBroadcaster(m.Server, port) - m.Server.BroadcastReceiver = pilosa.NewHTTPBroadcastReceiver(port, m.Stderr) - m.Server.Cluster.NodeSet = pilosa.NewHTTPNodeSet() - m.Server.Cluster.NodeSet.(*pilosa.HTTPNodeSet).Join(m.Server.Cluster.Nodes) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() + err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) + if err != nil { + return err + } case "gossip": gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) if err != nil { From 0da580d0e902e34671ffcff2de613bced9d7e749 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:18:43 -0500 Subject: [PATCH 36/63] change broadcaster-type to type It is nested under "cluster" in the config, and it controls cluster membership as well as broadcasting, so I think type is more appropriate. Also, brevity. --- cmd/server.go | 2 +- config.go | 12 ++++++------ server/server.go | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index e9d7fcfdf..2dd662bbe 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -83,7 +83,7 @@ on the configured port.`, flags.DurationVarP((*time.Duration)(&Server.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&Server.Config.Cluster.BroadcasterType, "cluster.broadcaster-type", "", "static", "Type of Broadcaster to use for inter-host messaging. Choose from [static, http, gossip]") + flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") diff --git a/config.go b/config.go index 8b718f4e1..094bb9d65 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultBroadcasterType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultClusterType = "static" + DefaultGossipPort = "14000" ) // Config represents the configuration for the command. @@ -17,7 +17,7 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` - BroadcasterType string `toml:"broadcaster-type"` + Type string `toml:"type"` Nodes []string `toml:"hosts"` PollingInterval Duration `toml:"polling-interval"` Gossip ConfigGossip `toml:"gossip"` @@ -45,7 +45,7 @@ func NewConfig() *Config { Host: DefaultHost + ":" + DefaultPort, } c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.BroadcasterType = DefaultBroadcasterType + c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) diff --git a/server/server.go b/server/server.go index 3571dcf3c..16d17407f 100644 --- a/server/server.go +++ b/server/server.go @@ -114,14 +114,13 @@ func (m *Command) SetupServer() error { m.Server.Index.Path = m.Config.DataDir m.Server.Index.Stats = pilosa.NewExpvarStatsClient() - // Build cluster from config file. var err error m.Server.Host, err = normalizeHost(m.Config.Host) if err != nil { return err } - switch m.Config.Cluster.BroadcasterType { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership + switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": port := strconv.Itoa(m.Config.Cluster.Gossip.Port) m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) @@ -157,7 +156,7 @@ func (m *Command) SetupServer() error { m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver default: - return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.BroadcasterType) + return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type) } // Set configuration options. From d10d8d94e1ee57b1555478c17f1d71f5f8567443 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:31:17 -0500 Subject: [PATCH 37/63] simplify config structure - remove gossip sub-struct --- cmd/server.go | 4 ++-- config.go | 16 ++++++---------- server/server.go | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 2dd662bbe..51d92824a 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -84,8 +84,8 @@ on the configured port.`, flags.StringVarP(&Server.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&Server.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]") - flags.StringVarP(&Server.Config.Cluster.Gossip.Seed, "cluster.gossip.seed", "", "", "Host with which to seed the gossip membership.") - flags.IntVarP(&Server.Config.Cluster.Gossip.Port, "cluster.gossip.port", "", 0, "Port to which pilosa should bind for gossip.") + flags.StringVarP(&Server.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.") + flags.StringVarP(&Server.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.") return serveCmd } diff --git a/config.go b/config.go index 094bb9d65..22750d72d 100644 --- a/config.go +++ b/config.go @@ -16,11 +16,12 @@ type Config struct { Host string `toml:"host"` Cluster struct { - ReplicaN int `toml:"replicas"` - Type string `toml:"type"` - Nodes []string `toml:"hosts"` - PollingInterval Duration `toml:"polling-interval"` - Gossip ConfigGossip `toml:"gossip"` + ReplicaN int `toml:"replicas"` + Type string `toml:"type"` + Nodes []string `toml:"hosts"` + PollingInterval Duration `toml:"polling-interval"` + InternalPort string `toml:"internal-port"` + GossipSeed string `toml:"gossip-seed"` } `toml:"cluster"` Plugins struct { @@ -34,11 +35,6 @@ type Config struct { LogPath string `toml:"log-path"` } -type ConfigGossip struct { - Port int `toml:"port"` - Seed string `toml:"seed"` -} - // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ diff --git a/server/server.go b/server/server.go index 16d17407f..a627daade 100644 --- a/server/server.go +++ b/server/server.go @@ -122,25 +122,25 @@ func (m *Command) SetupServer() error { switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership case "http": - port := strconv.Itoa(m.Config.Cluster.Gossip.Port) - m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, port) - m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(port, m.Stderr) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) if err != nil { return err } case "gossip": - gossipPort, err := strconv.Atoi(pilosa.DefaultGossipPort) + gossipPortStr := pilosa.DefaultGossipPort + if m.Config.Cluster.InternalPort != "" { + gossipPortStr = m.Config.Cluster.InternalPort + } + gossipPort, err := strconv.Atoi(gossipPortStr) if err != nil { - panic(err) // Atoi on a compile-time constant should never fail. + return err } gossipSeed := pilosa.DefaultHost - if m.Config.Cluster.Gossip.Port != 0 { - gossipPort = m.Config.Cluster.Gossip.Port - } - if m.Config.Cluster.Gossip.Seed != "" { - gossipSeed = m.Config.Cluster.Gossip.Seed + if m.Config.Cluster.GossipSeed != "" { + gossipSeed = m.Config.Cluster.GossipSeed } // get the host portion of addr to use for binding gossipHost, _, err := net.SplitHostPort(m.Config.Host) From a11daecfcbdecb0e409b3d24a5268f4095eaf0d0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Apr 2017 11:34:27 -0500 Subject: [PATCH 38/63] remove done TODO --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index a627daade..1bf3fe5ce 100644 --- a/server/server.go +++ b/server/server.go @@ -120,7 +120,7 @@ func (m *Command) SetupServer() error { return err } - switch m.Config.Cluster.Type { // TODO change name to something that encompasses broadcasting, receiving broadcasts, and tracking cluster membership + switch m.Config.Cluster.Type { case "http": m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) From b4a69be751f20f8e5c076e2b58af0088eb051699 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 09:09:21 -0500 Subject: [PATCH 39/63] Add support for Node.InternalHost. This was required for the HTTPBroadcaster to run locally on different InternalPorts. --- broadcast.go | 2 -- cluster.go | 3 ++- cmd/server.go | 1 + config.go | 20 ++++++-------------- handler_test.go | 2 +- httpbroadcast/messenger.go | 11 ++--------- server/server.go | 19 ++++++++++++------- 7 files changed, 24 insertions(+), 34 deletions(-) diff --git a/broadcast.go b/broadcast.go index 6eaf834cc..a3f3cb6c9 100644 --- a/broadcast.go +++ b/broadcast.go @@ -51,13 +51,11 @@ type nopBroadcaster struct{} // SendSync A no-op implemenetation of Broadcaster SendSync method. func (c *nopBroadcaster) SendSync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendSync") // TODO remove or log properly? return nil } // SendAsync A no-op implemenetation of Broadcaster SendAsync method. func (c *nopBroadcaster) SendAsync(pb proto.Message) error { - fmt.Println("NOPBroadcaster: SendAsync") // TODO remove or log properly? return nil } diff --git a/cluster.go b/cluster.go index d8f4401e3..6290f0050 100644 --- a/cluster.go +++ b/cluster.go @@ -19,7 +19,8 @@ const ( // Node represents a node in the cluster. type Node struct { - Host string `json:"host"` + Host string `json:"host"` + InternalHost string `json:"internal_host"` } // Nodes represents a list of nodes. diff --git a/cmd/server.go b/cmd/server.go index 51d92824a..d4e72a411 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -77,6 +77,7 @@ on the configured port.`, flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&Server.Config.Cluster.InternalNodes, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") diff --git a/config.go b/config.go index 22750d72d..b577f6281 100644 --- a/config.go +++ b/config.go @@ -4,10 +4,10 @@ import "time" const ( // DefaultHost is the default hostname and port to use. - DefaultHost = "localhost" - DefaultPort = "10101" - DefaultClusterType = "static" - DefaultGossipPort = "14000" + DefaultHost = "localhost" + DefaultPort = "10101" + DefaultClusterType = "static" + DefaultInternalPort = "14000" ) // Config represents the configuration for the command. @@ -19,6 +19,7 @@ type Config struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` Nodes []string `toml:"hosts"` + InternalNodes []string `toml:"internal-hosts"` PollingInterval Duration `toml:"polling-interval"` InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` @@ -44,20 +45,11 @@ func NewConfig() *Config { c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) c.Cluster.Nodes = []string{} + c.Cluster.InternalNodes = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) return c } -// NewConfigForHosts returns a Config object with Config.Cluster.Nodes already -// set up. -func NewConfigForHosts(hosts []string) *Config { - conf := NewConfig() - for _, hostport := range hosts { - conf.Cluster.Nodes = append(conf.Cluster.Nodes, hostport) - } - return conf -} - // Duration is a TOML wrapper type for time.Duration. type Duration time.Duration diff --git a/handler_test.go b/handler_test.go index 51f48fef7..2611da50e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -763,7 +763,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"host":"host1"},{"host":"host2"}]`+"\n" { + } else if w.Body.String() != `[{"host":"host1","internal_host":""},{"host":"host2","internal_host":""}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } diff --git a/httpbroadcast/messenger.go b/httpbroadcast/messenger.go index 1545e3328..ebd4b2e6a 100644 --- a/httpbroadcast/messenger.go +++ b/httpbroadcast/messenger.go @@ -11,8 +11,6 @@ import ( "golang.org/x/sync/errgroup" - "net" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" ) @@ -25,7 +23,7 @@ type HTTPBroadcaster struct { // NewHTTPBroadcaster returns a new instance of HTTPBroadcaster. func NewHTTPBroadcaster(s *pilosa.Server, internalPort string) *HTTPBroadcaster { - return &HTTPBroadcaster{server: s} + return &HTTPBroadcaster{server: s, internalPort: internalPort} } // SendSync sends a protobuf message to all nodes simultaneously. @@ -77,16 +75,11 @@ func (h *HTTPBroadcaster) sendNodeMessage(node *pilosa.Node, msg []byte) error { var client *http.Client client = http.DefaultClient - host, _, err := net.SplitHostPort(node.Host) - // Create HTTP request. req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", - Host: host + ":" + h.internalPort, + Host: node.InternalHost, }).String(), bytes.NewReader(msg)) - if err != nil { - return err - } // Require protobuf encoding. req.Header.Set("Content-Type", "application/x-protobuf") diff --git a/server/server.go b/server/server.go index 1bf3fe5ce..574941c42 100644 --- a/server/server.go +++ b/server/server.go @@ -96,6 +96,9 @@ func (m *Command) SetupServer() error { for _, hostport := range m.Config.Cluster.Nodes { cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) } + for i, internalhostport := range m.Config.Cluster.InternalNodes { + cluster.Nodes[i].InternalHost = internalhostport + } m.Server.Cluster = cluster // Setup logging output. @@ -120,21 +123,23 @@ func (m *Command) SetupServer() error { return err } + // Set internal port (string). + internalPortStr := pilosa.DefaultInternalPort + if m.Config.Cluster.InternalPort != "" { + internalPortStr = m.Config.Cluster.InternalPort + } + switch m.Config.Cluster.Type { case "http": - m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, m.Config.Cluster.InternalPort) - m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(m.Config.Cluster.InternalPort, m.Stderr) + m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, internalPortStr) + m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Stderr) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes) if err != nil { return err } case "gossip": - gossipPortStr := pilosa.DefaultGossipPort - if m.Config.Cluster.InternalPort != "" { - gossipPortStr = m.Config.Cluster.InternalPort - } - gossipPort, err := strconv.Atoi(gossipPortStr) + gossipPort, err := strconv.Atoi(internalPortStr) if err != nil { return err } From 4c8e46e17fbcfaf0483eae45a92787df6e572176 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 09:39:16 -0500 Subject: [PATCH 40/63] change Config.Nodes to Config.Hosts --- cmd/server.go | 4 ++-- cmd/server_test.go | 6 +++--- config.go | 8 ++++---- server/server.go | 7 +++++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d4e72a411..358793af3 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -76,8 +76,8 @@ on the configured port.`, flags.StringVarP(&Server.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") flags.StringVarP(&Server.Config.Host, "bind", "b", ":10101", "Default URI on which pilosa should listen.") flags.IntVarP(&Server.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") - flags.StringSliceVarP(&Server.Config.Cluster.Nodes, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.StringSliceVarP(&Server.Config.Cluster.InternalNodes, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") + flags.StringSliceVarP(&Server.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") + flags.StringSliceVarP(&Server.Config.Cluster.InternalHosts, "cluster.internal-hosts", "", []string{}, "Comma separated list of hosts in cluster used for internal communication.") flags.DurationVarP((*time.Duration)(&Server.Config.Cluster.PollingInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.StringVarP(&Server.Config.Plugins.Path, "plugins.path", "", "", "Path to plugin directory.") flags.StringVar(&Server.Config.LogPath, "log-path", "", "Log path") diff --git a/cmd/server_test.go b/cmd/server_test.go index f1310abc2..e51825e60 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -47,7 +47,7 @@ bind = "localhost:0" v.Check(cmd.Server.Config.DataDir, actualDataDir) v.Check(cmd.Server.Config.Host, "localhost:0") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"example.com:10101", "example.com:10110"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:10101", "example.com:10110"}) v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Second*182)) return v.Error() }, @@ -68,7 +68,7 @@ data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"example.com:1110", "example.com:1111"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"example.com:1110", "example.com:1111"}) v.Check(cmd.Server.Config.Plugins.Path, "/var/sloth") v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) return v.Error() @@ -94,7 +94,7 @@ data-dir = "` + actualDataDir + `" `, validation: func() error { v := validator{} - v.Check(cmd.Server.Config.Cluster.Nodes, []string{"localhost:19444"}) + v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) v.Check(cmd.Server.Config.Cluster.PollingInterval, pilosa.Duration(time.Minute*2)) v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) v.Check(cmd.Server.CPUProfile, profFile.Name()) diff --git a/config.go b/config.go index b577f6281..d5d725ece 100644 --- a/config.go +++ b/config.go @@ -18,8 +18,8 @@ type Config struct { Cluster struct { ReplicaN int `toml:"replicas"` Type string `toml:"type"` - Nodes []string `toml:"hosts"` - InternalNodes []string `toml:"internal-hosts"` + Hosts []string `toml:"hosts"` + InternalHosts []string `toml:"internal-hosts"` PollingInterval Duration `toml:"polling-interval"` InternalPort string `toml:"internal-port"` GossipSeed string `toml:"gossip-seed"` @@ -44,8 +44,8 @@ func NewConfig() *Config { c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Type = DefaultClusterType c.Cluster.PollingInterval = Duration(DefaultPollingInterval) - c.Cluster.Nodes = []string{} - c.Cluster.InternalNodes = []string{} + c.Cluster.Hosts = []string{} + c.Cluster.InternalHosts = []string{} c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) return c } diff --git a/server/server.go b/server/server.go index 574941c42..ec2b55c7c 100644 --- a/server/server.go +++ b/server/server.go @@ -93,10 +93,13 @@ func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN - for _, hostport := range m.Config.Cluster.Nodes { + for _, hostport := range m.Config.Cluster.Hosts { cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: hostport}) } - for i, internalhostport := range m.Config.Cluster.InternalNodes { + // TODO: if InternalHosts is not provided then pilosa.Node.InternalHost is empty. + // This will throw an error when trying to Broadcast messages over HTTP. + // One option may be to fall back to using host from hostport + config.InternalPort. + for i, internalhostport := range m.Config.Cluster.InternalHosts { cluster.Nodes[i].InternalHost = internalhostport } m.Server.Cluster = cluster From 26b524a60e6bcd35f3fc9853211728ea18d6e8fb Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 11:35:02 -0500 Subject: [PATCH 41/63] fix some tests and update glide.lock --- fragment_test.go | 33 +- glide.lock | 4 +- glide.yaml | 2 - handler.go | 12 +- internal/private.pb.go | 1744 +++++++++++++++++++++++++++++++++++++--- internal/private.proto | 55 +- 6 files changed, 1726 insertions(+), 124 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index e54b11f82..e03d5111c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -280,15 +280,34 @@ func TestFragment_TopN_BitmapIDs(t *testing.T) { // Ensure the fragment cache limit works func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) - cacheLimit := uint32(3) - file, err := ioutil.TempFile("", "pilosa-fragment-") + cacheSize := uint32(3) + + // Create DB. + db := MustOpenDB() + defer db.Close() + + // Create frame. + frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { - panic(err) + t.Fatal(err) } - file.Close() + + // Create view. + view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + if err != nil { + t.Fatal(err) + } + + // Create fragment. + frag, err := view.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } + // Close the storage so we can re-open it without encountering a flock. + frag.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), "d", "f.n", pilosa.ViewStandard, slice, cacheLimit), + Fragment: frag, BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -316,8 +335,8 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Retrieve top bitmaps. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) - } else if len(pairs) > int(cacheLimit) { - t.Fatalf("TopN count cannot exceed cache size: %d", cacheLimit) + } else if len(pairs) > int(cacheSize) { + t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { diff --git a/glide.lock b/glide.lock index ae504264c..c142737ff 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 743e8f978eb4ad8f80a2ab71b05caebbf50b6769b71aa457bc4f144fef8c6595 -updated: 2017-04-18T15:33:39.035615802-05:00 +hash: 4bdea17c62dcd469584382515052e7a246fae6deefc2d62da70bf768e18e1f0c +updated: 2017-04-19T10:51:10.409094081-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 diff --git a/glide.yaml b/glide.yaml index d81cbaf28..a7c0e98eb 100644 --- a/glide.yaml +++ b/glide.yaml @@ -31,7 +31,5 @@ import: - package: github.com/spf13/viper - package: github.com/gorilla/mux version: ^1.3.0 -- package: github.com/aws/aws-sdk-go - version: ^1.6.10 - package: github.com/hashicorp/memberlist - package: golang.org/x/sync diff --git a/handler.go b/handler.go index c8dda53c2..c9140867b 100644 --- a/handler.go +++ b/handler.go @@ -339,9 +339,9 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Send the delete message to all nodes. - err := h.Broadcaster.SendSync( + err = h.Broadcaster.SendSync( &internal.DeleteDBMessage{ - DB: req.DB, + DB: dbName, }) if err != nil { h.logger().Printf("problem sending DeleteDB message: %s", err) @@ -485,8 +485,8 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { // Send the create message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ - DB: req.DB, - Frame: req.Frame, + DB: dbName, + Frame: frameName, Meta: &internal.FrameMeta{ RowLabel: req.Options.RowLabel, TimeQuantum: string(req.Options.TimeQuantum), @@ -557,8 +557,8 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { // Send the delete message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ - DB: req.DB, - Frame: req.Frame, + DB: dbName, + Frame: frameName, }) if err != nil { h.logger().Printf("problem sending DeleteFrame message: %s", err) diff --git a/internal/private.pb.go b/internal/private.pb.go index 1ffa98885..5ceb800c2 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -9,13 +9,21 @@ private.proto It has these top-level messages: - DB - Frame + DBMeta + FrameMeta ImportResponse BlockDataRequest BlockDataResponse Cache MaxSlicesResponse + CreateSliceMessage + DeleteDBMessage + CreateDBMessage + CreateFrameMessage + DeleteFrameMessage + Frame + DB + NodeState */ package internal @@ -36,27 +44,28 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type DB struct { +type DBMeta struct { TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` } -func (m *DB) Reset() { *m = DB{} } -func (m *DB) String() string { return proto.CompactTextString(m) } -func (*DB) ProtoMessage() {} -func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *DBMeta) Reset() { *m = DBMeta{} } +func (m *DBMeta) String() string { return proto.CompactTextString(m) } +func (*DBMeta) ProtoMessage() {} +func (*DBMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -type Frame struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - RowLabel string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` - InverseEnabled bool `protobuf:"varint,3,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` - CacheType string `protobuf:"bytes,4,opt,name=CacheType,proto3" json:"CacheType,omitempty"` +type FrameMeta struct { + RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` + InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } -func (m *Frame) Reset() { *m = Frame{} } -func (m *Frame) String() string { return proto.CompactTextString(m) } -func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +func (m *FrameMeta) Reset() { *m = FrameMeta{} } +func (m *FrameMeta) String() string { return proto.CompactTextString(m) } +func (*FrameMeta) ProtoMessage() {} +func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` @@ -115,16 +124,149 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { return nil } +type CreateSliceMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` +} + +func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } +func (m *CreateSliceMessage) String() string { return proto.CompactTextString(m) } +func (*CreateSliceMessage) ProtoMessage() {} +func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } + +type DeleteDBMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` +} + +func (m *DeleteDBMessage) Reset() { *m = DeleteDBMessage{} } +func (m *DeleteDBMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteDBMessage) ProtoMessage() {} +func (*DeleteDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } + +type CreateDBMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *CreateDBMessage) Reset() { *m = CreateDBMessage{} } +func (m *CreateDBMessage) String() string { return proto.CompactTextString(m) } +func (*CreateDBMessage) ProtoMessage() {} +func (*CreateDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } + +func (m *CreateDBMessage) GetMeta() *DBMeta { + if m != nil { + return m.Meta + } + return nil +} + +type CreateFrameMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } +func (m *CreateFrameMessage) String() string { return proto.CompactTextString(m) } +func (*CreateFrameMessage) ProtoMessage() {} +func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } + +func (m *CreateFrameMessage) GetMeta() *FrameMeta { + if m != nil { + return m.Meta + } + return nil +} + +type DeleteFrameMessage struct { + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` +} + +func (m *DeleteFrameMessage) Reset() { *m = DeleteFrameMessage{} } +func (m *DeleteFrameMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteFrameMessage) ProtoMessage() {} +func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } + +type Frame struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +} + +func (m *Frame) Reset() { *m = Frame{} } +func (m *Frame) String() string { return proto.CompactTextString(m) } +func (*Frame) ProtoMessage() {} +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } + +func (m *Frame) GetMeta() *FrameMeta { + if m != nil { + return m.Meta + } + return nil +} + +type DB struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` +} + +func (m *DB) Reset() { *m = DB{} } +func (m *DB) String() string { return proto.CompactTextString(m) } +func (*DB) ProtoMessage() {} +func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } + +func (m *DB) GetMeta() *DBMeta { + if m != nil { + return m.Meta + } + return nil +} + +func (m *DB) GetFrames() []*Frame { + if m != nil { + return m.Frames + } + return nil +} + +type NodeState struct { + Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + DBs []*DB `protobuf:"bytes,3,rep,name=DBs" json:"DBs,omitempty"` +} + +func (m *NodeState) Reset() { *m = NodeState{} } +func (m *NodeState) String() string { return proto.CompactTextString(m) } +func (*NodeState) ProtoMessage() {} +func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } + +func (m *NodeState) GetDBs() []*DB { + if m != nil { + return m.DBs + } + return nil +} + func init() { - proto.RegisterType((*DB)(nil), "internal.DB") - proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*DBMeta)(nil), "internal.DBMeta") + proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") + proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") + proto.RegisterType((*DeleteDBMessage)(nil), "internal.DeleteDBMessage") + proto.RegisterType((*CreateDBMessage)(nil), "internal.CreateDBMessage") + proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") + proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") + proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*NodeState)(nil), "internal.NodeState") } -func (m *DB) Marshal() (dAtA []byte, err error) { +func (m *DBMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -134,7 +276,7 @@ func (m *DB) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DB) MarshalTo(dAtA []byte) (int, error) { +func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -154,7 +296,7 @@ func (m *DB) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Frame) Marshal() (dAtA []byte, err error) { +func (m *FrameMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -164,25 +306,19 @@ func (m *Frame) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Frame) MarshalTo(dAtA []byte) (int, error) { +func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } if len(m.RowLabel) > 0 { - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.RowLabel))) i += copy(dAtA[i:], m.RowLabel) } if m.InverseEnabled { - dAtA[i] = 0x18 + dAtA[i] = 0x10 i++ if m.InverseEnabled { dAtA[i] = 1 @@ -192,11 +328,22 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { i++ } if len(m.CacheType) > 0 { - dAtA[i] = 0x22 + dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.CacheType))) i += copy(dAtA[i:], m.CacheType) } + if m.CacheSize != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.CacheSize)) + } + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0x2a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } return i, nil } @@ -391,6 +538,290 @@ func (m *MaxSlicesResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CreateSliceMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if m.Slice != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) + } + return i, nil +} + +func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteDBMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + return i, nil +} + +func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateDBMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n7, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + return i, nil +} + +func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Meta != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n8, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n8 + } + return i, nil +} + +func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + return i, nil +} + +func (m *Frame) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frame) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n9, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n9 + } + return i, nil +} + +func (m *DB) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DB) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if m.Meta != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n10, err := m.Meta.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n10 + } + if m.MaxSlice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlice)) + } + if len(m.Frames) > 0 { + for _, msg := range m.Frames { + dAtA[i] = 0x22 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *NodeState) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NodeState) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Host) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Host))) + i += copy(dAtA[i:], m.Host) + } + if len(m.State) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) + i += copy(dAtA[i:], m.State) + } + if len(m.DBs) > 0 { + for _, msg := range m.DBs { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -418,7 +849,7 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *DB) Size() (n int) { +func (m *DBMeta) Size() (n int) { var l int _ = l l = len(m.TimeQuantum) @@ -432,13 +863,9 @@ func (m *DB) Size() (n int) { return n } -func (m *Frame) Size() (n int) { +func (m *FrameMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.RowLabel) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -450,6 +877,13 @@ func (m *Frame) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } + if m.CacheSize != 0 { + n += 1 + sovPrivate(uint64(m.CacheSize)) + } + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } return n } @@ -534,6 +968,132 @@ func (m *MaxSlicesResponse) Size() (n int) { return n } +func (m *CreateSliceMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovPrivate(uint64(m.Slice)) + } + return n +} + +func (m *DeleteDBMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *CreateDBMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *CreateFrameMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DeleteFrameMessage) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *Frame) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + +func (m *DB) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Meta != nil { + l = m.Meta.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + if m.MaxSlice != 0 { + n += 1 + sovPrivate(uint64(m.MaxSlice)) + } + if len(m.Frames) > 0 { + for _, e := range m.Frames { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + +func (m *NodeState) Size() (n int) { + var l int + _ = l + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.State) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.DBs) > 0 { + for _, e := range m.DBs { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -547,7 +1107,7 @@ func sovPrivate(x uint64) (n int) { func sozPrivate(x uint64) (n int) { return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *DB) Unmarshal(dAtA []byte) error { +func (m *DBMeta) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -570,10 +1130,10 @@ func (m *DB) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DB: wiretype end group for non-group") + return fmt.Errorf("proto: DBMeta: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: DBMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -655,7 +1215,7 @@ func (m *DB) Unmarshal(dAtA []byte) error { } return nil } -func (m *Frame) Unmarshal(dAtA []byte) error { +func (m *FrameMeta) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -678,42 +1238,13 @@ func (m *Frame) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Frame: wiretype end group for non-group") + return fmt.Errorf("proto: FrameMeta: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) } @@ -742,7 +1273,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } m.RowLabel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) } @@ -762,7 +1293,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } } m.InverseEnabled = bool(v != 0) - case 4: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType) } @@ -791,6 +1322,54 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } m.CacheType = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CacheSize", wireType) + } + m.CacheSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CacheSize |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -1508,6 +2087,957 @@ func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateSliceMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateSliceMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteDBMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateDBMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &DBMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CreateFrameMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CreateFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &FrameMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteFrameMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frame) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Frame: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &FrameMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DB) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DB: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Meta == nil { + m.Meta = &DBMeta{} + } + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxSlice", wireType) + } + m.MaxSlice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxSlice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frames = append(m.Frames, &Frame{}) + if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NodeState) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NodeState: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NodeState: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DBs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DBs = append(m.DBs, &DB{}) + if err := m.DBs[len(m.DBs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -1616,31 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 406 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x92, 0xd1, 0x8a, 0xd3, 0x40, - 0x14, 0x86, 0x9d, 0x36, 0x95, 0xf6, 0x2c, 0x86, 0xee, 0xb0, 0x17, 0x61, 0x59, 0x42, 0x18, 0x50, - 0x8a, 0x17, 0xbd, 0xd0, 0x1b, 0x11, 0xaf, 0xb2, 0xad, 0x6c, 0x41, 0xc1, 0x1d, 0x17, 0xef, 0xa7, - 0xf5, 0x88, 0xa1, 0x93, 0x99, 0x38, 0x99, 0x74, 0x37, 0xcf, 0xe0, 0x0b, 0x08, 0xbe, 0x90, 0x97, - 0x3e, 0x82, 0xd4, 0x17, 0x91, 0x99, 0xa4, 0x4d, 0x54, 0x10, 0xef, 0xe6, 0x7c, 0x73, 0xce, 0x3f, - 0xff, 0xf9, 0x19, 0x78, 0x50, 0x98, 0x6c, 0x27, 0x2c, 0xce, 0x0b, 0xa3, 0xad, 0xa6, 0xe3, 0x4c, - 0x59, 0x34, 0x4a, 0x48, 0x76, 0x05, 0x83, 0x45, 0x4a, 0x13, 0x38, 0xb9, 0xc9, 0x72, 0xbc, 0xae, - 0x84, 0xb2, 0x55, 0x1e, 0x91, 0x84, 0xcc, 0x26, 0xbc, 0x8f, 0x5c, 0xc7, 0xa5, 0x96, 0x55, 0xae, - 0x5e, 0x89, 0x35, 0xca, 0x68, 0xd0, 0x74, 0xf4, 0x10, 0xfb, 0x4c, 0x60, 0xf4, 0xd2, 0x88, 0x1c, - 0xff, 0x43, 0xed, 0x1c, 0xc6, 0x5c, 0xdf, 0xf6, 0xa5, 0x8e, 0x35, 0x7d, 0x04, 0xe1, 0x4a, 0xed, - 0xd0, 0x94, 0xb8, 0x54, 0x62, 0x2d, 0xf1, 0x7d, 0x34, 0x4c, 0xc8, 0x6c, 0xcc, 0xff, 0xa0, 0xf4, - 0x02, 0x26, 0x97, 0x62, 0xf3, 0x11, 0x6f, 0xea, 0x02, 0xa3, 0xc0, 0x8b, 0x74, 0x80, 0x31, 0x08, - 0x57, 0x79, 0xa1, 0x8d, 0xe5, 0x58, 0x16, 0x5a, 0x95, 0x48, 0xa7, 0x30, 0x5c, 0x1a, 0xd3, 0xba, - 0x71, 0x47, 0x76, 0x07, 0xd3, 0x54, 0xea, 0xcd, 0x76, 0x21, 0xac, 0xe0, 0xf8, 0xa9, 0xc2, 0xd2, - 0xd2, 0xd0, 0xe5, 0xd1, 0x36, 0xb9, 0x64, 0xce, 0xda, 0xa5, 0x5a, 0x9b, 0xed, 0x86, 0x67, 0x30, - 0xf2, 0x93, 0xde, 0x5a, 0xc0, 0x9b, 0xc2, 0xd1, 0xb7, 0x32, 0xdb, 0x34, 0x6e, 0x02, 0xde, 0x14, - 0x94, 0x42, 0xf0, 0x2e, 0xc3, 0xdb, 0x68, 0xe4, 0x05, 0xfc, 0x99, 0x5d, 0xc3, 0x69, 0xef, 0xe5, - 0xd6, 0xe0, 0x05, 0x4c, 0xd2, 0xcc, 0xe6, 0xa2, 0x58, 0x2d, 0xca, 0x88, 0x24, 0xc3, 0x59, 0xc0, - 0x3b, 0x40, 0x63, 0x80, 0x37, 0x46, 0x7f, 0xc8, 0x24, 0xba, 0xeb, 0x81, 0xbf, 0xee, 0x11, 0xf6, - 0x10, 0x46, 0x7e, 0xfb, 0x7f, 0xcb, 0xb0, 0xaf, 0x04, 0x4e, 0x5f, 0x8b, 0x3b, 0x6f, 0xad, 0x3c, - 0x3e, 0x7d, 0x05, 0x93, 0x23, 0xf4, 0x33, 0x27, 0x4f, 0x1e, 0xcf, 0x0f, 0x7f, 0x64, 0xfe, 0x57, - 0x7f, 0x47, 0x96, 0xca, 0x9a, 0x9a, 0x77, 0xc3, 0xe7, 0x2f, 0x20, 0xfc, 0xfd, 0xd2, 0xe5, 0xbe, - 0xc5, 0xfa, 0x90, 0xfb, 0x16, 0x6b, 0x97, 0xd3, 0x4e, 0xc8, 0xaa, 0xc9, 0x34, 0xe0, 0x4d, 0xf1, - 0x7c, 0xf0, 0x8c, 0xa4, 0xd3, 0x6f, 0xfb, 0x98, 0x7c, 0xdf, 0xc7, 0xe4, 0xc7, 0x3e, 0x26, 0x5f, - 0x7e, 0xc6, 0xf7, 0xd6, 0xf7, 0xfd, 0x87, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x99, 0x45, - 0xbe, 0x6d, 0xc1, 0x02, 0x00, 0x00, + // 603 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, + 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, + 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, + 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, + 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xfa, 0xab, + 0xaf, 0xbe, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, + 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, + 0xbd, 0x08, 0x63, 0x3c, 0x1f, 0xf3, 0x44, 0x8c, 0x63, 0xd7, 0xe8, 0x19, 0x3b, 0x0e, 0x6b, 0x43, + 0xb2, 0xe2, 0x20, 0x8d, 0xc6, 0x71, 0x72, 0xc2, 0x07, 0x18, 0xb9, 0xa6, 0xae, 0x68, 0x41, 0xf4, + 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0xba, 0x58, 0xd3, + 0xd5, 0x31, 0x79, 0x0c, 0x9d, 0xe3, 0x64, 0x82, 0x79, 0x81, 0x87, 0x09, 0x1f, 0x44, 0x78, 0xa9, + 0xe8, 0x56, 0xd8, 0x1c, 0x4a, 0xb6, 0xc0, 0x39, 0xe0, 0xc3, 0xf7, 0x78, 0x31, 0xcd, 0xd0, 0xb5, + 0x14, 0x49, 0x03, 0xd4, 0xd9, 0x7e, 0xf8, 0x11, 0x5d, 0xbb, 0x67, 0xec, 0xac, 0xb1, 0x06, 0x98, + 0x9f, 0x68, 0xe9, 0xc6, 0x44, 0x94, 0x42, 0xe7, 0x38, 0xce, 0xd2, 0x5c, 0x30, 0x2c, 0xb2, 0x34, + 0x29, 0x90, 0x74, 0xc1, 0x3a, 0xcc, 0xf3, 0x52, 0xae, 0x3c, 0xd2, 0x6b, 0xe8, 0xfa, 0x51, 0x3a, + 0x1c, 0x05, 0x5c, 0x70, 0x86, 0x1f, 0xc6, 0x58, 0x08, 0xd2, 0x01, 0x33, 0xf0, 0xcb, 0x22, 0x33, + 0xf0, 0xc9, 0x7d, 0x58, 0x52, 0x63, 0x97, 0x9e, 0xe8, 0x40, 0xa2, 0xea, 0xa6, 0xd2, 0x6d, 0x33, + 0x1d, 0x48, 0xb4, 0x1f, 0x85, 0x43, 0xad, 0xd7, 0x66, 0x3a, 0x20, 0x04, 0xec, 0x37, 0x21, 0x5e, + 0x95, 0x22, 0xd5, 0x99, 0x9e, 0xc3, 0x46, 0xab, 0x73, 0x29, 0x70, 0x0b, 0x1c, 0x3f, 0x14, 0x31, + 0xcf, 0x8e, 0x83, 0xc2, 0x35, 0x7a, 0xd6, 0x8e, 0xcd, 0x1a, 0x80, 0x78, 0x00, 0xaf, 0xf2, 0xf4, + 0x6d, 0x18, 0xa1, 0x4c, 0x9b, 0x2a, 0xdd, 0x42, 0xe8, 0x23, 0x58, 0x52, 0xfe, 0xfc, 0x99, 0x86, + 0x7e, 0x31, 0x60, 0xe3, 0x94, 0x5f, 0x2b, 0x69, 0x45, 0xdd, 0xfa, 0x08, 0x9c, 0x1a, 0x54, 0x77, + 0x56, 0xf7, 0x9e, 0xec, 0x56, 0x9b, 0xb4, 0x7b, 0xa3, 0xbe, 0x41, 0x0e, 0x13, 0x91, 0x4f, 0x59, + 0x73, 0x79, 0xf3, 0x39, 0x74, 0x7e, 0x4f, 0x4a, 0xdf, 0x47, 0x38, 0xad, 0x7c, 0x1f, 0xe1, 0x54, + 0xfa, 0x34, 0xe1, 0xd1, 0x58, 0x7b, 0x6a, 0x33, 0x1d, 0xec, 0x9b, 0xcf, 0x0c, 0xba, 0x0f, 0xe4, + 0x20, 0x47, 0x2e, 0x50, 0x11, 0x9c, 0x62, 0x51, 0xf0, 0x77, 0xb8, 0xe8, 0x9b, 0x68, 0x9f, 0xcd, + 0x96, 0xcf, 0xf4, 0x01, 0xac, 0x07, 0x18, 0xa1, 0x40, 0xb9, 0xf5, 0x0b, 0x2f, 0xd2, 0x97, 0xb0, + 0xae, 0xe9, 0x6f, 0x2d, 0x21, 0x0f, 0xc1, 0x96, 0x1b, 0xae, 0xa8, 0x57, 0xf7, 0xba, 0x8d, 0x09, + 0xfa, 0x2d, 0x31, 0x95, 0xa5, 0xc3, 0x4a, 0x67, 0xf9, 0x24, 0x6e, 0xd5, 0xb9, 0x60, 0x77, 0xb6, + 0xcb, 0x0e, 0x96, 0xea, 0x70, 0xaf, 0xe9, 0x50, 0x3f, 0xaf, 0xb2, 0xc9, 0x3e, 0x10, 0x3d, 0xd0, + 0xff, 0x37, 0xa1, 0x41, 0x89, 0xca, 0xed, 0x3b, 0x93, 0x59, 0x7d, 0x41, 0x9d, 0x6b, 0x05, 0xe6, + 0xdf, 0x14, 0x7c, 0x32, 0x64, 0xb3, 0x85, 0x1c, 0xff, 0xe4, 0x93, 0xfc, 0x4f, 0x54, 0xdb, 0x50, + 0x3e, 0x95, 0x3a, 0x26, 0xdb, 0xb0, 0xac, 0xfa, 0x15, 0xae, 0xad, 0x16, 0x6e, 0x7d, 0x4e, 0x07, + 0x2b, 0xd3, 0xf4, 0x35, 0x38, 0x67, 0xe9, 0x25, 0xf6, 0x05, 0x17, 0x6a, 0x9e, 0xa3, 0xb4, 0x10, + 0x95, 0x16, 0x79, 0x56, 0xfb, 0x20, 0x93, 0x95, 0x05, 0xba, 0xd2, 0x03, 0x2b, 0xf0, 0x0b, 0xd7, + 0x52, 0xe4, 0x77, 0xdb, 0x02, 0x99, 0x4c, 0xf8, 0xdd, 0x6f, 0x33, 0xcf, 0xf8, 0x3e, 0xf3, 0x8c, + 0x1f, 0x33, 0xcf, 0xf8, 0xfc, 0xd3, 0xbb, 0x33, 0x58, 0x56, 0xbf, 0xd0, 0xa7, 0xbf, 0x02, 0x00, + 0x00, 0xff, 0xff, 0xc3, 0x9a, 0x29, 0xff, 0x53, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index acd9ec43c..40e83eb6b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -2,16 +2,17 @@ syntax = "proto3"; package internal; -message DB { +message DBMeta { string TimeQuantum = 1; string ColumnLabel = 2; } -message Frame { - string TimeQuantum = 1; - string RowLabel = 2; - bool InverseEnabled = 3; - string CacheType = 4; +message FrameMeta { + string RowLabel = 1; + bool InverseEnabled = 2; + string CacheType = 3; + uint32 CacheSize = 4; + string TimeQuantum = 5; } message ImportResponse { @@ -39,3 +40,45 @@ message MaxSlicesResponse { map MaxSlices = 1; } +message CreateSliceMessage { + string DB = 1; + uint64 Slice = 2; +} + +message DeleteDBMessage { + string DB = 1; +} + +message CreateDBMessage { + string DB = 1; + DBMeta Meta = 2; +} + +message CreateFrameMessage { + string DB = 1; + string Frame = 2; + FrameMeta Meta = 3; +} + +message DeleteFrameMessage { + string DB = 1; + string Frame = 2; +} + +message Frame { + string Name = 1; + FrameMeta Meta = 2; +} + +message DB { + string Name = 1; + DBMeta Meta = 2; + uint64 MaxSlice = 3; + repeated Frame Frames = 4; +} + +message NodeState { + string Host = 1; + string State = 2; + repeated DB DBs = 3; +} From 25e649add8c7d4e10608e0e55f4da00bcbd02617 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 11:41:41 -0500 Subject: [PATCH 42/63] remove `cacheSize` argument from `NewFragment()` --- fragment.go | 4 ++-- fragment_test.go | 6 +++--- view.go | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fragment.go b/fragment.go index 5704e774d..9bee4288e 100644 --- a/fragment.go +++ b/fragment.go @@ -94,7 +94,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) *Fragment { +func NewFragment(path, db, frame, view string, slice uint64) *Fragment { return &Fragment{ path: path, db: db, @@ -102,7 +102,7 @@ func NewFragment(path, db, frame, view string, slice uint64, cacheSize uint32) * view: view, slice: slice, cacheType: DefaultCacheType, - cacheSize: cacheSize, + cacheSize: DefaultCacheSize, LogOutput: ioutil.Discard, MaxOpN: DefaultFragmentMaxOpN, diff --git a/fragment_test.go b/fragment_test.go index e03d5111c..f54f5415b 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -593,7 +593,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0, pilosa.DefaultCacheSize) + f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -654,7 +654,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice, pilosa.DefaultCacheSize), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), BitmapAttrStore: MustOpenAttrStore(), } f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore @@ -685,7 +685,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice(), pilosa.DefaultCacheSize) + f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore if err := f.Open(); err != nil { return err diff --git a/view.go b/view.go index 9ded867ce..c7ec92733 100644 --- a/view.go +++ b/view.go @@ -216,8 +216,9 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice, v.cacheSize) + frag := NewFragment(path, v.db, v.frame, v.name, slice) frag.cacheType = v.cacheType + frag.cacheSize = v.cacheSize frag.LogOutput = v.LogOutput frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice)) return frag From dcaf7617de1ac5d1df4870123af9d7ab1f24e3f8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 13:36:58 -0500 Subject: [PATCH 43/63] changed some json tag names to Lower Camel Case to be consistent with the rest --- cluster.go | 2 +- ctl/import.go | 2 +- handler.go | 6 +++--- handler_test.go | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cluster.go b/cluster.go index 6290f0050..f92b8c6e9 100644 --- a/cluster.go +++ b/cluster.go @@ -20,7 +20,7 @@ const ( // Node represents a node in the cluster. type Node struct { Host string `json:"host"` - InternalHost string `json:"internal_host"` + InternalHost string `json:"internalHost"` } // Nodes represents a list of nodes. diff --git a/ctl/import.go b/ctl/import.go index ad3281813..df8fb00eb 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -27,7 +27,7 @@ type ImportCommand struct { Paths []string `json:"paths"` // Size of buffer used to chunk import. - BufferSize int `json:"buffer-size"` + BufferSize int `json:"bufferSize"` // Reusable client. Client *pilosa.Client `json:"-"` diff --git a/handler.go b/handler.go index c9140867b..a6309e44a 100644 --- a/handler.go +++ b/handler.go @@ -214,7 +214,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { } type sliceMaxResponse struct { - MaxSlices map[string]uint64 `json:"MaxSlices"` + MaxSlices map[string]uint64 `json:"maxSlices"` } // handleGetDBs handles GET /db request. @@ -391,7 +391,7 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques } type patchDBTimeQuantumRequest struct { - TimeQuantum string `json:"time_quantum"` + TimeQuantum string `json:"timeQuantum"` } type patchDBTimeQuantumResponse struct{} @@ -611,7 +611,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req } type patchFrameTimeQuantumRequest struct { - TimeQuantum string `json:"time_quantum"` + TimeQuantum string `json:"timeQuantum"` } type patchFrameTimeQuantumResponse struct{} diff --git a/handler_test.go b/handler_test.go index 2611da50e..478720e23 100644 --- a/handler_test.go +++ b/handler_test.go @@ -83,7 +83,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -123,7 +123,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -556,7 +556,7 @@ func TestHandler_SetDBTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"time_quantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -579,7 +579,7 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { h := NewHandler() h.Index = idx.Index w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"time_quantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { @@ -763,7 +763,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if w.Body.String() != `[{"host":"host1","internal_host":""},{"host":"host2","internal_host":""}]`+"\n" { + } else if w.Body.String() != `[{"host":"host1","internalHost":""},{"host":"host2","internalHost":""}]`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) } } From caf35bb4d9461b9e4ea20f26b7af039f6398d2b6 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 16:07:30 -0500 Subject: [PATCH 44/63] rename messenger_test to align with broadcast --- messenger_test.go => broadcast_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename messenger_test.go => broadcast_test.go (100%) diff --git a/messenger_test.go b/broadcast_test.go similarity index 100% rename from messenger_test.go rename to broadcast_test.go From 5a806850527c802494cd5db4b44970be44d19e5d Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 16:59:56 -0500 Subject: [PATCH 45/63] add support for CreateDB and CreateFrame without POST data (i.e. use defaults) --- handler.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/handler.go b/handler.go index a6309e44a..b94a9a0ed 100644 --- a/handler.go +++ b/handler.go @@ -309,6 +309,15 @@ func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { return } + // Send the delete message to all nodes. + err := h.Broadcaster.SendSync( + &internal.DeleteDBMessage{ + DB: dbName, + }) + if err != nil { + h.logger().Printf("problem sending DeleteDB message: %s", err) + } + // Encode response. if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -323,13 +332,17 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Decode request. var req postDBRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the database + // with default values. + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // Create database. - _, err := h.Index.CreateDB(dbName, req.Options) + _, err = h.Index.CreateDB(dbName, req.Options) if err == ErrDatabaseExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -340,11 +353,15 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Send the delete message to all nodes. err = h.Broadcaster.SendSync( - &internal.DeleteDBMessage{ + &internal.CreateDBMessage{ DB: dbName, + Meta: &internal.DBMeta{ + ColumnLabel: req.Options.ColumnLabel, + TimeQuantum: string(req.Options.TimeQuantum), + }, }) if err != nil { - h.logger().Printf("problem sending DeleteDB message: %s", err) + h.logger().Printf("problem sending CreateDB message: %s", err) } // Encode response. @@ -460,7 +477,11 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { // Decode request. var req postFrameRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the frame + // with default values. + } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -473,7 +494,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Create frame. - _, err := db.CreateFrame(frameName, req.Options) + _, err = db.CreateFrame(frameName, req.Options) if err == ErrFrameExists { http.Error(w, err.Error(), http.StatusConflict) return From d2e68b298050187329d3a9e06204f50c47dc64c8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 19 Apr 2017 17:08:10 -0500 Subject: [PATCH 46/63] adjust some handler comments. remove debugging line --- handler.go | 8 ++++---- server.go | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/handler.go b/handler.go index b94a9a0ed..d9c79eeba 100644 --- a/handler.go +++ b/handler.go @@ -309,7 +309,7 @@ func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the delete database message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteDBMessage{ DB: dbName, @@ -351,7 +351,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the create database message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateDBMessage{ DB: dbName, @@ -503,7 +503,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } - // Send the create message to all nodes. + // Send the create frame message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ DB: dbName, @@ -575,7 +575,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { return } - // Send the delete message to all nodes. + // Send the delete frame message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ DB: dbName, diff --git a/server.go b/server.go index 229214805..e93b8e111 100644 --- a/server.go +++ b/server.go @@ -265,7 +265,6 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return err } case *internal.DeleteDBMessage: - fmt.Println("DELETE:", obj.DB) if err := s.Index.DeleteDB(obj.DB); err != nil { return err } From 95bd4913c840882af22499f0eaf21499b607e372 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 10:41:17 -0500 Subject: [PATCH 47/63] adjusted DBOptions/FrameOptions logic and removed some unused tests --- db.go | 8 +++ frame.go | 23 +++++-- handler.go | 12 +--- handler_test.go | 54 --------------- index_test.go | 28 -------- internal/private.pb.go | 148 ++++++++++++++++++++--------------------- internal/private.proto | 4 +- 7 files changed, 105 insertions(+), 172 deletions(-) diff --git a/db.go b/db.go index d2ceaf13a..8bf33300e 100644 --- a/db.go +++ b/db.go @@ -536,6 +536,14 @@ type DBOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } +// Encode converts o into its internal representation. +func (o *DBOptions) Encode() *internal.DBMeta { + return &internal.DBMeta{ + ColumnLabel: o.ColumnLabel, + TimeQuantum: string(o.TimeQuantum), + } +} + // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/frame.go b/frame.go index a497e6906..30fd41406 100644 --- a/frame.go +++ b/frame.go @@ -302,11 +302,11 @@ func (f *Frame) loadMeta() error { func (f *Frame) saveMeta() error { // Marshal metadata. buf, err := proto.Marshal(&internal.FrameMeta{ - TimeQuantum: string(f.timeQuantum), RowLabel: f.rowLabel, - CacheType: f.cacheType, InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, CacheSize: f.cacheSize, + TimeQuantum: string(f.timeQuantum), }) if err != nil { return err @@ -593,9 +593,11 @@ func encodeFrame(f *Frame) *internal.Frame { return &internal.Frame{ Name: f.name, Meta: &internal.FrameMeta{ - TimeQuantum: string(f.timeQuantum), - RowLabel: f.rowLabel, - CacheSize: f.cacheSize, + RowLabel: f.rowLabel, + InverseEnabled: f.inverseEnabled, + CacheType: f.cacheType, + CacheSize: f.cacheSize, + TimeQuantum: string(f.timeQuantum), }, } } @@ -627,6 +629,17 @@ type FrameOptions struct { TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` } +// Encode converts o into its internal representation. +func (o *FrameOptions) Encode() *internal.FrameMeta { + return &internal.FrameMeta{ + RowLabel: o.RowLabel, + InverseEnabled: o.InverseEnabled, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + TimeQuantum: string(o.TimeQuantum), + } +} + // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { diff --git a/handler.go b/handler.go index d9c79eeba..8b79267bf 100644 --- a/handler.go +++ b/handler.go @@ -354,11 +354,8 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { // Send the create database message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateDBMessage{ - DB: dbName, - Meta: &internal.DBMeta{ - ColumnLabel: req.Options.ColumnLabel, - TimeQuantum: string(req.Options.TimeQuantum), - }, + DB: dbName, + Meta: req.Options.Encode(), }) if err != nil { h.logger().Printf("problem sending CreateDB message: %s", err) @@ -508,10 +505,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { &internal.CreateFrameMessage{ DB: dbName, Frame: frameName, - Meta: &internal.FrameMeta{ - RowLabel: req.Options.RowLabel, - TimeQuantum: string(req.Options.TimeQuantum), - }, + Meta: req.Options.Encode(), }) if err != nil { h.logger().Printf("problem sending CreateFrame message: %s", err) diff --git a/handler_test.go b/handler_test.go index 478720e23..049fdc036 100644 --- a/handler_test.go +++ b/handler_test.go @@ -874,57 +874,3 @@ func MustReadAll(r io.Reader) []byte { } return buf } - -/* -// TODO: move this test to messenger.go (with NewServer()) - -// Ensure that an HTTP message sent to the cluster reaches all nodes. -func TestHTTPNodeSet_Base(t *testing.T) { - - // servers - s1 := NewServer() - s1.Messenger = pilosa.NewMessenger() - n1 := NewHTTPMessageBroker() - n1.messenger = s1.Messenger - s1.Messenger.Broker = n1 - - s2 := NewServer() - s2.Messenger = pilosa.NewMessenger() - n2 := NewHTTPMessageBroker() - n2.messenger = s2.Messenger - s2.Messenger.Broker = n2 - - s3 := NewServer() - s3.Messenger = pilosa.NewMessenger() - n3 := NewHTTPMessageBroker() - n3.messenger = s3.Messenger - s3.Messenger.Broker = n3 - - nodes := []*pilosa.Node{ - {Host: s1.Host()}, - {Host: s2.Host()}, - {Host: s3.Host()}, - } - - // message - msg := &internal.CreateSliceMessage{ - DB: "d", - Slice: 8, - } - - // send message - if err := s1.Messenger.SendMessage(msg, ""); err != nil { - t.Fatalf("failure sending message: %s", err) - } - - if !reflect.DeepEqual(mb1.messageReceived, msg) { - t.Fatalf("unexpected message received by node1: %s", mb1.messageReceived) - } - if !reflect.DeepEqual(mb2.messageReceived, msg) { - t.Fatalf("unexpected message received by node2: %s", mb2.messageReceived) - } - if !reflect.DeepEqual(mb3.messageReceived, msg) { - t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived) - } -} -*/ diff --git a/index_test.go b/index_test.go index c09fe69f5..4a1be4cf2 100644 --- a/index_test.go +++ b/index_test.go @@ -162,34 +162,6 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } } -/* TODO: move this to messenger.go -// Ensure index can handle Messenger messages. -func TestIndex_HandleMessage(t *testing.T) { - // Create a local index. - idx0 := MustOpenIndex() - defer idx0.Close() - - idx0.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - - msg0 := &internal.CreateSliceMessage{ - DB: "d", - Slice: 8, - } - idx0.HandleMessage(msg0) - if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{"d": 8}) { - t.Fatalf("unexpected max slice: %s", ms) - } - - msg1 := &internal.DeleteDBMessage{ - DB: "d", - } - idx0.HandleMessage(msg1) - if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{}) { - t.Fatalf("unexpected delete db: %s", ms) - } -} -*/ - // Index is a test wrapper for pilosa.Index. type Index struct { *pilosa.Index diff --git a/internal/private.pb.go b/internal/private.pb.go index 5ceb800c2..0423cfa08 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -45,8 +45,8 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type DBMeta struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` + ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` + TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *DBMeta) Reset() { *m = DBMeta{} } @@ -281,18 +281,18 @@ func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } if len(m.ColumnLabel) > 0 { - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.ColumnLabel))) i += copy(dAtA[i:], m.ColumnLabel) } + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } return i, nil } @@ -852,11 +852,11 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *DBMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) + l = len(m.ColumnLabel) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.ColumnLabel) + l = len(m.TimeQuantum) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -1137,35 +1137,6 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) } @@ -1194,6 +1165,35 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error { } m.ColumnLabel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3146,43 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 603 bytes of a gzipped FileDescriptorProto + // 600 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, - 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xfa, 0xab, - 0xaf, 0xbe, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, + 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xea, 0xab, + 0xaf, 0xbf, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, - 0xbd, 0x08, 0x63, 0x3c, 0x1f, 0xf3, 0x44, 0x8c, 0x63, 0xd7, 0xe8, 0x19, 0x3b, 0x0e, 0x6b, 0x43, - 0xb2, 0xe2, 0x20, 0x8d, 0xc6, 0x71, 0x72, 0xc2, 0x07, 0x18, 0xb9, 0xa6, 0xae, 0x68, 0x41, 0xf4, - 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0xba, 0x58, 0xd3, - 0xd5, 0x31, 0x79, 0x0c, 0x9d, 0xe3, 0x64, 0x82, 0x79, 0x81, 0x87, 0x09, 0x1f, 0x44, 0x78, 0xa9, - 0xe8, 0x56, 0xd8, 0x1c, 0x4a, 0xb6, 0xc0, 0x39, 0xe0, 0xc3, 0xf7, 0x78, 0x31, 0xcd, 0xd0, 0xb5, - 0x14, 0x49, 0x03, 0xd4, 0xd9, 0x7e, 0xf8, 0x11, 0x5d, 0xbb, 0x67, 0xec, 0xac, 0xb1, 0x06, 0x98, - 0x9f, 0x68, 0xe9, 0xc6, 0x44, 0x94, 0x42, 0xe7, 0x38, 0xce, 0xd2, 0x5c, 0x30, 0x2c, 0xb2, 0x34, - 0x29, 0x90, 0x74, 0xc1, 0x3a, 0xcc, 0xf3, 0x52, 0xae, 0x3c, 0xd2, 0x6b, 0xe8, 0xfa, 0x51, 0x3a, - 0x1c, 0x05, 0x5c, 0x70, 0x86, 0x1f, 0xc6, 0x58, 0x08, 0xd2, 0x01, 0x33, 0xf0, 0xcb, 0x22, 0x33, - 0xf0, 0xc9, 0x7d, 0x58, 0x52, 0x63, 0x97, 0x9e, 0xe8, 0x40, 0xa2, 0xea, 0xa6, 0xd2, 0x6d, 0x33, - 0x1d, 0x48, 0xb4, 0x1f, 0x85, 0x43, 0xad, 0xd7, 0x66, 0x3a, 0x20, 0x04, 0xec, 0x37, 0x21, 0x5e, - 0x95, 0x22, 0xd5, 0x99, 0x9e, 0xc3, 0x46, 0xab, 0x73, 0x29, 0x70, 0x0b, 0x1c, 0x3f, 0x14, 0x31, - 0xcf, 0x8e, 0x83, 0xc2, 0x35, 0x7a, 0xd6, 0x8e, 0xcd, 0x1a, 0x80, 0x78, 0x00, 0xaf, 0xf2, 0xf4, - 0x6d, 0x18, 0xa1, 0x4c, 0x9b, 0x2a, 0xdd, 0x42, 0xe8, 0x23, 0x58, 0x52, 0xfe, 0xfc, 0x99, 0x86, - 0x7e, 0x31, 0x60, 0xe3, 0x94, 0x5f, 0x2b, 0x69, 0x45, 0xdd, 0xfa, 0x08, 0x9c, 0x1a, 0x54, 0x77, - 0x56, 0xf7, 0x9e, 0xec, 0x56, 0x9b, 0xb4, 0x7b, 0xa3, 0xbe, 0x41, 0x0e, 0x13, 0x91, 0x4f, 0x59, - 0x73, 0x79, 0xf3, 0x39, 0x74, 0x7e, 0x4f, 0x4a, 0xdf, 0x47, 0x38, 0xad, 0x7c, 0x1f, 0xe1, 0x54, - 0xfa, 0x34, 0xe1, 0xd1, 0x58, 0x7b, 0x6a, 0x33, 0x1d, 0xec, 0x9b, 0xcf, 0x0c, 0xba, 0x0f, 0xe4, - 0x20, 0x47, 0x2e, 0x50, 0x11, 0x9c, 0x62, 0x51, 0xf0, 0x77, 0xb8, 0xe8, 0x9b, 0x68, 0x9f, 0xcd, - 0x96, 0xcf, 0xf4, 0x01, 0xac, 0x07, 0x18, 0xa1, 0x40, 0xb9, 0xf5, 0x0b, 0x2f, 0xd2, 0x97, 0xb0, - 0xae, 0xe9, 0x6f, 0x2d, 0x21, 0x0f, 0xc1, 0x96, 0x1b, 0xae, 0xa8, 0x57, 0xf7, 0xba, 0x8d, 0x09, - 0xfa, 0x2d, 0x31, 0x95, 0xa5, 0xc3, 0x4a, 0x67, 0xf9, 0x24, 0x6e, 0xd5, 0xb9, 0x60, 0x77, 0xb6, - 0xcb, 0x0e, 0x96, 0xea, 0x70, 0xaf, 0xe9, 0x50, 0x3f, 0xaf, 0xb2, 0xc9, 0x3e, 0x10, 0x3d, 0xd0, - 0xff, 0x37, 0xa1, 0x41, 0x89, 0xca, 0xed, 0x3b, 0x93, 0x59, 0x7d, 0x41, 0x9d, 0x6b, 0x05, 0xe6, - 0xdf, 0x14, 0x7c, 0x32, 0x64, 0xb3, 0x85, 0x1c, 0xff, 0xe4, 0x93, 0xfc, 0x4f, 0x54, 0xdb, 0x50, - 0x3e, 0x95, 0x3a, 0x26, 0xdb, 0xb0, 0xac, 0xfa, 0x15, 0xae, 0xad, 0x16, 0x6e, 0x7d, 0x4e, 0x07, - 0x2b, 0xd3, 0xf4, 0x35, 0x38, 0x67, 0xe9, 0x25, 0xf6, 0x05, 0x17, 0x6a, 0x9e, 0xa3, 0xb4, 0x10, - 0x95, 0x16, 0x79, 0x56, 0xfb, 0x20, 0x93, 0x95, 0x05, 0xba, 0xd2, 0x03, 0x2b, 0xf0, 0x0b, 0xd7, - 0x52, 0xe4, 0x77, 0xdb, 0x02, 0x99, 0x4c, 0xf8, 0xdd, 0x6f, 0x33, 0xcf, 0xf8, 0x3e, 0xf3, 0x8c, - 0x1f, 0x33, 0xcf, 0xf8, 0xfc, 0xd3, 0xbb, 0x33, 0x58, 0x56, 0xbf, 0xd0, 0xa7, 0xbf, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xc3, 0x9a, 0x29, 0xff, 0x53, 0x05, 0x00, 0x00, + 0x3d, 0x48, 0xa3, 0x71, 0x9c, 0x9c, 0xf0, 0x01, 0x46, 0xae, 0xd1, 0x33, 0x76, 0x1c, 0xd6, 0x86, + 0x64, 0xc5, 0x45, 0x18, 0xe3, 0xf9, 0x98, 0x27, 0x62, 0x1c, 0xbb, 0xa6, 0xae, 0x68, 0x41, 0xf4, + 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0x6d, 0xba, 0x3a, + 0x26, 0x8f, 0xa1, 0x73, 0x9c, 0x4c, 0x30, 0x2f, 0xf0, 0x30, 0xe1, 0x83, 0x08, 0x2f, 0x15, 0xdd, + 0x0a, 0x9b, 0x43, 0xc9, 0x16, 0x38, 0x07, 0x7c, 0xf8, 0x1e, 0x2f, 0xa6, 0x19, 0xba, 0x96, 0x22, + 0x69, 0x80, 0x3a, 0xdb, 0x0f, 0x3f, 0xa2, 0x6b, 0xf7, 0x8c, 0x9d, 0x35, 0xd6, 0x00, 0xf3, 0x7a, + 0x97, 0x6e, 0xea, 0xa5, 0xd0, 0x39, 0x8e, 0xb3, 0x34, 0x17, 0x0c, 0x8b, 0x2c, 0x4d, 0x0a, 0x24, + 0x5d, 0xb0, 0x0e, 0xf3, 0xbc, 0x94, 0x2b, 0x8f, 0xf4, 0x1a, 0xba, 0x7e, 0x94, 0x0e, 0x47, 0x01, + 0x17, 0x9c, 0xe1, 0x87, 0x31, 0x16, 0x82, 0x74, 0xc0, 0x0c, 0xfc, 0xb2, 0xc8, 0x0c, 0x7c, 0x72, + 0x1f, 0x96, 0xd4, 0xb5, 0x4b, 0x4f, 0x74, 0x20, 0x51, 0xd5, 0xa9, 0x74, 0xdb, 0x4c, 0x07, 0x12, + 0xed, 0x47, 0xe1, 0x50, 0xeb, 0xb5, 0x99, 0x0e, 0x08, 0x01, 0xfb, 0x4d, 0x88, 0x57, 0xa5, 0x48, + 0x75, 0xa6, 0xe7, 0xb0, 0xd1, 0x9a, 0x5c, 0x0a, 0xdc, 0x02, 0xc7, 0x0f, 0x45, 0xcc, 0xb3, 0xe3, + 0xa0, 0x70, 0x8d, 0x9e, 0xb5, 0x63, 0xb3, 0x06, 0x20, 0x1e, 0xc0, 0xab, 0x3c, 0x7d, 0x1b, 0x46, + 0x28, 0xd3, 0xa6, 0x4a, 0xb7, 0x10, 0xfa, 0x08, 0x96, 0x94, 0x3f, 0x7f, 0xa6, 0xa1, 0x5f, 0x0c, + 0xd8, 0x38, 0xe5, 0xd7, 0x4a, 0x5a, 0x51, 0x8f, 0x3e, 0x02, 0xa7, 0x06, 0x55, 0xcf, 0xea, 0xde, + 0x93, 0xdd, 0x6a, 0x93, 0x76, 0x6f, 0xd4, 0x37, 0xc8, 0x61, 0x22, 0xf2, 0x29, 0x6b, 0x9a, 0x37, + 0x9f, 0x43, 0xe7, 0xf7, 0xa4, 0xf4, 0x7d, 0x84, 0xd3, 0xca, 0xf7, 0x11, 0x4e, 0xa5, 0x4f, 0x13, + 0x1e, 0x8d, 0xb5, 0xa7, 0x36, 0xd3, 0xc1, 0xbe, 0xf9, 0xcc, 0xa0, 0xfb, 0x40, 0x0e, 0x72, 0xe4, + 0x02, 0x15, 0xc1, 0x29, 0x16, 0x05, 0x7f, 0x87, 0x8b, 0xbe, 0x89, 0xf6, 0xd9, 0x6c, 0xf9, 0x4c, + 0x1f, 0xc0, 0x7a, 0x80, 0x11, 0x0a, 0x94, 0x5b, 0xbf, 0xb0, 0x91, 0xbe, 0x84, 0x75, 0x4d, 0x7f, + 0x6b, 0x09, 0x79, 0x08, 0xb6, 0xdc, 0x70, 0x45, 0xbd, 0xba, 0xd7, 0x6d, 0x4c, 0xd0, 0x6f, 0x89, + 0xa9, 0x2c, 0x1d, 0x56, 0x3a, 0xcb, 0x27, 0x71, 0xab, 0xce, 0x05, 0xbb, 0xb3, 0x5d, 0x4e, 0xb0, + 0xd4, 0x84, 0x7b, 0xcd, 0x84, 0xfa, 0x79, 0x95, 0x43, 0xf6, 0x81, 0xe8, 0x0b, 0xfd, 0xff, 0x10, + 0x1a, 0x94, 0xa8, 0xdc, 0xbe, 0x33, 0x99, 0xd5, 0x0d, 0xea, 0x5c, 0x2b, 0x30, 0xff, 0xa6, 0xe0, + 0x93, 0x21, 0x87, 0x2d, 0xe4, 0xf8, 0x27, 0x9f, 0xe4, 0x7f, 0xa2, 0xda, 0x86, 0xf2, 0xa9, 0xd4, + 0x31, 0xd9, 0x86, 0x65, 0x35, 0xaf, 0x70, 0x6d, 0xb5, 0x70, 0xeb, 0x73, 0x3a, 0x58, 0x99, 0xa6, + 0xaf, 0xc1, 0x39, 0x4b, 0x2f, 0xb1, 0x2f, 0xb8, 0x50, 0xf7, 0x39, 0x4a, 0x0b, 0x51, 0x69, 0x91, + 0x67, 0xb5, 0x0f, 0x32, 0x59, 0x59, 0xa0, 0x2b, 0x3d, 0xb0, 0x02, 0xbf, 0x70, 0x2d, 0x45, 0x7e, + 0xb7, 0x2d, 0x90, 0xc9, 0x84, 0xdf, 0xfd, 0x36, 0xf3, 0x8c, 0xef, 0x33, 0xcf, 0xf8, 0x31, 0xf3, + 0x8c, 0xcf, 0x3f, 0xbd, 0x3b, 0x83, 0x65, 0xf5, 0x0b, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, + 0x3a, 0x23, 0x0f, 0xb4, 0x53, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 40e83eb6b..ec060e355 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,8 +3,8 @@ syntax = "proto3"; package internal; message DBMeta { - string TimeQuantum = 1; - string ColumnLabel = 2; + string ColumnLabel = 1; + string TimeQuantum = 2; } message FrameMeta { From a4921ac55a3e6c48efdb0d4fa42235cc814c61bc Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 15:14:11 -0500 Subject: [PATCH 48/63] remove leftover Receive() method from gossip implementation --- gossip/gossip.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 050435441..772892926 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -154,13 +154,6 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error { return nil } -func (g *GossipNodeSet) Receive(pb proto.Message) error { - if err := g.handler.ReceiveMessage(pb); err != nil { - return err - } - return nil -} - // implementation of the memberlist.Delegate interface func (g *GossipNodeSet) NodeMeta(limit int) []byte { return []byte{} @@ -172,7 +165,7 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) { g.logger().Printf("unmarshal message error: %s", err) return } - if err := g.Receive(m); err != nil { + if err := g.handler.ReceiveMessage(m); err != nil { g.logger().Printf("receive message error: %s", err) return } From 3487bc373f136bd3bb89cd1e472ebff8693d666a Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 20 Apr 2017 15:45:00 -0500 Subject: [PATCH 49/63] group Server interface implementation function together --- server.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/server.go b/server.go index e93b8e111..567461ccc 100644 --- a/server.go +++ b/server.go @@ -236,20 +236,7 @@ func (s *Server) monitorMaxSlices() { } } -// LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. -// In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) LocalState() (proto.Message, error) { - if s.Index == nil { - return nil, errors.New("Server.Index is nil.") - } - return &internal.NodeState{ - Host: s.Host, - State: "OK", // TODO: make this work, pull from s.Cluster.Node - DBs: encodeDBs(s.Index.DBs()), - }, nil -} - +// ReceiveMessage represents an implementation of BroadcastHandler. func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { case *internal.CreateSliceMessage: @@ -284,6 +271,21 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return nil } +// Server implements gossip.StateHandler. +// LocalState returns the state of the local node as well as the +// index (dbs/frames) according to the local node. +// In a gossip implementation, memberlist.Delegate.LocalState() uses this. +func (s *Server) LocalState() (proto.Message, error) { + if s.Index == nil { + return nil, errors.New("Server.Index is nil.") + } + return &internal.NodeState{ + Host: s.Host, + State: "OK", // TODO: make this work, pull from s.Cluster.Node + DBs: encodeDBs(s.Index.DBs()), + }, nil +} + // HandleRemoteState receives incoming NodeState from remote nodes. func (s *Server) HandleRemoteState(pb proto.Message) error { return s.mergeRemoteState(pb.(*internal.NodeState)) From 9c02edc4e3d09e9d14668825ccd4e1f64c96bfaa Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 20 Apr 2017 17:28:22 -0500 Subject: [PATCH 50/63] #457 rewrite validateOptions, update frame options --- handler.go | 69 ++++++++++++++++++++++++++-------------- handler_internal_test.go | 3 ++ 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/handler.go b/handler.go index 0e25c3f2f..4f6f2e3bf 100644 --- a/handler.go +++ b/handler.go @@ -234,23 +234,25 @@ type postDBRequest struct { // Custom Unmarshal JSON to validate request body when creating a new database func (p *postDBRequest) UnmarshalJSON(b []byte) error { + validDBOptions := []string{"columnLabel"} var data map[string]interface{} if err := json.Unmarshal(b, &data); err != nil { return err } + p.Options = DBOptions{} for key, value := range data { switch key { case "options": - value, err := validateOptions(data, "columnLabel") + values, err := validateOptions(data, validDBOptions) if err != nil { return err } - if value == "" { - p.Options = DBOptions{} - } else { - p.Options = DBOptions{ColumnLabel: value} + for k, v := range values { + switch k { + case "columnLabel": + p.Options.ColumnLabel = v + } } - default: return fmt.Errorf("Unknown key: %v:%v", key, value) } @@ -258,31 +260,39 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error { return nil } -func validateOptions(data map[string]interface{}, field string) (string, error) { +func validateOptions(data map[string]interface{}, field []string) (map[string]string, error) { options, ok := data["options"].(map[string]interface{}) if !ok { - return "", errors.New("options is not map[string]interface{}") + return map[string]string{}, errors.New("options is not map[string]interface{}") } - var optionValue string + optionValue := make(map[string]string) if len(options) == 0 { - optionValue = "" + optionValue = map[string]string{} } else { for k, v := range options { - switch k { - case field: - val, ok := options[field].(string) + if foundItem(field, k) { + val, ok := options[k].(string) if !ok { - return "", fmt.Errorf("invalid option %v: {%v:%v}", field, k, v) + return map[string]string{}, fmt.Errorf("invalid option %v: {%v:%v}", field, k, v) } - optionValue = val - default: - return "", fmt.Errorf("invalid key for options {%v:%v}", k, v) + optionValue[k] = val + } else { + return map[string]string{}, fmt.Errorf("invalid key for options {%v:%v}", k, v) } } } return optionValue, nil } +func foundItem(items []string, item string) bool { + for _, i := range items { + if item == i { + return true + } + } + return false +} + type postDBResponse struct{} // handleDeleteDB handles DELETE /db request. @@ -465,24 +475,37 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } } -// Custom Unmarshal JSON to validate request body when creating a new frame +// Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions, +// adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed func (p *postFrameRequest) UnmarshalJSON(b []byte) error { + validFrameOptions := []string{"rowLabel", "cacheType", "inverseEnabled"} var data map[string]interface{} if err := json.Unmarshal(b, &data); err != nil { return err } + p.Options = FrameOptions{} for key, value := range data { switch key { case "options": - value, err := validateOptions(data, "rowLabel") + values, err := validateOptions(data, validFrameOptions) if err != nil { return err } - if value == "" { - p.Options = FrameOptions{} - } else { - p.Options = FrameOptions{RowLabel: value} + for k, v := range values { + switch k { + case "rowLabel": + p.Options.RowLabel = v + case "cacheType": + p.Options.CacheType = v + case "inverseEnabled": + inverse, err := strconv.ParseBool(v) + if err != nil { + continue + } + p.Options.InverseEnabled = inverse + } } + default: return fmt.Errorf("Unknown key: {%v:%v}", key, value) } diff --git a/handler_internal_test.go b/handler_internal_test.go index 8a7bc910f..71b705f0a 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -54,6 +54,9 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"option": {}}`, err: "Unknown key: {option:map[]}"}, {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, {json: `{"options": {"rowLabl": "test"}}`, err: "invalid key for options {rowLabl:test}"}, + {json: `{"options": {"rowLabel": "test", "inverseEnabled": "true"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, + {json: `{"options": {"rowLabel": "test", "inverseEnabled": "true", "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"rowLabel": "test", "inverse": "true", "cacheType": "type"}}`, err: "invalid key for options {inverse:true}"}, } for _, test := range tests { actual := &postFrameRequest{} From 9198dfab59d3cefd2eae1ca60a273b0a42db58bf Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Thu, 20 Apr 2017 22:11:01 -0500 Subject: [PATCH 51/63] inverseEnabled check for bool type --- handler.go | 41 +++++++++++++++++++++++----------------- handler_internal_test.go | 6 +++--- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/handler.go b/handler.go index 4f6f2e3bf..08ca21b31 100644 --- a/handler.go +++ b/handler.go @@ -250,7 +250,7 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error { for k, v := range values { switch k { case "columnLabel": - p.Options.ColumnLabel = v + p.Options.ColumnLabel = v.(string) } } default: @@ -260,24 +260,35 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error { return nil } -func validateOptions(data map[string]interface{}, field []string) (map[string]string, error) { +func validateOptions(data map[string]interface{}, field []string) (map[string]interface{}, error) { options, ok := data["options"].(map[string]interface{}) + optionValue := make(map[string]interface{}) if !ok { - return map[string]string{}, errors.New("options is not map[string]interface{}") + return nil, errors.New("options is not map[string]interface{}") } - optionValue := make(map[string]string) + if len(options) == 0 { - optionValue = map[string]string{} + optionValue = nil } else { for k, v := range options { if foundItem(field, k) { - val, ok := options[k].(string) - if !ok { - return map[string]string{}, fmt.Errorf("invalid option %v: {%v:%v}", field, k, v) + switch k { + case "inverseEnabled": + val, ok := options[k].(bool) + if !ok { + return nil, fmt.Errorf("invalid option type %v: {%v:%v}", field, k, v) + } + optionValue[k] = val + default: + val, ok := options[k].(string) + if !ok { + return nil, fmt.Errorf("invalid option %v: {%v:%v}", field, k, v) + } + optionValue[k] = val } - optionValue[k] = val + } else { - return map[string]string{}, fmt.Errorf("invalid key for options {%v:%v}", k, v) + return nil, fmt.Errorf("invalid key for options {%v:%v}", k, v) } } } @@ -494,15 +505,11 @@ func (p *postFrameRequest) UnmarshalJSON(b []byte) error { for k, v := range values { switch k { case "rowLabel": - p.Options.RowLabel = v + p.Options.RowLabel = v.(string) case "cacheType": - p.Options.CacheType = v + p.Options.CacheType = v.(string) case "inverseEnabled": - inverse, err := strconv.ParseBool(v) - if err != nil { - continue - } - p.Options.InverseEnabled = inverse + p.Options.InverseEnabled = v.(bool) } } diff --git a/handler_internal_test.go b/handler_internal_test.go index 71b705f0a..5942bd12c 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -54,9 +54,9 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"option": {}}`, err: "Unknown key: {option:map[]}"}, {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, {json: `{"options": {"rowLabl": "test"}}`, err: "invalid key for options {rowLabl:test}"}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": "true"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, - {json: `{"options": {"rowLabel": "test", "inverseEnabled": "true", "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, - {json: `{"options": {"rowLabel": "test", "inverse": "true", "cacheType": "type"}}`, err: "invalid key for options {inverse:true}"}, + {json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, + {json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "invalid key for options {inverse:true}"}, } for _, test := range tests { actual := &postFrameRequest{} From 7e935dd90a04203817bd2449a909e8779de4da8a Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 21 Apr 2017 11:37:32 -0500 Subject: [PATCH 52/63] rewrite custom UnmarshalJSON for postDBRequest and postFrameRequest --- handler.go | 139 ++++++++++++++++++--------------------- handler_internal_test.go | 8 +-- 2 files changed, 69 insertions(+), 78 deletions(-) diff --git a/handler.go b/handler.go index 08ca21b31..adb3967e1 100644 --- a/handler.go +++ b/handler.go @@ -21,6 +21,7 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "reflect" ) // Handler represents an HTTP handler. @@ -232,67 +233,53 @@ type postDBRequest struct { Options DBOptions `json:"options"` } +//_postDBRequest is necessary to avoid recursion while decoding. +type _postDBRequest postDBRequest + // Custom Unmarshal JSON to validate request body when creating a new database func (p *postDBRequest) UnmarshalJSON(b []byte) error { - validDBOptions := []string{"columnLabel"} - var data map[string]interface{} - if err := json.Unmarshal(b, &data); err != nil { + + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { return err } - p.Options = DBOptions{} - for key, value := range data { - switch key { - case "options": - values, err := validateOptions(data, validDBOptions) - if err != nil { - return err - } - for k, v := range values { - switch k { - case "columnLabel": - p.Options.ColumnLabel = v.(string) - } - } - default: - return fmt.Errorf("Unknown key: %v:%v", key, value) - } + + validDBOptions := getValidOptions(DBOptions{}) + err := validateOptions(m, validDBOptions) + if err != nil { + return err } + // Unmarshal expected values. + var _p _postDBRequest + if err := json.Unmarshal(b, &_p); err != nil { + return err + } + + p.Options = _p.Options + return nil } -func validateOptions(data map[string]interface{}, field []string) (map[string]interface{}, error) { - options, ok := data["options"].(map[string]interface{}) - optionValue := make(map[string]interface{}) - if !ok { - return nil, errors.New("options is not map[string]interface{}") - } - - if len(options) == 0 { - optionValue = nil - } else { - for k, v := range options { - if foundItem(field, k) { - switch k { - case "inverseEnabled": - val, ok := options[k].(bool) - if !ok { - return nil, fmt.Errorf("invalid option type %v: {%v:%v}", field, k, v) - } - optionValue[k] = val - default: - val, ok := options[k].(string) - if !ok { - return nil, fmt.Errorf("invalid option %v: {%v:%v}", field, k, v) - } - optionValue[k] = val - } - - } else { - return nil, fmt.Errorf("invalid key for options {%v:%v}", k, v) +// Raise errors for any unknown key +func validateOptions(data map[string]interface{}, validDBOptions []string) error { + for k, v := range data { + switch k { + case "options": + options, ok := v.(map[string]interface{}) + if !ok { + return errors.New("options is not map[string]interface{}") } + for kk, vv := range options { + if !foundItem(validDBOptions, kk) { + return fmt.Errorf("Unknown key: %v:%v", kk, vv) + } + } + default: + return fmt.Errorf("Unknown key: %v:%v", k, v) } } - return optionValue, nil + return nil } func foundItem(items []string, item string) bool { @@ -486,41 +473,45 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } } +type _postFrameRequest postFrameRequest + // Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions, // adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed func (p *postFrameRequest) UnmarshalJSON(b []byte) error { - validFrameOptions := []string{"rowLabel", "cacheType", "inverseEnabled"} - var data map[string]interface{} - if err := json.Unmarshal(b, &data); err != nil { + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { return err } - p.Options = FrameOptions{} - for key, value := range data { - switch key { - case "options": - values, err := validateOptions(data, validFrameOptions) - if err != nil { - return err - } - for k, v := range values { - switch k { - case "rowLabel": - p.Options.RowLabel = v.(string) - case "cacheType": - p.Options.CacheType = v.(string) - case "inverseEnabled": - p.Options.InverseEnabled = v.(bool) - } - } - default: - return fmt.Errorf("Unknown key: {%v:%v}", key, value) - } + validFrameOptions := getValidOptions(FrameOptions{}) + err := validateOptions(m, validFrameOptions) + if err != nil { + return err } + + // Unmarshal expected values. + var _p _postFrameRequest + if err := json.Unmarshal(b, &_p); err != nil { + return err + } + + p.Options = _p.Options return nil } +func getValidOptions(option interface{}) []string { + validFrameOptions := []string{} + val := reflect.ValueOf(option) + for i := 0; i < val.Type().NumField(); i++ { + jsonTag := val.Type().Field(i).Tag.Get("json") + s := strings.Split(jsonTag, ",") + validFrameOptions = append(validFrameOptions, s[0]) + } + return validFrameOptions +} + type postFrameRequest struct { Options FrameOptions `json:"options"` } diff --git a/handler_internal_test.go b/handler_internal_test.go index 5942bd12c..92221f1d7 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -17,7 +17,7 @@ func TestPostDBRequestUnmarshalJSON(t *testing.T) { {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"columnLabel": "test"}}`, expected: postDBRequest{Options: DBOptions{ColumnLabel: "test"}}}, - {json: `{"options": {"columnLabl": "test"}}`, err: "invalid key for options {columnLabl:test}"}, + {json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"}, } for _, test := range tests { actual := &postDBRequest{} @@ -51,12 +51,12 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { }{ {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, - {json: `{"option": {}}`, err: "Unknown key: {option:map[]}"}, + {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}}, - {json: `{"options": {"rowLabl": "test"}}`, err: "invalid key for options {rowLabl:test}"}, + {json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"}, {json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}}, {json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}}, - {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "invalid key for options {inverse:true}"}, + {json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { actual := &postFrameRequest{} From e1404cf330881d89286c42cc578e3dfba0c24a49 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Fri, 21 Apr 2017 11:42:31 -0500 Subject: [PATCH 53/63] change variable name of getValidOptions --- handler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handler.go b/handler.go index adb3967e1..5af464764 100644 --- a/handler.go +++ b/handler.go @@ -502,14 +502,14 @@ func (p *postFrameRequest) UnmarshalJSON(b []byte) error { } func getValidOptions(option interface{}) []string { - validFrameOptions := []string{} + validOptions := []string{} val := reflect.ValueOf(option) for i := 0; i < val.Type().NumField(); i++ { jsonTag := val.Type().Field(i).Tag.Get("json") s := strings.Split(jsonTag, ",") - validFrameOptions = append(validFrameOptions, s[0]) + validOptions = append(validOptions, s[0]) } - return validFrameOptions + return validOptions } type postFrameRequest struct { From 93f36d0e9ab18f51f30aae4f74d52c128ee43029 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 21 Apr 2017 13:54:39 -0500 Subject: [PATCH 54/63] adds a basic BroadcastReceiver test --- broadcast_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/broadcast_test.go b/broadcast_test.go index ee2d720a9..a2842a919 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -35,3 +35,57 @@ func testMessageMarshal(t *testing.T, m proto.Message) { t.Fatalf("unexpected message marshalling: %s", unmarshalled) } } + +// Ensure that BroadcastReceiver can register a BroadcastHandler. +func TestBroadcast_BroadcastReceiver(t *testing.T) { + + s := pilosa.NewServer() + + sbr := NewSimpleBroadcastReceiver() + sbh := NewSimpleBroadcastHandler() + + s.BroadcastReceiver = sbr + s.BroadcastReceiver.Start(sbh) + + msg := &internal.DeleteDBMessage{ + DB: "d", + } + + s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) + + // Make sure the message received is what was sentd + if !reflect.DeepEqual(sbh.receivedMessage, msg) { + t.Fatalf("unexpected message: %s", sbh.receivedMessage) + } +} + +type SimpleBroadcastReceiver struct { + broadcastHandler pilosa.BroadcastHandler +} + +func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver { + return &SimpleBroadcastReceiver{} +} + +func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error { + r.broadcastHandler = h + return nil +} + +func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error { + r.broadcastHandler.ReceiveMessage(pb) + return nil +} + +type SimpleBroadcastHandler struct { + receivedMessage proto.Message +} + +func NewSimpleBroadcastHandler() *SimpleBroadcastHandler { + return &SimpleBroadcastHandler{} +} + +func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error { + h.receivedMessage = pb.(proto.Message) + return nil +} From d34687121f4100986b5d68c6aad3f87826e10fa0 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 21 Apr 2017 15:59:41 -0600 Subject: [PATCH 55/63] Rename bitmap/profile to row/column. --- NOTES | 4 +- README.md | 14 +- attr.go | 2 +- attr_test.go | 6 +- cache.go | 64 ++++----- client.go | 50 +++---- client_test.go | 38 +++--- cmd/export.go | 2 +- cmd/import.go | 2 +- cmd/sort.go | 2 +- ctl/bench.go | 10 +- ctl/import.go | 16 +-- ctl/sort.go | 26 ++-- db.go | 22 ++-- executor.go | 76 +++++------ executor_test.go | 66 +++++----- fragment.go | 292 ++++++++++++++++++++--------------------- fragment_test.go | 171 ++++++++++-------------- frame.go | 58 ++++---- frame_test.go | 4 +- handler.go | 88 ++++++------- handler_test.go | 50 +++---- index.go | 22 ++-- index_test.go | 16 +-- internal/private.pb.go | 132 +++++++++---------- internal/private.proto | 6 +- internal/public.pb.go | 215 +++++++++++++++--------------- internal/public.proto | 14 +- iterator.go | 104 +++++++-------- pilosa.go | 44 +++---- server/server_test.go | 126 +++++++++--------- view.go | 20 +-- view_test.go | 28 ++-- 33 files changed, 880 insertions(+), 910 deletions(-) diff --git a/NOTES b/NOTES index 49d372099..6ba447da0 100644 --- a/NOTES +++ b/NOTES @@ -1,10 +1,10 @@ - DB Profile + DB Column ┌───────────▼────────────────────────────┐ │0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ -Bitmap──▶0000000000000000000000000000000000000000│ + Row──▶0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ │────────────────────────────────────────┤ │0000000000000000000000000000000000000000│ diff --git a/README.md b/README.md index 9943bf15f..d8318820b 100644 --- a/README.md +++ b/README.md @@ -209,17 +209,17 @@ A return value of `{"results":[true]}` indicates that the bit was toggled from 1 A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed. --- -#### SetBitmapAttrs() +#### SetRowAttrs() ``` -SetBitmapAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true) +SetRowAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true) ``` Returns `{"results":[null]}` --- -#### SetProfileAttrs() +#### SetColumnAttrs() --- ``` -SetProfileAttrs(user=10, friends=123, username="mrpi", active=true) +SetColumnAttrs(user=10, friends=123, username="mrpi", active=true) ``` Returns `{"results":[null]}` @@ -230,11 +230,11 @@ Returns `{"results":[null]}` Bitmap(project=10, frame="collaboration") ``` Returns `{"results":[{"attrs":{"stars":123, "url":"http://projects.pilosa.com/10", "active":true},"bits":[1,2]}]}` where `attrs` are the -attributes set using `SetBitmapAttrs()` and `bits` are the bits set using `SetBit()`. +attributes set using `SetRowAttrs()` and `bits` are the bits set using `SetBit()`. -In order to return profile attributes attached to the profiles of a bitmap, add `&profiles=true` to the query string. Sample response: +In order to return column attributes attached to the columns of a bitmap, add `&columnAttrs=true` to the query string. Sample response: ``` -{"results":[{"attrs":{},"bits":[10]}],"profiles":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]} +{"results":[{"attrs":{},"bits":[10]}],"columnAttrs":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]} ``` --- diff --git a/attr.go b/attr.go index f619dda9f..3859a1dc7 100644 --- a/attr.go +++ b/attr.go @@ -226,7 +226,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro return m, nil } -// txAttrs returns a map of attributes for a bitmap. +// txAttrs returns a map of attributes for an id. func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) if v == nil { diff --git a/attr_test.go b/attr_test.go index f026f6244..cfc31d54b 100644 --- a/attr_test.go +++ b/attr_test.go @@ -9,7 +9,7 @@ import ( "github.com/pilosa/pilosa" ) -// Ensure database can set and retrieve profile attributes. +// Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { s := MustOpenAttrStore() defer s.Close() @@ -23,14 +23,14 @@ func TestAttrStore_Attrs(t *testing.T) { t.Fatal(err) } - // Retrieve attributes for profile #1. + // Retrieve attributes for column #1. if m, err := s.Attrs(1); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) { t.Fatalf("unexpected attrs(1): %#v", m) } - // Retrieve attributes for profile #2. + // Retrieve attributes for column #2. if m, err := s.Attrs(2); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { diff --git a/cache.go b/cache.go index 7da28eb74..1c338302e 100644 --- a/cache.go +++ b/cache.go @@ -17,15 +17,15 @@ const ( ThresholdFactor = 1.1 ) -// Cache represents a cache for bitmap counts. +// Cache represents a cache of counts. type Cache interface { - Add(bitmapID uint64, n uint64) - BulkAdd(bitmapID uint64, n uint64) - Get(bitmapID uint64) uint64 + Add(id uint64, n uint64) + BulkAdd(id uint64, n uint64) + Get(id uint64) uint64 Len() int - // Returns a list of all bitmap IDs. - BitmapIDs() []uint64 + // Returns a list of all IDs. + IDs() []uint64 // Updates the cache, if necessary. Invalidate() @@ -53,19 +53,19 @@ func NewLRUCache(maxEntries uint32) *LRUCache { return c } -func (c *LRUCache) BulkAdd(bitmapID, n uint64) { - c.Add(bitmapID, n) +func (c *LRUCache) BulkAdd(id, n uint64) { + c.Add(id, n) } -// Add adds a bitmap to the cache. -func (c *LRUCache) Add(bitmapID, n uint64) { - c.cache.Add(bitmapID, n) - c.counts[bitmapID] = n +// Add adds a count to the cache. +func (c *LRUCache) Add(id, n uint64) { + c.cache.Add(id, n) + c.counts[id] = n } -// Get returns a bitmap with a given id. -func (c *LRUCache) Get(bitmapID uint64) uint64 { - n, _ := c.cache.Get(bitmapID) +// Get returns a count for a given id. +func (c *LRUCache) Get(id uint64) uint64 { + n, _ := c.cache.Get(id) nn, _ := n.(uint64) return nn } @@ -79,8 +79,8 @@ func (c *LRUCache) Invalidate() {} // Recalculate is a no-op. func (c *LRUCache) Recalculate() {} -// BitmapIDs returns a list of all bitmap IDs in the cache. -func (c *LRUCache) BitmapIDs() []uint64 { +// IDs returns a list of all IDs in the cache. +func (c *LRUCache) IDs() []uint64 { a := make([]uint64, 0, len(c.counts)) for id := range c.counts { a = append(a, id) @@ -136,36 +136,36 @@ func NewRankCache(maxEntries uint32) *RankCache { } } -// Add adds a bitmap to the cache. -func (c *RankCache) Add(bitmapID uint64, n uint64) { +// Add adds a count to the cache. +func (c *RankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the bit count on the bitmap is below the threshold. + // Ignore if the bit count is below the threshold. if n < c.thresholdValue { return } - c.entries[bitmapID] = n + c.entries[id] = n c.invalidate() } -// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion. -func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { +// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion. +func (c *RankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() if n < c.thresholdValue { return } - c.entries[bitmapID] = n + c.entries[id] = n } -// Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) uint64 { +// Get returns a count for a given id. +func (c *RankCache) Get(id uint64) uint64 { c.mu.Lock() defer c.mu.Unlock() - return c.entries[bitmapID] + return c.entries[id] } // Len returns the number of items in the cache. @@ -175,8 +175,8 @@ func (c *RankCache) Len() int { return len(c.entries) } -// BitmapIDs returns a list of all bitmap IDs in the cache. -func (c *RankCache) BitmapIDs() []uint64 { +// IDs returns a list of all IDs in the cache. +func (c *RankCache) IDs() []uint64 { c.mu.Lock() defer c.mu.Unlock() a := make([]uint64, 0, len(c.entries)) @@ -242,7 +242,7 @@ func (c *RankCache) recalculate() { } } -// Top returns an ordered list of bitmaps. +// Top returns an ordered list of pairs. func (c *RankCache) Top() []BitmapPair { return c.rankings } // WriteTo writes the cache to w. @@ -258,7 +258,7 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) { // Ensure RankCache implements Cache. var _ Cache = &RankCache{} -// BitmapPair represents a bitmap with an associated identifier. +// BitmapPair represents a id/count pair with an associated identifier. type BitmapPair struct { ID uint64 Count uint64 @@ -271,7 +271,7 @@ func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitmapPairs) Len() int { return len(p) } func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } -// Pair holds a bitmap id and its count. +// Pair holds an id/count pair. type Pair struct { ID uint64 `json:"id"` Count uint64 `json:"count"` diff --git a/client.go b/client.go index e4401a69a..46a27c02f 100644 --- a/client.go +++ b/client.go @@ -316,9 +316,9 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit } func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { - // Separate bitmap and profile IDs to reduce allocations. - bitmapIDs := Bits(bits).BitmapIDs() - profileIDs := Bits(bits).ProfileIDs() + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() // Marshal bits to protobufs. @@ -326,8 +326,8 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e DB: db, Frame: frame, Slice: slice, - BitmapIDs: bitmapIDs, - ProfileIDs: profileIDs, + RowIDs: rowIDs, + ColumnIDs: columnIDs, Timestamps: timestamps, }) if err != nil { @@ -823,7 +823,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli return rsp.Blocks, nil } -// BlockData returns bitmap/profile id pairs for a block. +// BlockData returns row/column id pairs for a block. func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ DB: db, @@ -867,11 +867,11 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui } else if err := proto.Unmarshal(body, &rsp); err != nil { return nil, nil, err } - return rsp.BitmapIDs, rsp.ProfileIDs, nil + return rsp.RowIDs, rsp.ColumnIDs, nil } -// ProfileAttrDiff returns data from differing blocks on a remote host. -func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -913,8 +913,8 @@ func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBloc return rsp.Attrs, nil } -// BitmapAttrDiff returns data from differing blocks on a remote host. -func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *Client) RowAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -960,8 +960,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At // Bit represents the location of a single bit. type Bit struct { - BitmapID uint64 - ProfileID uint64 + RowID uint64 + ColumnID uint64 Timestamp int64 } @@ -972,29 +972,29 @@ func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Bits) Len() int { return len(p) } func (p Bits) Less(i, j int) bool { - if p[i].BitmapID == p[j].BitmapID { - if p[i].ProfileID < p[j].ProfileID { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { return p[i].Timestamp < p[j].Timestamp } - return p[i].ProfileID < p[j].ProfileID + return p[i].ColumnID < p[j].ColumnID } - return p[i].BitmapID < p[j].BitmapID + return p[i].RowID < p[j].RowID } -// BitmapIDs returns a slice of all the bitmap IDs. -func (a Bits) BitmapIDs() []uint64 { +// RowIDs returns a slice of all the row IDs. +func (a Bits) RowIDs() []uint64 { other := make([]uint64, len(a)) for i := range a { - other[i] = a[i].BitmapID + other[i] = a[i].RowID } return other } -// ProfileIDs returns a slice of all the profile IDs. -func (a Bits) ProfileIDs() []uint64 { +// ColumnIDs returns a slice of all the column IDs. +func (a Bits) ColumnIDs() []uint64 { other := make([]uint64, len(a)) for i := range a { - other[i] = a[i].ProfileID + other[i] = a[i].ColumnID } return other } @@ -1012,7 +1012,7 @@ func (a Bits) Timestamps() []int64 { func (a Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) for _, bit := range a { - slice := bit.ProfileID / SliceWidth + slice := bit.ColumnID / SliceWidth m[slice] = append(m[slice], bit) } @@ -1030,7 +1030,7 @@ type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID) + p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp } diff --git a/client_test.go b/client_test.go index f78ee69c4..a05032d57 100644 --- a/client_test.go +++ b/client_test.go @@ -28,7 +28,7 @@ func createCluster(c *pilosa.Cluster) ([]*Server, []*Index) { return server, idx } -// Test distributed TopN Bitmap count across 3 nodes. +// Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { cluster := NewCluster(3) s, idx := createCluster(cluster) @@ -162,7 +162,7 @@ func TestClient_Import(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) - f.Bitmap(0) + f.Row(0) s := NewServer() defer s.Close() @@ -174,18 +174,18 @@ func TestClient_Import(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ - {BitmapID: 0, ProfileID: 1}, - {BitmapID: 0, ProfileID: 5}, - {BitmapID: 200, ProfileID: 6}, + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, }); err != nil { t.Fatal(err) } // Verify data. - if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { + if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -213,7 +213,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { } // Load bitmap into cache to ensure cache gets updated. - f.Bitmap(0) + f.Row(0) s := NewServer() defer s.Close() @@ -225,22 +225,22 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ - {BitmapID: 0, ProfileID: 1}, - {BitmapID: 0, ProfileID: 5}, - {BitmapID: 200, ProfileID: 5}, - {BitmapID: 200, ProfileID: 6}, + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, }); err != nil { t.Fatal(err) } // Verify data. - if a := f.Bitmap(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { + if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { + if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { + if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -279,16 +279,16 @@ func TestClient_BackupRestore(t *testing.T) { } // Verify data. - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { t.Fatalf("unexpected bits: %+v", a) } } diff --git a/cmd/export.go b/cmd/export.go index fb22ee369..e4511d40f 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -23,7 +23,7 @@ the output is written to STDOUT. The format of the CSV file is: - BITMAPID,PROFILEID + ROWID,COLUMNID The file does not contain any headers. `, diff --git a/cmd/import.go b/cmd/import.go index b7998f750..d8bc932e8 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -21,7 +21,7 @@ of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: - BITMAPID,PROFILEID,[TIME] + ROWID,COLUMNID,[TIME] The file should contain no headers. The TIME column is optional and can be omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. diff --git a/cmd/sort.go b/cmd/sort.go index 0ea7d9007..668206070 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -25,7 +25,7 @@ Sorts the import data at PATH into the optimal sort order for importing. The format of the CSV file is: - BITMAPID,PROFILEID + ROWID,COLUMNID The file should contain no headers. `, diff --git a/ctl/bench.go b/ctl/bench.go index 2216bace0..167017f2f 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -63,17 +63,17 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e return pilosa.ErrFrameRequired } - const maxBitmapID = 1000 - const maxProfileID = 100000 + const maxRowID = 1000 + const maxColumnID = 100000 startTime := time.Now() // Execute operation continuously. for i := 0; i < cmd.N; i++ { - bitmapID := rand.Intn(maxBitmapID) - profileID := rand.Intn(maxProfileID) + rowID := rand.Intn(maxRowID) + columnID := rand.Intn(maxColumnID) - q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID) + q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID) if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil { return err diff --git a/ctl/import.go b/ctl/import.go index df8fb00eb..1ba8677cb 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -123,19 +123,19 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { var bit pilosa.Bit - // Parse bitmap id. - bitmapID, err := strconv.ParseUint(record[0], 10, 64) + // Parse row id. + rowID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { - return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0]) + return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0]) } - bit.BitmapID = bitmapID + bit.RowID = rowID - // Parse bitmap id. - profileID, err := strconv.ParseUint(record[1], 10, 64) + // Parse column id. + columnID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { - return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1]) + return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - bit.ProfileID = profileID + bit.ColumnID = columnID // Parse time, if exists. if len(record) > 2 && record[2] != "" { diff --git a/ctl/sort.go b/ctl/sort.go index 9d4b14277..5590d2538 100644 --- a/ctl/sort.go +++ b/ctl/sort.go @@ -45,7 +45,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { r.FieldsPerRecord = -1 a := make([]pilosa.Bit, 0, 1000000) for { - bitmapID, profileID, timestamp, err := readCSVRow(r) + rowID, columnID, timestamp, err := readCSVRow(r) if err == io.EOF { break } else if err == errBlank { @@ -53,7 +53,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { } else if err != nil { return err } - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) + a = append(a, pilosa.Bit{RowID: rowID, ColumnID: columnID, Timestamp: timestamp}) } // Sort bits by position. @@ -65,10 +65,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error { for _, bit := range a { // Write CSV to buffer. buf = buf[:0] - buf = strconv.AppendUint(buf, bit.BitmapID, 10) + buf = strconv.AppendUint(buf, bit.RowID, 10) buf = append(buf, ',') - buf = strconv.AppendUint(buf, bit.ProfileID, 10) + buf = strconv.AppendUint(buf, bit.ColumnID, 10) if bit.Timestamp != 0 { buf = append(buf, ',') @@ -91,8 +91,8 @@ func (cmd *SortCommand) Run(ctx context.Context) error { return nil } -// readCSVRow reads a bitmap/profile pair from a CSV row. -func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { +// readCSVRow reads a row/column pair from a CSV row. +func readCSVRow(r *csv.Reader) (rowID, columnID uint64, timestamp int64, err error) { // Read CSV row. record, err := r.Read() if err != nil { @@ -106,16 +106,16 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) } - // Parse bitmap id. - bitmapID, err = strconv.ParseUint(record[0], 10, 64) + // Parse row id. + rowID, err = strconv.ParseUint(record[0], 10, 64) if err != nil { - return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) + return 0, 0, 0, fmt.Errorf("invalid row id: %q", record[0]) } - // Parse bitmap id. - profileID, err = strconv.ParseUint(record[1], 10, 64) + // Parse column id. + columnID, err = strconv.ParseUint(record[1], 10, 64) if err != nil { - return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) + return 0, 0, 0, fmt.Errorf("invalid column id: %q", record[1]) } // Parse timestamp, if available. @@ -127,7 +127,7 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err timestamp = t.UnixNano() } - return bitmapID, profileID, timestamp, nil + return rowID, columnID, timestamp, nil } // errBlank indicates a blank row in a CSV file. diff --git a/db.go b/db.go index 8bf33300e..049e81261 100644 --- a/db.go +++ b/db.go @@ -17,7 +17,7 @@ import ( // Default database settings. const ( - DefaultColumnLabel = "profileID" + DefaultColumnLabel = "columnID" ) // DB represents a container for frames. @@ -40,8 +40,8 @@ type DB struct { remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - // Profile attribute storage and cache - profileAttrStore *AttrStore + // Column attribute storage and cache + columnAttrStore *AttrStore broadcaster Broadcaster stats StatsClient @@ -64,7 +64,7 @@ func NewDB(path, name string) (*DB, error) { remoteMaxSlice: 0, remoteMaxInverseSlice: 0, - profileAttrStore: NewAttrStore(filepath.Join(path, ".data")), + columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), columnLabel: DefaultColumnLabel, @@ -79,8 +79,8 @@ func (db *DB) Name() string { return db.name } // Path returns the path the database was initialized with. func (db *DB) Path() string { return db.path } -// ProfileAttrStore returns the storage for profile attributes. -func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore } +// ColumnAttrStore returns the storage for column attributes. +func (db *DB) ColumnAttrStore() *AttrStore { return db.columnAttrStore } // SetColumnLabel sets the column label. Persists to meta file on update. func (db *DB) SetColumnLabel(v string) error { @@ -131,7 +131,7 @@ func (db *DB) Open() error { return err } - if err := db.profileAttrStore.Open(); err != nil { + if err := db.columnAttrStore.Open(); err != nil { return err } @@ -220,8 +220,8 @@ func (db *DB) Close() error { defer db.mu.Unlock() // Close the attribute store. - if db.profileAttrStore != nil { - db.profileAttrStore.Close() + if db.columnAttrStore != nil { + db.columnAttrStore.Close() } // Close all frames. @@ -560,6 +560,6 @@ type importKey struct { } type importData struct { - BitmapIDs []uint64 - ProfileIDs []uint64 + RowIDs []uint64 + ColumnIDs []uint64 } diff --git a/executor.go b/executor.go index 9c156ec56..6bf3395e6 100644 --- a/executor.go +++ b/executor.go @@ -21,7 +21,7 @@ const ( DefaultFrame = "general" // MinThreshold is the lowest count to use in a Top-N operation when - // looking for additional bitmap/count pairs. + // looking for additional id/count pairs. MinThreshold = 1 ) @@ -71,8 +71,8 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices } // Optimize handling for bulk attribute insertion. - if hasOnlySetBitmapAttrs(q.Calls) { - return e.executeBulkSetBitmapAttrs(ctx, db, q.Calls, opt) + if hasOnlySetRowAttrs(q.Calls) { + return e.executeBulkSetRowAttrs(ctx, db, q.Calls, opt) } // Execute each call serially. @@ -102,10 +102,10 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic return e.executeCount(ctx, db, c, slices, opt) case "SetBit": return e.executeSetBit(ctx, db, c, opt) - case "SetBitmapAttrs": - return nil, e.executeSetBitmapAttrs(ctx, db, c, opt) - case "SetProfileAttrs": - return nil, e.executeSetProfileAttrs(ctx, db, c, opt) + case "SetRowAttrs": + return nil, e.executeSetRowAttrs(ctx, db, c, opt) + case "SetColumnAttrs": + return nil, e.executeSetColumnAttrs(ctx, db, c, opt) case "TopN": return e.executeTopN(ctx, db, c, slices, opt) default: @@ -155,7 +155,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call } // Attach attributes for Bitmap() calls. - // If the column label is used then return profile attributes. + // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { @@ -164,7 +164,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call if d != nil { columnLabel := d.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { - attrs, err := d.ProfileAttrStore().Attrs(columnID) + attrs, err := d.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call if err != nil { return nil, err } - attrs, err := fr.BitmapAttrStore().Attrs(rowID) + attrs, err := fr.RowAttrStore().Attrs(rowID) if err != nil { return nil, err } @@ -214,7 +214,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - bitmapIDs, _, err := c.UintSliceArg("ids") + rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } @@ -231,7 +231,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote { + if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote { return pairs, nil } // Only the original caller should refetch the full counts. @@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, return nil, fmt.Errorf("executeTopNSlice: %v", err) } field, _ := c.Args["field"].(string) - bitmapIDs, _, err := c.UintSliceArg("ids") + rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } @@ -330,7 +330,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, return f.Top(TopOptions{ N: int(n), Src: src, - BitmapIDs: bitmapIDs, + RowIDs: rowIDs, FilterField: field, FilterValues: filters, MinThreshold: minThreshold, @@ -404,7 +404,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal if frag == nil { return NewBitmap(), nil } - return frag.Bitmap(id), nil + return frag.Row(id), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -483,7 +483,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call if f == nil { continue } - bm = bm.Union(f.Bitmap(rowID)) + bm = bm.Union(f.Row(rowID)) } return bm, nil } @@ -739,11 +739,11 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call return ret, nil } -// executeSetBitmapAttrs executes a SetBitmapAttrs() call. -func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { +// executeSetRowAttrs executes a SetRowAttrs() call. +func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { frameName, ok := c.Args["frame"].(string) if !ok { - return errors.New("SetBitmapAttrs() frame required") + return errors.New("SetRowAttrs() frame required") } // Retrieve frame. @@ -756,9 +756,9 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. // Parse labels. rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return fmt.Errorf("reading SetBitmapAttrs() row: %v", err) + return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetBitmapAttrs() row field '%v' required.", rowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -767,7 +767,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. delete(attrs, rowLabel) // Set attributes. - if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil { + if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } @@ -796,14 +796,14 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. return nil } -// executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls. -func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { +// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. +func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { // Collect attributes by frame/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { frame, ok := c.Args["frame"].(string) if !ok { - return nil, errors.New("SetBitmapAttrs() frame required") + return nil, errors.New("SetRowAttrs() frame required") } // Retrieve frame. @@ -815,9 +815,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return nil, fmt.Errorf("reading SetBitmapAttrs() row: %v", rowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { - return nil, fmt.Errorf("SetBitmapAttrs row field '%v' required.", rowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -852,7 +852,7 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal } // Set attributes. - if err := frame.BitmapAttrStore().SetBulkAttrs(frameMap); err != nil { + if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { return nil, err } } @@ -883,8 +883,8 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal return make([]interface{}, len(calls)), nil } -// executeSetProfileAttrs executes a SetProfileAttrs() call. -func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { +// executeSetColumnAttrs executes a SetColumnAttrs() call. +func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { // Retrieve database. d := e.Index.DB(db) if d == nil { @@ -898,7 +898,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql columnLabel := d.columnLabel col, okCol, errCol := c.UintArg(columnLabel) if errCol != nil || !okCol { - return fmt.Errorf("reading SetProfileAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) + return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) } id = col colName = columnLabel @@ -911,7 +911,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql delete(attrs, colName) // Set attributes. - if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil { + if err := d.ColumnAttrStore().SetAttrs(id, attrs); err != nil { return err } @@ -1011,8 +1011,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query v, err = pb.Results[i].Changed, nil case "ClearBit": v, err = pb.Results[i].Changed, nil - case "SetBitmapAttrs": - case "SetProfileAttrs": + case "SetRowAttrs": + case "SetColumnAttrs": default: v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil } @@ -1207,14 +1207,14 @@ func decodeError(s string) error { return errors.New(s) } -// hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls. -func hasOnlySetBitmapAttrs(calls []*pql.Call) bool { +// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls. +func hasOnlySetRowAttrs(calls []*pql.Call) bool { if len(calls) == 0 { return false } for _, call := range calls { - if call.Name != "SetBitmapAttrs" { + if call.Name != "SetRowAttrs" { return false } } @@ -1227,7 +1227,7 @@ func needsSlices(calls []*pql.Call) bool { } for _, call := range calls { switch call.Name { - case "ClearBit", "SetBit", "SetBitmapAttrs", "SetProfileAttrs": + case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs": continue case "Count", "TopN": return true diff --git a/executor_test.go b/executor_test.go index 7a0c95622..97a89f3ae 100644 --- a/executor_test.go +++ b/executor_test.go @@ -27,13 +27,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } - if err := f.BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } @@ -58,17 +58,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } - if err := db.ProfileAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := db.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(profileID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -195,11 +195,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) - if n := f.Bitmap(11).Count(); n != 0 { + if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -207,10 +207,10 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := f.Bitmap(11).Count(); n != 1 { + if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -219,8 +219,8 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } -// Ensure a SetBitmapAttrs() query can be executed. -func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { +// Ensure a SetRowAttrs() query can be executed. +func TestExecutor_Execute_SetRowAttrs(t *testing.T) { idx := MustOpenIndex() defer idx.Close() @@ -235,21 +235,21 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(idx.Index, NewCluster(1)) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } f := idx.Frame("d", "f") - if m, err := f.BitmapAttrStore().Attrs(10); err != nil { + if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { t.Fatalf("unexpected bitmap attr: %#v", m) @@ -261,7 +261,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -287,7 +287,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) @@ -345,7 +345,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -355,7 +355,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) - // Create an intersecting bitmap. + // Create an intersecting row. idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) @@ -382,7 +382,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) @@ -405,7 +405,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) @@ -445,7 +445,7 @@ func TestExecutor_Execute_Range(t *testing.T) { f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late - f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap + f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { @@ -542,7 +542,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `SetBit(frame="f", id=10, profileID=2)` { + } else if query.String() != `SetBit(columnID=2, frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -559,12 +559,12 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := NewExecutor(idx.Index, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -587,7 +587,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -606,12 +606,12 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := NewExecutor(idx.Index, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { diff --git a/fragment.go b/fragment.go index 9bee4288e..9f68df30e 100644 --- a/fragment.go +++ b/fragment.go @@ -29,7 +29,7 @@ import ( ) const ( - // SliceWidth is the number of profile IDs in a slice. + // SliceWidth is the number of column IDs in a slice. SliceWidth = 1048576 // SnapshotExt is the file extension used for an in-process snapshot. @@ -41,7 +41,7 @@ const ( // CacheExt is the file extension for persisted cache ids. CacheExt = ".cache" - // HashBlockSize is the number of bitmaps in a merkle hash block. + // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 ) @@ -67,13 +67,13 @@ type Fragment struct { storageData []byte opN int // number of ops since snapshot - // Cache for bitmap counts. + // Cache for row counts. cacheType string // passed in by frame cache Cache cacheSize uint32 - // Cache containing full bitmaps (not just counts). - bitmapCache BitmapCache + // Cache containing full rows (not just counts). + rowCache BitmapCache // Cached checksums for each block. checksums map[int][]byte @@ -86,9 +86,9 @@ type Fragment struct { // Writer used for out-of-band log entries. LogOutput io.Writer - // Bitmap attribute storage. + // Row attribute storage. // This is set by the parent frame unless overridden for testing. - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore stats StatsClient } @@ -144,7 +144,7 @@ func (f *Fragment) Open() error { return err } - // Fill cache with bitmaps persisted to disk. + // Fill cache with rows persisted to disk. if err := f.openCache(); err != nil { return err } @@ -213,13 +213,13 @@ func (f *Fragment) openStorage() error { // Attach the file to the bitmap to act as a write-ahead log. f.storage.OpWriter = f.file - f.bitmapCache = &SimpleCache{make(map[uint64]*Bitmap)} + f.rowCache = &SimpleCache{make(map[uint64]*Bitmap)} return nil } -// openCache initializes the cache from bitmap ids persisted to disk. +// openCache initializes the cache from row ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { @@ -247,12 +247,12 @@ func (f *Fragment) openCache() error { return nil } - // Read in all bitmaps by ID. + // Read in all rows by ID. // This will cause them to be added to the cache. - for _, bitmapID := range pb.BitmapIDs { - //n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) - n := f.bitmap(bitmapID, true, true).Count() - f.cache.BulkAdd(bitmapID, n) + for _, id := range pb.IDs { + //n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) + n := f.row(id, true, true).Count() + f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -314,17 +314,16 @@ func (f *Fragment) closeStorage() error { // logger returns a logger instance for the fragment.nt. func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) } -// Bitmap returns a bitmap by ID. -func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { +// Row returns a row by ID. +func (f *Fragment) Row(rowID uint64) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - return f.bitmap(bitmapID, true, true) + return f.row(rowID, true, true) } -func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCache bool) *Bitmap { - - if checkBitmapCache { - r, ok := f.bitmapCache.Fetch(bitmapID) +func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Bitmap { + if checkRowCache { + r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { return r } @@ -332,11 +331,11 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by - data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) + data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth) // Reference bitmap subrange in storage. // We Clone() data because otherwise bm will contains pointers to containers in storage. - // This causes unexpected results when we cache the bitmap and try to use it later. + // This causes unexpected results when we cache the row and try to use it later. bm := &Bitmap{ segments: []BitmapSegment{{ data: *data.Clone(), @@ -346,25 +345,25 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa } bm.InvalidateCount() - if updateBitmapCache { - f.bitmapCache.Add(bitmapID, bm) + if updateRowCache { + f.rowCache.Add(rowID, bm) } return bm } -// SetBit sets a bit for a given profile & bitmap within the fragment. +// SetBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(bitmapID, profileID) + return f.setBit(rowID, columnID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return false, err } @@ -380,37 +379,37 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { return false, err } - // Get the bitmap from bitmapCache or fragment.storage. - bm := f.bitmap(bitmapID, true, true) - bm.SetBit(profileID) + // Get the row from row cache or fragment.storage. + bm := f.row(rowID, true, true) + bm.SetBit(columnID) // Update the cache. - f.cache.Add(bitmapID, bm.Count()) + f.cache.Add(rowID, bm.Count()) f.stats.Count("setN", 1) return changed, nil } -// ClearBit clears a bit for a given profile & bitmap within the fragment. +// ClearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { +func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(bitmapID, profileID) + return f.clearBit(rowID, columnID) } -func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return false, err } @@ -426,38 +425,38 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { return false, err } - // Get the bitmap from bitmapCache or fragment.storage. - bm := f.bitmap(bitmapID, true, true) - bm.ClearBit(profileID) + // Get the row from cache or fragment.storage. + bm := f.row(rowID, true, true) + bm.ClearBit(columnID) // Update the cache. - f.cache.Add(bitmapID, bm.Count()) + f.cache.Add(rowID, bm.Count()) f.stats.Count("clearN", 1) return changed, nil } -// pos translates the bitmap ID and profile ID into a position in the storage bitmap. -func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) { - // Return an error if the profile ID is out of the range of the fragment's slice. - minProfileID := f.slice * SliceWidth - if profileID < minProfileID || profileID >= minProfileID+SliceWidth { - return 0, errors.New("profile out of bounds") +// pos translates the row ID and column ID into a position in the storage bitmap. +func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { + // Return an error if the column ID is out of the range of the fragment's slice. + minColumnID := f.slice * SliceWidth + if columnID < minColumnID || columnID >= minColumnID+SliceWidth { + return 0, errors.New("column out of bounds") } - return Pos(bitmapID, profileID), nil + return Pos(rowID, columnID), nil } // ForEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error { +func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -474,15 +473,15 @@ func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error { return err } -// Top returns the top bitmaps from the fragment. -// If opt.Src is specified then only bitmaps which intersect src are returned. -// If opt.FilterValues exist then the bitmap attribute specified by field is matched. +// Top returns the top rows from the fragment. +// If opt.Src is specified then only rows which intersect src are returned. +// If opt.FilterValues exist then the row attribute specified by field is matched. func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { - // Retrieve pairs. If no bitmap ids specified then return from cache. - pairs := f.topBitmapPairs(opt.BitmapIDs) + // Retrieve pairs. If no row ids specified then return from cache. + pairs := f.topBitmapPairs(opt.RowIDs) - // If BitmapIDs are provided, we don't want to truncate the result set - if len(opt.BitmapIDs) > 0 { + // If row ids are provided, we don't want to truncate the result set + if len(opt.RowIDs) > 0 { opt.N = 0 } @@ -509,9 +508,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Iterate over rankings and add to results until we have enough. results := &PairHeap{} for _, pair := range pairs { - bitmapID, cnt := pair.ID, pair.Count + rowID, cnt := pair.ID, pair.Count - // Ignore empty bitmaps. + // Ignore empty rows. if cnt <= 0 { continue } @@ -531,7 +530,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Apply filter, if set. if filters != nil { - attr, err := f.BitmapAttrStore.Attrs(bitmapID) + attr, err := f.RowAttrStore.Attrs(rowID) if err != nil { return nil, err } else if attr == nil { @@ -548,7 +547,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Bitmap(bitmapID)) + count = opt.Src.IntersectionCount(f.Row(rowID)) } if count == 0 { continue @@ -566,7 +565,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } } - heap.Push(results, Pair{ID: bitmapID, Count: count}) + heap.Push(results, Pair{ID: rowID, Count: count}) // If we reach the requested number of pairs and we are not computing // intersections then simply exit. If we are intersecting then sort @@ -584,20 +583,20 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If it's too low then don't try finding anymore pairs. threshold := results.Pairs[0].Count - // If the bitmap doesn't have enough bits set before the intersection - // then we can assume that any remaining bitmaps also have a count too low. + // If the row doesn't have enough bits set before the intersection + // then we can assume that any remaining rows also have a count too low. if threshold < opt.MinThreshold || cnt < threshold { break } // Calculate the intersecting bit count and skip if it's below our - // last bitmap in our current result set. - count := opt.Src.IntersectionCount(f.Bitmap(bitmapID)) + // last row in our current result set. + count := opt.Src.IntersectionCount(f.Row(rowID)) if count < threshold { continue } - heap.Push(results, Pair{ID: bitmapID, Count: count}) + heap.Push(results, Pair{ID: rowID, Count: count}) } //Pop first opt.N elements out of heap @@ -611,32 +610,32 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return r, nil } -func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { - // If no specific bitmaps are requested, retrieve top bitmaps. - if len(bitmapIDs) == 0 { +func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { + // If no specific rows are requested, retrieve top rows. + if len(rowIDs) == 0 { f.mu.Lock() defer f.mu.Unlock() f.cache.Invalidate() return f.cache.Top() } - // Otherwise retrieve specific bitmaps. - pairs := make([]BitmapPair, 0, len(bitmapIDs)) - for _, bitmapID := range bitmapIDs { + // Otherwise retrieve specific rows. + pairs := make([]BitmapPair, 0, len(rowIDs)) + for _, rowID := range rowIDs { // Look up cache first, if available. - if n := f.cache.Get(bitmapID); n > 0 { + if n := f.cache.Get(rowID); n > 0 { pairs = append(pairs, BitmapPair{ - ID: bitmapID, + ID: rowID, Count: n, }) continue } - bm := f.Bitmap(bitmapID) + bm := f.Row(rowID) if bm.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, BitmapPair{ - ID: bitmapID, + ID: rowID, Count: bm.Count(), }) } @@ -647,14 +646,14 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { // TopOptions represents options passed into the Top() function. type TopOptions struct { - // Number of bitmaps to return. + // Number of rows to return. N int // Bitmap to intersect with. Src *Bitmap - // Specific bitmaps to filter against. - BitmapIDs []uint64 + // Specific rows to filter against. + RowIDs []uint64 MinThreshold uint64 // Filter field name & values. @@ -768,14 +767,14 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as bitmap & profile ID pairs. -func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) { +// BlockData returns bits in a block as row & column ID pairs. +func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) { - bitmapIDs = append(bitmapIDs, i/SliceWidth) - profileIDs = append(profileIDs, i%SliceWidth) + rowIDs = append(rowIDs, i/SliceWidth) + columnIDs = append(columnIDs, i%SliceWidth) }) return } @@ -789,8 +788,8 @@ func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) { func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].BitmapIDs) != len(data[i].ProfileIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].BitmapIDs), len(data[i].ProfileIDs)) + if len(data[i].RowIDs) != len(data[i].ColumnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) } } @@ -801,22 +800,22 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e sets = make([]PairSet, len(data)+1) clears = make([]PairSet, len(data)+1) - // Limit upper bitmap/profile pair. - maxBitmapID := uint64(id+1) * HashBlockSize - maxProfileID := uint64(SliceWidth) + // Limit upper row/column pair. + maxRowID := uint64(id+1) * HashBlockSize + maxColumnID := uint64(SliceWidth) // Create buffered iterator for local block. itrs := make([]*BufIterator, 1, len(data)+1) itrs[0] = NewBufIterator( NewLimitIterator( - NewRoaringIterator(f.storage.Iterator()), maxBitmapID, maxProfileID, + NewRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID, ), ) // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].BitmapIDs, data[i].ProfileIDs) - itr = NewLimitIterator(itr, maxBitmapID, maxProfileID) + var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) + itr = NewLimitIterator(itr, maxRowID, maxColumnID) itrs = append(itrs, NewBufIterator(itr)) } @@ -833,8 +832,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e values := make([]bool, len(itrs)) for { var min struct { - bitmapID uint64 - profileID uint64 + rowID uint64 + columnID uint64 } // Find the lowest pair. @@ -844,9 +843,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e if eof { // no more data continue } else if !hasData { // first pair - min.bitmapID, min.profileID, hasData = bid, pid, true - } else if bid < min.bitmapID || (bid == min.bitmapID && pid < min.profileID) { // lower pair - min.bitmapID, min.profileID = bid, pid + min.rowID, min.columnID, hasData = bid, pid, true + } else if bid < min.rowID || (bid == min.rowID && pid < min.columnID) { // lower pair + min.rowID, min.columnID = bid, pid } } @@ -860,7 +859,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e for i, itr := range itrs { bid, pid, eof := itr.Next() - values[i] = !eof && bid == min.bitmapID && pid == min.profileID + values[i] = !eof && bid == min.rowID && pid == min.columnID if values[i] { setN++ // set } else { @@ -880,25 +879,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID) - sets[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID) + sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) + sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) } else { - clears[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID) - clears[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID) + clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) + clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ProfileIDs { - if _, err := f.setBit(sets[0].BitmapIDs[i], (f.Slice()*SliceWidth)+sets[0].ProfileIDs[i]); err != nil { + for i := range sets[0].ColumnIDs { + if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { return nil, nil, err } } // Clear local bits. - for i := range clears[0].ProfileIDs { - if _, err := f.clearBit(clears[0].BitmapIDs[i], (f.Slice()*SliceWidth)+clears[0].ProfileIDs[i]); err != nil { + for i := range clears[0].ColumnIDs { + if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { return nil, nil, err } } @@ -908,12 +907,12 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Import bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { +func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() - // Verify that there are an equal number of bitmap ids and profile ids. - if len(bitmapIDs) != len(profileIDs) { - return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs)) + // Verify that there are an equal number of row ids and column ids. + if len(rowIDs) != len(columnIDs) { + return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } // Disconnect op writer so we don't append updates. @@ -924,11 +923,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { lastID := uint64(0) if err := func() error { set := make(map[uint64]struct{}) - for i := range bitmapIDs { - bitmapID, profileID := bitmapIDs[i], profileIDs[i] + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return err } @@ -942,21 +941,21 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // import optimization to avoid linear foreach calls // slight risk of concurrent cache counter being off but // no real danger - if i == 0 || bitmapID != lastID { - lastID = bitmapID - set[bitmapID] = struct{}{} + if i == 0 || rowID != lastID { + lastID = rowID + set[rowID] = struct{}{} } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) } - // Update cache counts for all bitmaps. - for bitmapID := range set { - // Import should ALWAYS have bitmap() load a new bm from fragment.storage - // because the bitmap that's in bitmapCache hasn't been updated with + // Update cache counts for all rows. + for rowID := range set { + // Import should ALWAYS have row() load a new bm from fragment.storage + // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false, false).Count()) + f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) } f.cache.Invalidate() @@ -995,6 +994,7 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } + func track(start time.Time, name string, logger *log.Logger) { elapsed := time.Since(start) logger.Printf("%s took %s", name, elapsed) @@ -1061,13 +1061,11 @@ func (f *Fragment) flushCache() error { return nil } - // Retrieve a list of bitmap ids from the cache. - bitmapIDs := f.cache.BitmapIDs() + // Retrieve a list of row ids from the cache. + ids := f.cache.IDs() // Marshal cache data to bytes. - buf, err := proto.Marshal(&internal.Cache{ - BitmapIDs: bitmapIDs, - }) + buf, err := proto.Marshal(&internal.Cache{IDs: ids}) if err != nil { return err } @@ -1253,7 +1251,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error { return nil } -// FragmentBlock represents info about a subsection of the bitmaps in a block. +// FragmentBlock represents info about a subsection of the rows in a block. // This is used for comparing data in remote blocks for active anti-entropy. type FragmentBlock struct { ID int `json:"id"` @@ -1386,7 +1384,7 @@ func (s *FragmentSyncer) SyncFragment() error { return nil } -// syncBlock sends and receives all bitmaps for a given block. +// syncBlock sends and receives all rows for a given block. // Returns an error if any remote hosts are unreachable. func (s *FragmentSyncer) syncBlock(id int) error { f := s.Fragment @@ -1411,14 +1409,14 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) if err != nil { return err } pairSets = append(pairSets, PairSet{ - ProfileIDs: profileIDs, - BitmapIDs: bitmapIDs, + ColumnIDs: columnIDs, + RowIDs: rowIDs, }) } @@ -1438,7 +1436,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { set, clear := sets[i], clears[i] // Ignore if there are no differences. - if len(set.ProfileIDs) == 0 && len(clear.ProfileIDs) == 0 { + if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { continue } @@ -1446,11 +1444,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { var buf bytes.Buffer // Only sync the standard block. - for j := 0; j < len(set.ProfileIDs); j++ { - fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j]) + for j := 0; j < len(set.ColumnIDs); j++ { + fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) } - for j := 0; j < len(clear.ProfileIDs); j++ { - fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), clear.BitmapIDs[j], (f.Slice()*SliceWidth)+clear.ProfileIDs[j]) + for j := 0; j < len(clear.ColumnIDs); j++ { + fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) } // Verify sync is not prematurely closing. @@ -1476,10 +1474,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length bitmap and profile id lists. +// PairSet is a list of equal length row and column id lists. type PairSet struct { - BitmapIDs []uint64 - ProfileIDs []uint64 + RowIDs []uint64 + ColumnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1496,7 +1494,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the bitmap position of a bitmap/profile pair. -func Pos(bitmapID, profileID uint64) uint64 { - return (bitmapID * SliceWidth) + (profileID % SliceWidth) +// Pos returns the row position of a row/column pair. +func Pos(rowID, columnID uint64) uint64 { + return (rowID * SliceWidth) + (columnID % SliceWidth) } diff --git a/fragment_test.go b/fragment_test.go index f54f5415b..fd5dc95ac 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -35,19 +35,19 @@ func TestFragment_SetBit(t *testing.T) { t.Fatal(err) } - // Verify counts on bitmaps. - if n := f.Bitmap(120).Count(); n != 2 { + // Verify counts on rows. + if n := f.Row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Bitmap(121).Count(); n != 1 { + } else if n := f.Row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(120).Count(); n != 2 { + } else if n := f.Row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Bitmap(121).Count(); n != 1 { + } else if n := f.Row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -66,15 +66,15 @@ func TestFragment_ClearBit(t *testing.T) { t.Fatal(err) } - // Verify count on bitmap. - if n := f.Bitmap(1000).Count(); n != 1 { + // Verify count on row. + if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -96,14 +96,14 @@ func TestFragment_Snapshot(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -124,8 +124,8 @@ func TestFragment_ForEachBit(t *testing.T) { // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(bitmapID, profileID uint64) error { - result = append(result, [2]uint64{bitmapID, profileID}) + if err := f.ForEachBit(func(rowID, columnID uint64) error { + result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { t.Fatal(err) @@ -142,12 +142,12 @@ func TestFragment_Top(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { @@ -159,21 +159,21 @@ func TestFragment_Top(t *testing.T) { } } -// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps. +// Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) // Assign attributes. - f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) - f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) + f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) + f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{ N: 2, FilterField: "x", @@ -189,21 +189,21 @@ func TestFragment_Top_Filter(t *testing.T) { } } -// Ensure a fragment can return top bitmaps that intersect with an input bitmap. +// Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Create an intersecting input bitmap. + // Create an intersecting input row. src := pilosa.NewBitmap(1, 2, 3) - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 10, 11, 12) // one intersection f.MustSetBits(101, 1, 2, 3, 4) // three intersections f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections f.MustSetBits(103, 1000, 1001, 1002) // no intersection - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ @@ -215,7 +215,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { } } -// Ensure a fragment can return top bitmaps that have many bits set. +// Ensure a fragment can return top rows that have many bits set. func TestFragment_TopN_Intersect_Large(t *testing.T) { if testing.Short() { t.Skip("short mode") @@ -224,20 +224,20 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Create an intersecting input bitmap. + // Create an intersecting input row. src := pilosa.NewBitmap( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) - // Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts. + // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { f.MustSetBits(i, j) } } - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ @@ -256,18 +256,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } } -// Ensure a fragment can return top bitmaps when specified by ID. -func TestFragment_TopN_BitmapIDs(t *testing.T) { +// Ensure a fragment can return top rows when specified by ID. +func TestFragment_TopN_IDs(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) - // Retrieve top bitmaps. - if pairs, err := f.Top(pilosa.TopOptions{BitmapIDs: []uint64{100, 101, 200}}); err != nil { + // Retrieve top rows. + if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ {ID: 101, Count: 4}, @@ -307,16 +307,16 @@ func TestFragment_TopN_CacheSize(t *testing.T) { frag.Close() f := &Fragment{ - Fragment: frag, - BitmapAttrStore: MustOpenAttrStore(), + Fragment: frag, + RowAttrStore: MustOpenAttrStore(), } - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { panic(err) } defer f.Close() - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) @@ -332,7 +332,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { {ID: 102, Count: 5}, } - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { @@ -381,7 +381,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different bitmap. + // Set bit on different row. if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } @@ -391,7 +391,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different profile. + // Set bit on different column. if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } @@ -544,7 +544,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected bits: %+v", a) } @@ -553,40 +553,11 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.Cache().Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected bits (reopen): %+v", a) } } -/* -func BenchmarkFragment_BlockChecksum_Fill1(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.01) } -func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.10) } -func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) } - -func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) - defer f.Close() - - // Fill fragment. - bitmapIDs, profileIDs := GenerateImportFill(pilosa.HashBlockSize, fillPercent) - if err := f.Import(bitmapIDs, profileIDs); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.ReportAllocs() - - // Calculate block checksum. - for i := 0; i < b.N; i++ { - f.InvalidateChecksums() - - if chksum := f.BlockChecksum(0); chksum == nil { - b.Fatal("expected checksum") - } - } -} -*/ - func BenchmarkFragment_Blocks(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -633,7 +604,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Bitmap(1).IntersectionCount(f.Bitmap(2)); n == 0 { + if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } @@ -642,7 +613,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { *pilosa.Fragment - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore } // NewFragment returns a new instance of Fragment with a temporary path. @@ -654,10 +625,10 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), - BitmapAttrStore: MustOpenAttrStore(), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + RowAttrStore: MustOpenAttrStore(), } - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore return f } @@ -674,7 +645,7 @@ func MustOpenFragment(db, frame, view string, slice uint64) *Fragment { func (f *Fragment) Close() error { defer os.Remove(f.Path()) defer os.Remove(f.CachePath()) - defer f.BitmapAttrStore.Close() + defer f.RowAttrStore.Close() return f.Fragment.Close() } @@ -686,64 +657,64 @@ func (f *Fragment) Reopen() error { } f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { return err } return nil } -// MustSetBits sets bits on a bitmap. Panic on error. +// MustSetBits sets bits on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID); err != nil { +func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.SetBit(rowID, columnID); err != nil { panic(err) } } } -// MustClearBits clears bits on a bitmap. Panic on error. -func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := f.ClearBit(bitmapID, profileID); err != nil { +// MustClearBits clears bits on a row. Panic on error. +func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.ClearBit(rowID, columnID); err != nil { panic(err) } } } -// BitmapAttrStore provides simple storage for attributes. -type BitmapAttrStore struct { +// RowAttrStore provides simple storage for attributes. +type RowAttrStore struct { attrs map[uint64]map[string]interface{} } -// NewBitmapAttrStore returns a new instance of BitmapAttrStore. -func NewBitmapAttrStore() *BitmapAttrStore { - return &BitmapAttrStore{ +// NewRowAttrStore returns a new instance of RowAttrStore. +func NewRowAttrStore() *RowAttrStore { + return &RowAttrStore{ attrs: make(map[uint64]map[string]interface{}), } } -// BitmapAttrs returns the attributes set to a bitmap id. -func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) { +// RowAttrs returns the attributes set to a row id. +func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { return s.attrs[id], nil } -// SetBitmapAttrs assigns a set of attributes to a bitmap id. -func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) { +// SetRowAttrs assigns a set of attributes to a row id. +func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { s.attrs[id] = m } // GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. -func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint64) { +func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { ipct := int(pct * 100) - for i := 0; i < SliceWidth*bitmapN; i++ { + for i := 0; i < SliceWidth*rowN; i++ { if i%100 >= ipct { continue } - bitmapIDs = append(bitmapIDs, uint64(i%SliceWidth)) - profileIDs = append(profileIDs, uint64(i/SliceWidth)) + rowIDs = append(rowIDs, uint64(i%SliceWidth)) + columnIDs = append(columnIDs, uint64(i/SliceWidth)) } return } @@ -754,7 +725,7 @@ func TestFragment_Tanimoto(t *testing.T) { src := pilosa.NewBitmap(1, 2, 3) - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) @@ -776,7 +747,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { src := pilosa.NewBitmap(1, 2, 3) - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) diff --git a/frame.go b/frame.go index 684b2225d..df4c8182f 100644 --- a/frame.go +++ b/frame.go @@ -35,8 +35,8 @@ type Frame struct { views map[string]*View - // Bitmap attribute storage and cache - bitmapAttrStore *AttrStore + // Row attribute storage and cache + rowAttrStore *AttrStore broadcaster Broadcaster stats StatsClient @@ -64,8 +64,8 @@ func NewFrame(path, db, name string) (*Frame, error) { db: db, name: name, - views: make(map[string]*View), - bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), + views: make(map[string]*View), + rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), stats: NopStatsClient, @@ -87,8 +87,8 @@ func (f *Frame) DB() string { return f.db } // Path returns the path the frame was initialized with. func (f *Frame) Path() string { return f.path } -// BitmapAttrStore returns the attribute storage. -func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore } +// RowAttrStore returns the attribute storage. +func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. func (f *Frame) MaxSlice() uint64 { @@ -215,7 +215,7 @@ func (f *Frame) Open() error { return err } - if err := f.bitmapAttrStore.Open(); err != nil { + if err := f.rowAttrStore.Open(); err != nil { return err } @@ -253,7 +253,7 @@ func (f *Frame) openViews() error { if err := view.Open(); err != nil { return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) } - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view f.stats.Count("maxSlice", 1) @@ -326,8 +326,8 @@ func (f *Frame) Close() error { defer f.mu.Unlock() // Close the attribute store. - if f.bitmapAttrStore != nil { - _ = f.bitmapAttrStore.Close() + if f.rowAttrStore != nil { + _ = f.rowAttrStore.Close() } // Close all views. @@ -411,7 +411,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { if err := view.Open(); err != nil { return nil, err } - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view return view, nil @@ -421,7 +421,7 @@ func (f *Frame) newView(path, name string) *View { view := NewView(path, f.db, f.name, name, f.cacheSize) view.cacheType = f.cacheType view.LogOutput = f.LogOutput - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name)) return view } @@ -511,7 +511,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Import bulk imports data. -func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { +func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. q := f.TimeQuantum() if hasTime(timestamps) && q == "" { @@ -520,8 +520,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Split import data by fragment. dataByFragment := make(map[importKey]importData) - for i := range bitmapIDs { - bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + for i := range rowIDs { + rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i] var standard, inverse []string if timestamp == nil { @@ -534,20 +534,20 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Attach bit to each standard view. for _, name := range standard { - key := importKey{View: name, Slice: profileID / SliceWidth} + key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, bitmapID) - data.ProfileIDs = append(data.ProfileIDs, profileID) + data.RowIDs = append(data.RowIDs, rowID) + data.ColumnIDs = append(data.ColumnIDs, columnID) dataByFragment[key] = data } if f.inverseEnabled { // Attach reversed bits to each inverse view. for _, name := range inverse { - key := importKey{View: name, Slice: bitmapID / SliceWidth} + key := importKey{View: name, Slice: rowID / SliceWidth} data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed - data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed + data.RowIDs = append(data.RowIDs, columnID) // reversed + data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed dataByFragment[key] = data } } @@ -563,8 +563,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Re-sort data for inverse views. if IsInverseView(key.View) { sort.Sort(importBitSet{ - bitmapIDs: data.BitmapIDs, - profileIDs: data.ProfileIDs, + rowIDs: data.RowIDs, + columnIDs: data.ColumnIDs, }) } @@ -578,7 +578,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) return err } - if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { return err } } @@ -650,15 +650,15 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { - bitmapIDs, profileIDs []uint64 + rowIDs, columnIDs []uint64 } func (p importBitSet) Swap(i, j int) { - p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i] - p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i] + p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i] + p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i] } -func (p importBitSet) Len() int { return len(p.bitmapIDs) } -func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] } +func (p importBitSet) Len() int { return len(p.rowIDs) } +func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } // Cache types. const ( diff --git a/frame_test.go b/frame_test.go index 4814ca67e..0d10841db 100644 --- a/frame_test.go +++ b/frame_test.go @@ -119,8 +119,8 @@ func (f *Frame) Reopen() error { } // MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time) (changed bool) { - changed, err := f.SetBit(view, bitmapID, profileID, t) +func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { + changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { panic(err) } diff --git a/handler.go b/handler.go index ef5ad4f8d..7040a05ab 100644 --- a/handler.go +++ b/handler.go @@ -159,26 +159,26 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { results, err := h.Executor.Execute(r.Context(), dbName, q, req.Slices, opt) resp := &QueryResponse{Results: results, Err: err} - // Fill profile attributes if requested. - if req.Profiles { - // Consolidate all profile ids across all calls. - var profileIDs []uint64 + // Fill column attributes if requested. + if req.ColumnAttrs { + // Consolidate all column ids across all calls. + var columnIDs []uint64 for _, result := range results { bm, ok := result.(*Bitmap) if !ok { continue } - profileIDs = uint64Slice(profileIDs).merge(bm.Bits()) + columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) } - // Retrieve profile attributes across all calls. - profiles, err := h.readProfiles(h.Index.DB(dbName), profileIDs) + // Retrieve column attributes across all calls. + columnAttrSets, err := h.readColumnAttrSets(h.Index.DB(dbName), columnIDs) if err != nil { w.WriteHeader(http.StatusInternalServerError) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } - resp.Profiles = profiles + resp.ColumnAttrSets = columnAttrSets } // Set appropriate status code, if there is an error. @@ -437,7 +437,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { } // Retrieve local blocks. - blks, err := db.ProfileAttrStore().Blocks() + blks, err := db.ColumnAttrStore().Blocks() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -447,7 +447,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { attrs := make(map[uint64]map[string]interface{}) for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { // Retrieve block data. - m, err := db.ProfileAttrStore().BlockData(blockID) + m, err := db.ColumnAttrStore().BlockData(blockID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -701,7 +701,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request } // Retrieve local blocks. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -711,7 +711,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request attrs := make(map[uint64]map[string]interface{}) for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { // Retrieve block data. - m, err := f.BitmapAttrStore().BlockData(blockID) + m, err := f.RowAttrStore().BlockData(blockID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -739,24 +739,24 @@ type postFrameAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } -// readProfiles returns a list of profile objects by id. -func (h *Handler) readProfiles(db *DB, ids []uint64) ([]*Profile, error) { +// readColumnAttrSets returns a list of column attribute objects by id. +func (h *Handler) readColumnAttrSets(db *DB, ids []uint64) ([]*ColumnAttrSet, error) { if db == nil { return nil, nil } - a := make([]*Profile, 0, len(ids)) + a := make([]*ColumnAttrSet, 0, len(ids)) for _, id := range ids { - // Read attributes for profile. Skip profile if empty. - attrs, err := db.ProfileAttrStore().Attrs(id) + // Read attributes for column. Skip column if empty. + attrs, err := db.ColumnAttrStore().Attrs(id) if err != nil { return nil, err } else if len(attrs) == 0 { continue } - // Append profile with attributes. - a = append(a, &Profile{ID: id, Attrs: attrs}) + // Append column with attributes. + a = append(a, &ColumnAttrSet{ID: id, Attrs: attrs}) } return a, nil @@ -817,10 +817,10 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } return &QueryRequest{ - Query: query, - Slices: slices, - Profiles: q.Get("profiles") == "true", - Quantum: quantum, + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + Quantum: quantum, }, nil } @@ -907,9 +907,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Import into fragment. - err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps) + err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -965,10 +965,10 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { cw := csv.NewWriter(w) // Iterate over each bit. - if err := f.ForEachBit(func(bitmapID, profileID uint64) error { + if err := f.ForEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ - strconv.FormatUint(bitmapID, 10), - strconv.FormatUint(profileID, 10), + strconv.FormatUint(rowID, 10), + strconv.FormatUint(columnID, 10), }) }); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1083,7 +1083,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Read data var resp internal.BlockDataResponse if f != nil { - resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) } // Encode response. @@ -1271,8 +1271,8 @@ type QueryRequest struct { // If empty, all slices are included. Slices []uint64 - // Return profile attributes, if true. - Profiles bool + // Return column attributes, if true. + ColumnAttrs bool // Time granularity to use with the timestamp. Quantum TimeQuantum @@ -1284,11 +1284,11 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - Profiles: pb.Profiles, - Quantum: TimeQuantum(pb.Quantum), - Remote: pb.Remote, + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Quantum: TimeQuantum(pb.Quantum), + Remote: pb.Remote, } return req @@ -1300,8 +1300,8 @@ type QueryResponse struct { // Can be a Bitmap, Pairs, or uint64. Results []interface{} - // Set of profiles matching IDs returned in Result. - Profiles []*Profile + // Set of column attribute objects matching IDs returned in Result. + ColumnAttrSets []*ColumnAttrSet // Error during parsing or execution. Err error @@ -1309,12 +1309,12 @@ type QueryResponse struct { func (resp *QueryResponse) MarshalJSON() ([]byte, error) { var output struct { - Results []interface{} `json:"results,omitempty"` - Profiles []*Profile `json:"profiles,omitempty"` - Err string `json:"error,omitempty"` + Results []interface{} `json:"results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"` + Err string `json:"error,omitempty"` } output.Results = resp.Results - output.Profiles = resp.Profiles + output.ColumnAttrSets = resp.ColumnAttrSets if resp.Err != nil { output.Err = resp.Err.Error() @@ -1324,8 +1324,8 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - Profiles: encodeProfiles(resp.Profiles), + Results: make([]*internal.QueryResult, len(resp.Results)), + ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), } for i := range resp.Results { diff --git a/handler_test.go b/handler_test.go index 049fdc036..af4da1dd7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -253,18 +253,18 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { } } -// Ensure the handler can execute a query that returns a bitmap with profiles as JSON. -func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { +// Ensure the handler can execute a query that returns a bitmap with column attributes as JSON. +func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { idx := NewIndex() defer idx.Close() - // Create database and set profile attributes. + // Create database and set column attributes. db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) } @@ -277,10 +277,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?profiles=true", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -318,16 +318,16 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { } } -// Ensure the handler can execute a query that returns a bitmap with profiles as protobuf. -func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { +// Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf. +func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { idx := NewIndex() defer idx.Close() - // Create database and set profile attributes. + // Create database and set column attributes. db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) } @@ -341,8 +341,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", - Profiles: true, + Query: "Bitmap(id=100)", + ColumnAttrs: true, }) if err != nil { t.Fatal(err) @@ -373,12 +373,12 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } - if a := resp.Profiles; len(a) != 1 { - t.Fatalf("unexpected profiles length: %d", len(a)) + if a := resp.ColumnAttrSets; len(a) != 1 { + t.Fatalf("unexpected column attributes length: %d", len(a)) } else if a[0].ID != 1 { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { - t.Fatalf("unexpected profile attr length: %d", len(a)) + t.Fatalf("unexpected column attr length: %d", len(a)) } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) } @@ -603,16 +603,16 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { if err != nil { t.Fatal(err) } - if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } // Retrieve block checksums. - blks, err := db.ProfileAttrStore().Blocks() + blks, err := db.ColumnAttrStore().Blocks() if err != nil { t.Fatal(err) } @@ -653,16 +653,16 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { if err != nil { t.Fatal(err) } - if err := f.BitmapAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := f.BitmapAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := f.BitmapAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } // Retrieve block checksums. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { t.Fatal(err) } @@ -732,7 +732,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { t.Fatal("fragment x/y/standard/0 not created") - } else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { + } else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { t.Fatalf("unexpected restored bits: %+v", bits) } } diff --git a/index.go b/index.go index eb8f4b204..93228bb14 100644 --- a/index.go +++ b/index.go @@ -34,7 +34,7 @@ type Index struct { // Data directory path. Path string - // The interval at which the cached bitmap ids are persisted to disk. + // The interval at which the cached row ids are persisted to disk. CacheFlushInterval time.Duration LogOutput io.Writer @@ -375,7 +375,7 @@ func (s *IndexSyncer) SyncIndex() error { return nil } - // Sync database profile attributes. + // Sync database column attributes. if err := s.syncDatabase(di.Name); err != nil { return fmt.Errorf("db sync error: db=%s, err=%s", di.Name, err) } @@ -386,7 +386,7 @@ func (s *IndexSyncer) SyncIndex() error { return nil } - // Sync frame bitmap attributes. + // Sync frame row attributes. if err := s.syncFrame(di.Name, fi.Name); err != nil { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } @@ -429,7 +429,7 @@ func (s *IndexSyncer) syncDatabase(db string) error { } // Read block checksums. - blks, err := d.ProfileAttrStore().Blocks() + blks, err := d.ColumnAttrStore().Blocks() if err != nil { return err } @@ -443,7 +443,7 @@ func (s *IndexSyncer) syncDatabase(db string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ProfileAttrDiff(context.Background(), db, blks) + m, err := client.ColumnAttrDiff(context.Background(), db, blks) if err != nil { return err } else if len(m) == 0 { @@ -451,12 +451,12 @@ func (s *IndexSyncer) syncDatabase(db string) error { } // Update local copy. - if err := d.ProfileAttrStore().SetBulkAttrs(m); err != nil { + if err := d.ColumnAttrStore().SetBulkAttrs(m); err != nil { return err } // Recompute blocks. - blks, err = d.ProfileAttrStore().Blocks() + blks, err = d.ColumnAttrStore().Blocks() if err != nil { return err } @@ -474,7 +474,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // Read block checksums. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { return err } @@ -488,7 +488,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.BitmapAttrDiff(context.Background(), db, name, blks) + m, err := client.RowAttrDiff(context.Background(), db, name, blks) if err == ErrFrameNotFound { continue // frame not created remotely yet, skip } else if err != nil { @@ -498,12 +498,12 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // Update local copy. - if err := f.BitmapAttrStore().SetBulkAttrs(m); err != nil { + if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { return err } // Recompute blocks. - blks, err = f.BitmapAttrStore().Blocks() + blks, err = f.RowAttrStore().Blocks() if err != nil { return err } diff --git a/index_test.go b/index_test.go index 4a1be4cf2..22efbc718 100644 --- a/index_test.go +++ b/index_test.go @@ -135,28 +135,28 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Verify data is the same on both nodes. for i, idx := range []*Index{idx0, idx1} { f := idx.Fragment("d", "f", pilosa.ViewStandard, 0) - if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Bitmap(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Bitmap(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1) - a := f.Bitmap(9).Bits() + a := f.Row(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } f = idx.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) } } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0423cfa08..f82ba23f8 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -90,8 +90,8 @@ func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } @@ -100,7 +100,7 @@ func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } @@ -432,10 +432,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.BitmapIDs) > 0 { - dAtA2 := make([]byte, len(m.BitmapIDs)*10) + if len(m.RowIDs) > 0 { + dAtA2 := make([]byte, len(m.RowIDs)*10) var j1 int - for _, num := range m.BitmapIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -449,10 +449,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j1)) i += copy(dAtA[i:], dAtA2[:j1]) } - if len(m.ProfileIDs) > 0 { - dAtA4 := make([]byte, len(m.ProfileIDs)*10) + if len(m.ColumnIDs) > 0 { + dAtA4 := make([]byte, len(m.ColumnIDs)*10) var j3 int - for _, num := range m.ProfileIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -484,10 +484,10 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.BitmapIDs) > 0 { - dAtA6 := make([]byte, len(m.BitmapIDs)*10) + if len(m.IDs) > 0 { + dAtA6 := make([]byte, len(m.IDs)*10) var j5 int - for _, num := range m.BitmapIDs { + for _, num := range m.IDs { for num >= 1<<7 { dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -924,16 +924,16 @@ func (m *BlockDataRequest) Size() (n int) { func (m *BlockDataResponse) Size() (n int) { var l int _ = l - if len(m.BitmapIDs) > 0 { + if len(m.RowIDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.RowIDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l } - if len(m.ProfileIDs) > 0 { + if len(m.ColumnIDs) > 0 { l = 0 - for _, e := range m.ProfileIDs { + for _, e := range m.ColumnIDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l @@ -944,9 +944,9 @@ func (m *BlockDataResponse) Size() (n int) { func (m *Cache) Size() (n int) { var l int _ = l - if len(m.BitmapIDs) > 0 { + if len(m.IDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.IDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l @@ -1714,7 +1714,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } } else if wireType == 0 { var v uint64 @@ -1732,9 +1732,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) } case 2: if wireType == 2 { @@ -1776,7 +1776,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } } else if wireType == 0 { var v uint64 @@ -1794,9 +1794,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } default: iNdEx = preIndex @@ -1888,7 +1888,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.IDs = append(m.IDs, v) } } else if wireType == 0 { var v uint64 @@ -1906,9 +1906,9 @@ func (m *Cache) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.IDs = append(m.IDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) } default: iNdEx = preIndex @@ -3146,43 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 600 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, - 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, - 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, - 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, - 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xea, 0xab, - 0xaf, 0xbf, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, - 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, - 0x3d, 0x48, 0xa3, 0x71, 0x9c, 0x9c, 0xf0, 0x01, 0x46, 0xae, 0xd1, 0x33, 0x76, 0x1c, 0xd6, 0x86, - 0x64, 0xc5, 0x45, 0x18, 0xe3, 0xf9, 0x98, 0x27, 0x62, 0x1c, 0xbb, 0xa6, 0xae, 0x68, 0x41, 0xf4, - 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0x6d, 0xba, 0x3a, - 0x26, 0x8f, 0xa1, 0x73, 0x9c, 0x4c, 0x30, 0x2f, 0xf0, 0x30, 0xe1, 0x83, 0x08, 0x2f, 0x15, 0xdd, - 0x0a, 0x9b, 0x43, 0xc9, 0x16, 0x38, 0x07, 0x7c, 0xf8, 0x1e, 0x2f, 0xa6, 0x19, 0xba, 0x96, 0x22, - 0x69, 0x80, 0x3a, 0xdb, 0x0f, 0x3f, 0xa2, 0x6b, 0xf7, 0x8c, 0x9d, 0x35, 0xd6, 0x00, 0xf3, 0x7a, - 0x97, 0x6e, 0xea, 0xa5, 0xd0, 0x39, 0x8e, 0xb3, 0x34, 0x17, 0x0c, 0x8b, 0x2c, 0x4d, 0x0a, 0x24, - 0x5d, 0xb0, 0x0e, 0xf3, 0xbc, 0x94, 0x2b, 0x8f, 0xf4, 0x1a, 0xba, 0x7e, 0x94, 0x0e, 0x47, 0x01, - 0x17, 0x9c, 0xe1, 0x87, 0x31, 0x16, 0x82, 0x74, 0xc0, 0x0c, 0xfc, 0xb2, 0xc8, 0x0c, 0x7c, 0x72, - 0x1f, 0x96, 0xd4, 0xb5, 0x4b, 0x4f, 0x74, 0x20, 0x51, 0xd5, 0xa9, 0x74, 0xdb, 0x4c, 0x07, 0x12, - 0xed, 0x47, 0xe1, 0x50, 0xeb, 0xb5, 0x99, 0x0e, 0x08, 0x01, 0xfb, 0x4d, 0x88, 0x57, 0xa5, 0x48, - 0x75, 0xa6, 0xe7, 0xb0, 0xd1, 0x9a, 0x5c, 0x0a, 0xdc, 0x02, 0xc7, 0x0f, 0x45, 0xcc, 0xb3, 0xe3, - 0xa0, 0x70, 0x8d, 0x9e, 0xb5, 0x63, 0xb3, 0x06, 0x20, 0x1e, 0xc0, 0xab, 0x3c, 0x7d, 0x1b, 0x46, - 0x28, 0xd3, 0xa6, 0x4a, 0xb7, 0x10, 0xfa, 0x08, 0x96, 0x94, 0x3f, 0x7f, 0xa6, 0xa1, 0x5f, 0x0c, - 0xd8, 0x38, 0xe5, 0xd7, 0x4a, 0x5a, 0x51, 0x8f, 0x3e, 0x02, 0xa7, 0x06, 0x55, 0xcf, 0xea, 0xde, - 0x93, 0xdd, 0x6a, 0x93, 0x76, 0x6f, 0xd4, 0x37, 0xc8, 0x61, 0x22, 0xf2, 0x29, 0x6b, 0x9a, 0x37, - 0x9f, 0x43, 0xe7, 0xf7, 0xa4, 0xf4, 0x7d, 0x84, 0xd3, 0xca, 0xf7, 0x11, 0x4e, 0xa5, 0x4f, 0x13, - 0x1e, 0x8d, 0xb5, 0xa7, 0x36, 0xd3, 0xc1, 0xbe, 0xf9, 0xcc, 0xa0, 0xfb, 0x40, 0x0e, 0x72, 0xe4, - 0x02, 0x15, 0xc1, 0x29, 0x16, 0x05, 0x7f, 0x87, 0x8b, 0xbe, 0x89, 0xf6, 0xd9, 0x6c, 0xf9, 0x4c, - 0x1f, 0xc0, 0x7a, 0x80, 0x11, 0x0a, 0x94, 0x5b, 0xbf, 0xb0, 0x91, 0xbe, 0x84, 0x75, 0x4d, 0x7f, - 0x6b, 0x09, 0x79, 0x08, 0xb6, 0xdc, 0x70, 0x45, 0xbd, 0xba, 0xd7, 0x6d, 0x4c, 0xd0, 0x6f, 0x89, - 0xa9, 0x2c, 0x1d, 0x56, 0x3a, 0xcb, 0x27, 0x71, 0xab, 0xce, 0x05, 0xbb, 0xb3, 0x5d, 0x4e, 0xb0, - 0xd4, 0x84, 0x7b, 0xcd, 0x84, 0xfa, 0x79, 0x95, 0x43, 0xf6, 0x81, 0xe8, 0x0b, 0xfd, 0xff, 0x10, - 0x1a, 0x94, 0xa8, 0xdc, 0xbe, 0x33, 0x99, 0xd5, 0x0d, 0xea, 0x5c, 0x2b, 0x30, 0xff, 0xa6, 0xe0, - 0x93, 0x21, 0x87, 0x2d, 0xe4, 0xf8, 0x27, 0x9f, 0xe4, 0x7f, 0xa2, 0xda, 0x86, 0xf2, 0xa9, 0xd4, - 0x31, 0xd9, 0x86, 0x65, 0x35, 0xaf, 0x70, 0x6d, 0xb5, 0x70, 0xeb, 0x73, 0x3a, 0x58, 0x99, 0xa6, - 0xaf, 0xc1, 0x39, 0x4b, 0x2f, 0xb1, 0x2f, 0xb8, 0x50, 0xf7, 0x39, 0x4a, 0x0b, 0x51, 0x69, 0x91, - 0x67, 0xb5, 0x0f, 0x32, 0x59, 0x59, 0xa0, 0x2b, 0x3d, 0xb0, 0x02, 0xbf, 0x70, 0x2d, 0x45, 0x7e, - 0xb7, 0x2d, 0x90, 0xc9, 0x84, 0xdf, 0xfd, 0x36, 0xf3, 0x8c, 0xef, 0x33, 0xcf, 0xf8, 0x31, 0xf3, - 0x8c, 0xcf, 0x3f, 0xbd, 0x3b, 0x83, 0x65, 0xf5, 0x0b, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, - 0x3a, 0x23, 0x0f, 0xb4, 0x53, 0x05, 0x00, 0x00, + // 596 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0x13, 0x41, + 0x14, 0x76, 0x7f, 0x68, 0xe8, 0x41, 0x4a, 0x19, 0x8d, 0x59, 0x89, 0x69, 0xea, 0xc4, 0x08, 0xf1, + 0x82, 0x0b, 0xbc, 0x31, 0xc4, 0xab, 0x65, 0x51, 0x9a, 0x00, 0x89, 0x03, 0x7a, 0x3f, 0x94, 0x13, + 0xdd, 0xb0, 0xdd, 0xad, 0xbb, 0x53, 0xa0, 0xde, 0xfa, 0x12, 0x26, 0x3e, 0x83, 0xef, 0xe1, 0xa5, + 0x8f, 0x60, 0xea, 0x8b, 0x98, 0x39, 0x33, 0xfb, 0x63, 0x29, 0x51, 0xef, 0xe6, 0x7c, 0xe7, 0xef, + 0x9b, 0x6f, 0xbf, 0x59, 0x58, 0x1d, 0xe7, 0xf1, 0xa5, 0x54, 0xb8, 0x3d, 0xce, 0x33, 0x95, 0xb1, + 0xe5, 0x38, 0x55, 0x98, 0xa7, 0x32, 0xe1, 0x87, 0xd0, 0x8a, 0xc2, 0x23, 0x54, 0x92, 0xf5, 0x61, + 0x65, 0x2f, 0x4b, 0x26, 0xa3, 0xf4, 0x50, 0x9e, 0x61, 0x12, 0x38, 0x7d, 0x67, 0xab, 0x2d, 0x9a, + 0x90, 0xae, 0x38, 0x8d, 0x47, 0xf8, 0x66, 0x22, 0x53, 0x35, 0x19, 0x05, 0xae, 0xa9, 0x68, 0x40, + 0xfc, 0x9b, 0x03, 0xed, 0x57, 0xb9, 0x1c, 0x21, 0x4d, 0xdc, 0x80, 0x65, 0x91, 0x5d, 0x35, 0xc7, + 0x55, 0x31, 0x7b, 0x0a, 0x9d, 0x41, 0x7a, 0x89, 0x79, 0x81, 0xfb, 0xa9, 0x3c, 0x4b, 0xf0, 0x9c, + 0xc6, 0x2d, 0x8b, 0x39, 0x94, 0x3d, 0x82, 0xf6, 0x9e, 0x1c, 0x7e, 0xc0, 0xd3, 0xe9, 0x18, 0x03, + 0x8f, 0x86, 0xd4, 0x40, 0x95, 0x3d, 0x89, 0x3f, 0x61, 0xe0, 0xf7, 0x9d, 0xad, 0x55, 0x51, 0x03, + 0xf3, 0x7c, 0x97, 0x6e, 0xf2, 0xe5, 0xd0, 0x19, 0x8c, 0xc6, 0x59, 0xae, 0x04, 0x16, 0xe3, 0x2c, + 0x2d, 0x90, 0x75, 0xc1, 0xdb, 0xcf, 0x73, 0x4b, 0x57, 0x1f, 0xf9, 0x35, 0x74, 0xc3, 0x24, 0x1b, + 0x5e, 0x44, 0x52, 0x49, 0x81, 0x1f, 0x27, 0x58, 0x28, 0xd6, 0x01, 0x37, 0x0a, 0x6d, 0x91, 0x1b, + 0x85, 0xec, 0x3e, 0x2c, 0xd1, 0xb5, 0xad, 0x26, 0x26, 0xd0, 0x28, 0x75, 0x12, 0x6f, 0x5f, 0x98, + 0x40, 0xa3, 0x27, 0x49, 0x3c, 0x34, 0x7c, 0x7d, 0x61, 0x02, 0xc6, 0xc0, 0x7f, 0x17, 0xe3, 0x95, + 0x25, 0x49, 0x67, 0x3e, 0x80, 0xf5, 0xc6, 0x66, 0x4b, 0xf0, 0x01, 0xb4, 0x44, 0x76, 0x35, 0x88, + 0x8a, 0xc0, 0xe9, 0x7b, 0x5b, 0xbe, 0xb0, 0x11, 0x49, 0x41, 0xdf, 0x4a, 0xa7, 0x5c, 0x4a, 0xd5, + 0x00, 0x7f, 0x08, 0x4b, 0xa4, 0x8b, 0xbe, 0x5f, 0xdd, 0xab, 0x8f, 0xfc, 0xab, 0x03, 0xeb, 0x47, + 0xf2, 0x9a, 0x68, 0x14, 0xd5, 0x9a, 0x03, 0x68, 0x57, 0x20, 0x55, 0xaf, 0xec, 0x3c, 0xdb, 0x2e, + 0x5d, 0xb3, 0x7d, 0xa3, 0xbe, 0x46, 0xf6, 0x53, 0x95, 0x4f, 0x45, 0xdd, 0xbc, 0xf1, 0x12, 0x3a, + 0x7f, 0x26, 0x35, 0x87, 0x0b, 0x9c, 0x96, 0x1a, 0x5f, 0xe0, 0x54, 0x6b, 0x72, 0x29, 0x93, 0x89, + 0xd1, 0xcf, 0x17, 0x26, 0xd8, 0x75, 0x5f, 0x38, 0x7c, 0x17, 0xd8, 0x5e, 0x8e, 0x52, 0x21, 0x0d, + 0x38, 0xc2, 0xa2, 0x90, 0xef, 0x71, 0x91, 0xfe, 0x46, 0x53, 0xb7, 0xa1, 0x29, 0x7f, 0x0c, 0x6b, + 0x11, 0x26, 0xa8, 0x50, 0x3b, 0x7c, 0x61, 0x23, 0x7f, 0x0d, 0x6b, 0x66, 0xfc, 0xad, 0x25, 0xec, + 0x09, 0xf8, 0xda, 0xcd, 0x34, 0x7a, 0x65, 0xa7, 0x5b, 0x8b, 0x60, 0xde, 0x8d, 0xa0, 0x2c, 0x1f, + 0x96, 0x3c, 0xad, 0xfd, 0x6f, 0xe5, 0xb9, 0xc0, 0x27, 0x9b, 0x76, 0x83, 0x47, 0x1b, 0xee, 0xd5, + 0x1b, 0xaa, 0xa7, 0x64, 0x97, 0xec, 0x02, 0x33, 0x17, 0xfa, 0xff, 0x25, 0x3c, 0xb2, 0xa8, 0x76, + 0xda, 0xb1, 0xce, 0x9a, 0x06, 0x3a, 0x57, 0x0c, 0xdc, 0xbf, 0x31, 0xf8, 0xec, 0xe8, 0x65, 0x0b, + 0x67, 0xfc, 0x93, 0x4e, 0xfa, 0x9f, 0x50, 0xba, 0xc1, 0x3e, 0x8b, 0x2a, 0x66, 0x9b, 0xd0, 0xa2, + 0x7d, 0x45, 0xe0, 0x93, 0xe1, 0xd6, 0xe6, 0x78, 0x08, 0x9b, 0xe6, 0x6f, 0xa1, 0x7d, 0x9c, 0x9d, + 0xe3, 0x89, 0x92, 0x8a, 0xee, 0x73, 0x90, 0x15, 0xaa, 0xe4, 0xa2, 0xcf, 0xe4, 0x07, 0x9d, 0x2c, + 0x25, 0x30, 0x95, 0x3d, 0xf0, 0xa2, 0xb0, 0x08, 0x3c, 0x1a, 0x7e, 0xb7, 0x49, 0x50, 0xe8, 0x44, + 0xd8, 0xfd, 0x3e, 0xeb, 0x39, 0x3f, 0x66, 0x3d, 0xe7, 0xe7, 0xac, 0xe7, 0x7c, 0xf9, 0xd5, 0xbb, + 0x73, 0xd6, 0xa2, 0xdf, 0xe5, 0xf3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcf, 0x20, 0xf1, + 0x3f, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index ec060e355..6294b1091 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -28,12 +28,12 @@ message BlockDataRequest { } message BlockDataResponse { - repeated uint64 BitmapIDs = 1; - repeated uint64 ProfileIDs = 2; + repeated uint64 RowIDs = 1; + repeated uint64 ColumnIDs = 2; } message Cache { - repeated uint64 BitmapIDs = 1; + repeated uint64 IDs = 1; } message MaxSlicesResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 249bbfddf..24fb7eb85 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -12,7 +12,7 @@ Bitmap Pair Bit - Profile + ColumnAttrSet Attr AttrMap QueryRequest @@ -67,8 +67,8 @@ func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } type Bit struct { - BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` - ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` + RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` + ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } @@ -77,17 +77,17 @@ func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -type Profile struct { +type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *Profile) Reset() { *m = Profile{} } -func (m *Profile) String() string { return proto.CompactTextString(m) } -func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } +func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } +func (*ColumnAttrSet) ProtoMessage() {} +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *Profile) GetAttrs() []*Attr { +func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { return m.Attrs } @@ -125,11 +125,11 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` - Profiles bool `protobuf:"varint,3,opt,name=Profiles,proto3" json:"Profiles,omitempty"` - Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -138,9 +138,9 @@ func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } @@ -155,9 +155,9 @@ func (m *QueryResponse) GetResults() []*QueryResult { return nil } -func (m *QueryResponse) GetProfiles() []*Profile { +func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { if m != nil { - return m.Profiles + return m.ColumnAttrSets } return nil } @@ -192,8 +192,8 @@ type ImportRequest struct { DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } @@ -206,7 +206,7 @@ func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") - proto.RegisterType((*Profile)(nil), "internal.Profile") + proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") proto.RegisterType((*AttrMap)(nil), "internal.AttrMap") proto.RegisterType((*QueryRequest)(nil), "internal.QueryRequest") @@ -304,15 +304,15 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.BitmapID != 0 { + if m.RowID != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.BitmapID)) + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) } - if m.ProfileID != 0 { + if m.ColumnID != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ProfileID)) + i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID)) } if m.Timestamp != 0 { dAtA[i] = 0x18 @@ -322,7 +322,7 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Profile) Marshal() (dAtA []byte, err error) { +func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -332,7 +332,7 @@ func (m *Profile) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Profile) MarshalTo(dAtA []byte) (int, error) { +func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -480,10 +480,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j3)) i += copy(dAtA[i:], dAtA4[:j3]) } - if m.Profiles { + if m.ColumnAttrs { dAtA[i] = 0x18 i++ - if m.Profiles { + if m.ColumnAttrs { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -542,8 +542,8 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.Profiles) > 0 { - for _, msg := range m.Profiles { + if len(m.ColumnAttrSets) > 0 { + for _, msg := range m.ColumnAttrSets { dAtA[i] = 0x1a i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -644,10 +644,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) } - if len(m.BitmapIDs) > 0 { - dAtA7 := make([]byte, len(m.BitmapIDs)*10) + if len(m.RowIDs) > 0 { + dAtA7 := make([]byte, len(m.RowIDs)*10) var j6 int - for _, num := range m.BitmapIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -661,10 +661,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j6)) i += copy(dAtA[i:], dAtA7[:j6]) } - if len(m.ProfileIDs) > 0 { - dAtA9 := make([]byte, len(m.ProfileIDs)*10) + if len(m.ColumnIDs) > 0 { + dAtA9 := make([]byte, len(m.ColumnIDs)*10) var j8 int - for _, num := range m.ProfileIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -760,11 +760,11 @@ func (m *Pair) Size() (n int) { func (m *Bit) Size() (n int) { var l int _ = l - if m.BitmapID != 0 { - n += 1 + sovPublic(uint64(m.BitmapID)) + if m.RowID != 0 { + n += 1 + sovPublic(uint64(m.RowID)) } - if m.ProfileID != 0 { - n += 1 + sovPublic(uint64(m.ProfileID)) + if m.ColumnID != 0 { + n += 1 + sovPublic(uint64(m.ColumnID)) } if m.Timestamp != 0 { n += 1 + sovPublic(uint64(m.Timestamp)) @@ -772,7 +772,7 @@ func (m *Bit) Size() (n int) { return n } -func (m *Profile) Size() (n int) { +func (m *ColumnAttrSet) Size() (n int) { var l int _ = l if m.ID != 0 { @@ -839,7 +839,7 @@ func (m *QueryRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if m.Profiles { + if m.ColumnAttrs { n += 2 } l = len(m.Quantum) @@ -865,8 +865,8 @@ func (m *QueryResponse) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if len(m.Profiles) > 0 { - for _, e := range m.Profiles { + if len(m.ColumnAttrSets) > 0 { + for _, e := range m.ColumnAttrSets { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -910,16 +910,16 @@ func (m *ImportRequest) Size() (n int) { if m.Slice != 0 { n += 1 + sovPublic(uint64(m.Slice)) } - if len(m.BitmapIDs) > 0 { + if len(m.RowIDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.RowIDs { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l } - if len(m.ProfileIDs) > 0 { + if len(m.ColumnIDs) > 0 { l = 0 - for _, e := range m.ProfileIDs { + for _, e := range m.ColumnIDs { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1209,9 +1209,9 @@ func (m *Bit) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) } - m.BitmapID = 0 + m.RowID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1221,16 +1221,16 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.BitmapID |= (uint64(b) & 0x7F) << shift + m.RowID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType) } - m.ProfileID = 0 + m.ColumnID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1240,7 +1240,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ProfileID |= (uint64(b) & 0x7F) << shift + m.ColumnID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1285,7 +1285,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } return nil } -func (m *Profile) Unmarshal(dAtA []byte) error { +func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1308,10 +1308,10 @@ func (m *Profile) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Profile: wiretype end group for non-group") + return fmt.Errorf("proto: ColumnAttrSet: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ColumnAttrSet: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1772,7 +1772,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrs", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -1789,7 +1789,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Profiles = bool(v != 0) + m.ColumnAttrs = bool(v != 0) case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType) @@ -1951,7 +1951,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrSets", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1975,8 +1975,8 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Profiles = append(m.Profiles, &Profile{}) - if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.ColumnAttrSets = append(m.ColumnAttrSets, &ColumnAttrSet{}) + if err := m.ColumnAttrSets[len(m.ColumnAttrSets)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -2300,7 +2300,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } } else if wireType == 0 { var v uint64 @@ -2318,9 +2318,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) } case 5: if wireType == 2 { @@ -2362,7 +2362,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } } else if wireType == 0 { var v uint64 @@ -2380,9 +2380,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } case 6: if wireType == 2 { @@ -2575,41 +2575,42 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 570 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x6e, 0xd4, 0x40, - 0x10, 0xa5, 0x6d, 0xcf, 0xaf, 0x26, 0x19, 0x0d, 0x2d, 0x40, 0x16, 0x42, 0x23, 0xcb, 0x62, 0xe1, - 0x0d, 0x13, 0x29, 0x1c, 0x00, 0xe1, 0x4c, 0x22, 0x8d, 0x10, 0x51, 0xd2, 0x89, 0xd8, 0xb1, 0x70, - 0x42, 0x13, 0x2c, 0xf9, 0x47, 0x77, 0x7b, 0x31, 0x4b, 0x16, 0x6c, 0x38, 0x01, 0x47, 0x80, 0x9b, - 0xb0, 0xe4, 0x08, 0x68, 0xb8, 0x08, 0xaa, 0xfe, 0xd8, 0x66, 0x83, 0xd8, 0xf5, 0x7b, 0xe5, 0xea, - 0xae, 0xf7, 0xaa, 0xca, 0x70, 0xd0, 0xb4, 0x37, 0x45, 0x7e, 0xbb, 0x6e, 0x44, 0xad, 0x6a, 0x3a, - 0xcd, 0x2b, 0xc5, 0x45, 0x95, 0x15, 0x71, 0x0a, 0xe3, 0x34, 0x57, 0x65, 0xd6, 0x50, 0x0a, 0x41, - 0x9a, 0x2b, 0x19, 0x92, 0xc8, 0x4f, 0x02, 0xa6, 0xcf, 0xf4, 0x29, 0x8c, 0x5e, 0x2a, 0x25, 0x64, - 0xe8, 0x45, 0x7e, 0x32, 0x3f, 0x5e, 0xac, 0x5d, 0xde, 0x1a, 0x69, 0x66, 0x82, 0xf1, 0x1a, 0x82, - 0x8b, 0x2c, 0x17, 0x74, 0x09, 0xfe, 0x2b, 0xbe, 0x0b, 0x49, 0x44, 0x92, 0x80, 0xe1, 0x91, 0x3e, - 0x80, 0xd1, 0x49, 0xdd, 0x56, 0x2a, 0xf4, 0x34, 0x67, 0x40, 0xfc, 0x16, 0xfc, 0x34, 0x57, 0xf4, - 0x31, 0x4c, 0xcd, 0xd3, 0xdb, 0x8d, 0xcd, 0xe9, 0x30, 0x7d, 0x02, 0xb3, 0x0b, 0x51, 0xbf, 0xcf, - 0x0b, 0xbe, 0xdd, 0xd8, 0xe4, 0x9e, 0xc0, 0xe8, 0x75, 0x5e, 0x72, 0xa9, 0xb2, 0xb2, 0x09, 0xfd, - 0x88, 0x24, 0x3e, 0xeb, 0x89, 0xf8, 0x05, 0x4c, 0xec, 0xa7, 0x74, 0x01, 0x5e, 0x77, 0xb9, 0xb7, - 0xdd, 0xfc, 0xa7, 0x9e, 0x6f, 0x04, 0x02, 0x3c, 0x0d, 0x05, 0xcd, 0x8c, 0x20, 0x0a, 0xc1, 0xf5, - 0xae, 0xe1, 0xb6, 0x24, 0x7d, 0xa6, 0x11, 0xcc, 0xaf, 0x94, 0xc8, 0xab, 0xbb, 0x37, 0x59, 0xd1, - 0x72, 0x5d, 0xcf, 0x8c, 0x0d, 0x29, 0x54, 0xba, 0xad, 0x94, 0x09, 0x07, 0xba, 0xdc, 0x0e, 0xa3, - 0x96, 0xb4, 0xae, 0x0b, 0x13, 0x1c, 0x45, 0x24, 0x99, 0xb2, 0x9e, 0xa0, 0x2b, 0x80, 0xb3, 0xa2, - 0xce, 0x6c, 0xee, 0x38, 0x22, 0x09, 0x61, 0x03, 0x26, 0x3e, 0x82, 0x09, 0x56, 0xfa, 0x3a, 0x6b, - 0x7a, 0x6d, 0xe4, 0x5f, 0xda, 0xbe, 0x10, 0x38, 0xb8, 0x6c, 0xb9, 0xd8, 0x31, 0xfe, 0xb1, 0xe5, - 0x52, 0x61, 0x8b, 0x34, 0xb6, 0x2a, 0x0d, 0xa0, 0x8f, 0x60, 0x7c, 0x55, 0xe4, 0xb7, 0xdc, 0x38, - 0x15, 0x30, 0x8b, 0x50, 0x89, 0xf5, 0x56, 0x6a, 0xa1, 0x53, 0xd6, 0x61, 0x1a, 0xc2, 0xe4, 0xb2, - 0xcd, 0x2a, 0xd5, 0x96, 0x5a, 0xe4, 0x8c, 0x39, 0x88, 0xb7, 0x31, 0x5e, 0xd6, 0xca, 0x09, 0xb4, - 0x28, 0xfe, 0x44, 0xe0, 0xd0, 0x16, 0x23, 0x9b, 0xba, 0x92, 0x1c, 0x1d, 0x3f, 0x15, 0xc2, 0x39, - 0x7e, 0x2a, 0x04, 0x3d, 0x82, 0x09, 0xe3, 0xb2, 0x2d, 0x94, 0x6b, 0xda, 0xc3, 0x5e, 0x98, 0xcb, - 0x6d, 0x0b, 0xc5, 0xdc, 0x57, 0xf4, 0xd9, 0x5f, 0x25, 0x62, 0xc6, 0xfd, 0x3e, 0xc3, 0x46, 0xfa, - 0xaa, 0xe3, 0xcf, 0x04, 0xe6, 0x83, 0x7b, 0x68, 0xe2, 0x16, 0x42, 0x17, 0x31, 0x3f, 0x5e, 0xf6, - 0xc9, 0x86, 0x67, 0x6e, 0x61, 0x0e, 0x80, 0x9c, 0xdb, 0x41, 0x20, 0xe7, 0x68, 0x3f, 0x2e, 0x81, - 0x7b, 0x73, 0x60, 0x3f, 0xd2, 0xcc, 0x04, 0xd1, 0xa3, 0x93, 0x0f, 0x59, 0x75, 0xc7, 0xdf, 0x69, - 0x8f, 0xa6, 0xcc, 0xc1, 0xf8, 0x3b, 0x81, 0xc3, 0x6d, 0xd9, 0xd4, 0x42, 0xb9, 0xce, 0x2c, 0xc0, - 0xdb, 0xa4, 0xd6, 0x0a, 0x6f, 0x93, 0x62, 0xa7, 0xce, 0x44, 0x56, 0x9a, 0xe1, 0x9b, 0x31, 0x03, - 0x90, 0xd5, 0xbd, 0xd1, 0xed, 0x08, 0x98, 0x01, 0x7a, 0xaa, 0xec, 0x2e, 0xc9, 0x30, 0xd0, 0x2d, - 0xec, 0x09, 0x9c, 0xaa, 0x6e, 0x99, 0x64, 0x38, 0xd2, 0xe1, 0x01, 0x83, 0xf1, 0x6e, 0x9d, 0x64, - 0x38, 0x8e, 0xfc, 0xc4, 0x67, 0x03, 0x26, 0x5d, 0xfe, 0xd8, 0xaf, 0xc8, 0xcf, 0xfd, 0x8a, 0xfc, - 0xda, 0xaf, 0xc8, 0xd7, 0xdf, 0xab, 0x7b, 0x37, 0x63, 0xfd, 0x5f, 0x79, 0xfe, 0x27, 0x00, 0x00, - 0xff, 0xff, 0x37, 0xb6, 0x15, 0x22, 0x67, 0x04, 0x00, 0x00, + // 579 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0x66, 0x6d, 0x27, 0x4d, 0x26, 0x6d, 0x14, 0xad, 0xf8, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x83, + 0x4f, 0xa9, 0x54, 0x1e, 0x00, 0xe1, 0x24, 0x95, 0x22, 0x44, 0x45, 0x27, 0x85, 0xbb, 0x5b, 0x56, + 0xc5, 0x92, 0xff, 0x58, 0xaf, 0x85, 0xf2, 0x00, 0xdc, 0x91, 0xb8, 0x70, 0xe5, 0xc6, 0xa3, 0x70, + 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0xbb, 0xde, 0xd8, 0xe5, 0x80, 0xb8, 0xed, 0xf7, 0xcd, 0xce, + 0x7a, 0xbe, 0xf9, 0x66, 0x0c, 0xc7, 0x55, 0x73, 0x9d, 0xa5, 0x37, 0x8b, 0x4a, 0x96, 0xaa, 0xe4, + 0xa3, 0xb4, 0x50, 0x42, 0x16, 0x49, 0x16, 0xc6, 0x30, 0x8c, 0x53, 0x95, 0x27, 0x15, 0xe7, 0xe0, + 0xc5, 0xa9, 0xaa, 0x7d, 0x16, 0xb8, 0x91, 0x87, 0xfa, 0xcc, 0x9f, 0xc2, 0xe0, 0x85, 0x52, 0xb2, + 0xf6, 0x9d, 0xc0, 0x8d, 0x26, 0x67, 0xd3, 0x85, 0xcd, 0x5b, 0x10, 0x8d, 0x26, 0x18, 0x2e, 0xc0, + 0x7b, 0x9d, 0xa4, 0x92, 0xcf, 0xc0, 0x7d, 0x29, 0x76, 0x3e, 0x0b, 0x58, 0xe4, 0x21, 0x1d, 0xf9, + 0x7d, 0x18, 0x2c, 0xcb, 0xa6, 0x50, 0xbe, 0xa3, 0x39, 0x03, 0xc2, 0x37, 0xe0, 0xc6, 0xa9, 0xa2, + 0x20, 0x96, 0x1f, 0x37, 0xab, 0x36, 0xc1, 0x00, 0xfe, 0x18, 0x46, 0xcb, 0x32, 0x6b, 0xf2, 0x62, + 0xb3, 0x6a, 0xb3, 0x0e, 0x98, 0x3f, 0x81, 0xf1, 0x55, 0x9a, 0x8b, 0x5a, 0x25, 0x79, 0xe5, 0xbb, + 0x01, 0x8b, 0x5c, 0xec, 0x88, 0x70, 0x0d, 0x27, 0xe6, 0x26, 0x55, 0xb5, 0x15, 0x8a, 0x4f, 0xc1, + 0x39, 0xbc, 0xee, 0x6c, 0x56, 0xff, 0xa9, 0xe6, 0x3b, 0x03, 0x8f, 0x4e, 0x7d, 0x39, 0x63, 0x23, + 0x87, 0x83, 0x77, 0xb5, 0xab, 0x44, 0x5b, 0x97, 0x3e, 0xf3, 0x00, 0x26, 0x5b, 0x25, 0xd3, 0xe2, + 0xf6, 0x6d, 0x92, 0x35, 0x42, 0x57, 0x35, 0xc6, 0x3e, 0x45, 0x8a, 0x36, 0x85, 0x32, 0x61, 0x4f, + 0x17, 0x7d, 0xc0, 0xa4, 0x28, 0x2e, 0xcb, 0xcc, 0x04, 0x07, 0x01, 0x8b, 0x46, 0xd8, 0x11, 0x7c, + 0x0e, 0x70, 0x9e, 0x95, 0x49, 0x9b, 0x3b, 0x0c, 0x58, 0xc4, 0xb0, 0xc7, 0x84, 0xa7, 0x70, 0x44, + 0x95, 0xbe, 0x4a, 0xaa, 0x4e, 0x1b, 0xfb, 0x97, 0xb6, 0xcf, 0x0c, 0x8e, 0x2f, 0x1b, 0x21, 0x77, + 0x28, 0x3e, 0x34, 0xa2, 0xd6, 0x1e, 0x68, 0xdc, 0xaa, 0x34, 0x80, 0x3f, 0x84, 0xe1, 0x36, 0x4b, + 0x6f, 0x84, 0xe9, 0x94, 0x87, 0x2d, 0x22, 0xad, 0x5d, 0x87, 0x6b, 0xad, 0x75, 0x84, 0x7d, 0x8a, + 0xfb, 0x70, 0x74, 0xd9, 0x24, 0x85, 0x6a, 0x72, 0x2d, 0x75, 0x8c, 0x16, 0xd2, 0x9b, 0x28, 0xf2, + 0x52, 0x59, 0x99, 0x2d, 0x0a, 0xbf, 0x30, 0x38, 0x69, 0x4b, 0xaa, 0xab, 0xb2, 0xa8, 0x05, 0xf5, + 0x7d, 0x2d, 0xa5, 0xed, 0xfb, 0x5a, 0x4a, 0x7e, 0x0a, 0x47, 0x28, 0xea, 0x26, 0x53, 0xd6, 0xba, + 0x07, 0x9d, 0x3c, 0x9b, 0xdb, 0x64, 0x0a, 0xed, 0x2d, 0xfe, 0x1c, 0xa6, 0x77, 0x46, 0x81, 0x6a, + 0xa5, 0xbc, 0x47, 0x5d, 0xde, 0x9d, 0x38, 0xfe, 0x75, 0x3d, 0xfc, 0xc4, 0x60, 0xd2, 0x7b, 0x99, + 0x47, 0x76, 0x4d, 0x74, 0x59, 0x93, 0xb3, 0x59, 0xf7, 0x90, 0xe1, 0xd1, 0xae, 0xd1, 0x31, 0xb0, + 0x8b, 0x76, 0x40, 0xd8, 0x05, 0xd9, 0x42, 0xab, 0x61, 0xbf, 0xdf, 0xb3, 0x85, 0x68, 0x34, 0x41, + 0xea, 0xda, 0xf2, 0x7d, 0x52, 0xdc, 0x8a, 0x77, 0xba, 0x6b, 0x23, 0xb4, 0x30, 0xfc, 0xc6, 0xe0, + 0x64, 0x93, 0x57, 0xa5, 0x54, 0xd6, 0xb1, 0x29, 0x38, 0xab, 0xb8, 0x6d, 0x8e, 0xb3, 0x8a, 0xc9, + 0xc1, 0x73, 0x99, 0xe4, 0x66, 0x28, 0xc7, 0x68, 0x00, 0xb1, 0xda, 0x33, 0xed, 0x91, 0x87, 0x06, + 0x68, 0x0f, 0x68, 0xc9, 0x6a, 0xdf, 0x33, 0xbe, 0x1a, 0x44, 0x53, 0x68, 0x77, 0xac, 0xf6, 0x07, + 0x3a, 0xd4, 0x11, 0x34, 0x85, 0x87, 0x25, 0xab, 0xfd, 0x61, 0xe0, 0x46, 0x2e, 0xf6, 0x98, 0x78, + 0xf6, 0x63, 0x3f, 0x67, 0x3f, 0xf7, 0x73, 0xf6, 0x6b, 0x3f, 0x67, 0x5f, 0x7f, 0xcf, 0xef, 0x5d, + 0x0f, 0xf5, 0x5f, 0xe6, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x79, 0x70, 0xf4, 0x75, + 0x04, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 03cfdaabd..91c1f31e8 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -13,12 +13,12 @@ message Pair { } message Bit { - uint64 BitmapID = 1; - uint64 ProfileID = 2; + uint64 RowID = 1; + uint64 ColumnID = 2; int64 Timestamp = 3; } -message Profile { +message ColumnAttrSet { uint64 ID = 1; repeated Attr Attrs = 2; } @@ -39,7 +39,7 @@ message AttrMap { message QueryRequest { string Query = 1; repeated uint64 Slices = 2; - bool Profiles = 3; + bool ColumnAttrs = 3; string Quantum = 4; bool Remote = 5; } @@ -47,7 +47,7 @@ message QueryRequest { message QueryResponse { string Err = 1; repeated QueryResult Results = 2; - repeated Profile Profiles = 3; + repeated ColumnAttrSet ColumnAttrSets = 3; } message QueryResult { @@ -61,7 +61,7 @@ message ImportRequest { string DB = 1; string Frame = 2; uint64 Slice = 3; - repeated uint64 BitmapIDs = 4; - repeated uint64 ProfileIDs = 5; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; repeated int64 Timestamps = 6; } diff --git a/iterator.go b/iterator.go index 282cb32ed..155914491 100644 --- a/iterator.go +++ b/iterator.go @@ -6,19 +6,19 @@ import ( "github.com/pilosa/pilosa/roaring" ) -// Iterator is an interface for looping over bitmap/profile pairs. +// Iterator is an interface for looping over row/column pairs. type Iterator interface { - Seek(bitmapID, profileID uint64) - Next() (bitmapID, profileID uint64, eof bool) + Seek(rowID, columnID uint64) + Next() (rowID, columnID uint64, eof bool) } // BufIterator wraps an iterator to provide the ability to unread values. type BufIterator struct { buf struct { - bitmapID uint64 - profileID uint64 - eof bool - full bool + rowID uint64 + columnID uint64 + eof bool + full bool } itr Iterator } @@ -29,28 +29,28 @@ func NewBufIterator(itr Iterator) *BufIterator { } // Seek moves to the first pair equal to or greater than pseek/bseek. -func (itr *BufIterator) Seek(bitmapID, profileID uint64) { +func (itr *BufIterator) Seek(rowID, columnID uint64) { itr.buf.full = false - itr.itr.Seek(bitmapID, profileID) + itr.itr.Seek(rowID, columnID) } -// Next returns the next pair in the bitmap. +// Next returns the next pair in the row. // If a value has been buffered then it is returned and the buffer is cleared. -func (itr *BufIterator) Next() (bitmapID, profileID uint64, eof bool) { +func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) { if itr.buf.full { itr.buf.full = false - return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof + return itr.buf.rowID, itr.buf.columnID, itr.buf.eof } // Read values onto buffer in case of unread. - itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof = itr.itr.Next() + itr.buf.rowID, itr.buf.columnID, itr.buf.eof = itr.itr.Next() - return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof + return itr.buf.rowID, itr.buf.columnID, itr.buf.eof } // Peek reads the next value but leaves it on the buffer. -func (itr *BufIterator) Peek() (bitmapID, profileID uint64, eof bool) { - bitmapID, profileID, eof = itr.Next() +func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) { + rowID, columnID, eof = itr.Next() itr.Unread() return } @@ -64,30 +64,30 @@ func (itr *BufIterator) Unread() { itr.buf.full = true } -// LimitIterator wraps an Iterator and limits it to a max profile/bitmap pair. +// LimitIterator wraps an Iterator and limits it to a max column/row pair. type LimitIterator struct { - itr Iterator - maxBitmapID uint64 - maxProfileID uint64 + itr Iterator + maxRowID uint64 + maxColumnID uint64 eof bool } // NewLimitIterator returns a new LimitIterator. -func NewLimitIterator(itr Iterator, maxBitmapID, maxProfileID uint64) *LimitIterator { +func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator { return &LimitIterator{ - itr: itr, - maxBitmapID: maxBitmapID, - maxProfileID: maxProfileID, + itr: itr, + maxRowID: maxRowID, + maxColumnID: maxColumnID, } } -// Seek moves the underlying iterator to a profile/bitmap pair. -func (itr *LimitIterator) Seek(bitmapID, profileID uint64) { itr.itr.Seek(bitmapID, profileID) } +// Seek moves the underlying iterator to a column/row pair. +func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) } -// Next returns the next bitmap/profile ID pair. +// Next returns the next row/column ID pair. // If the underlying iterator returns a pair higher than the max then EOF is returned. -func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) { +func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) { // Always return EOF once it is reached by limit or the underlying iterator. if itr.eof { return 0, 0, true @@ -95,35 +95,35 @@ func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) { // Retrieve pair from underlying iterator. // Mark as EOF if it is beyond the limit (or at EOF). - bitmapID, profileID, eof = itr.itr.Next() - if eof || bitmapID > itr.maxBitmapID || (bitmapID == itr.maxBitmapID && profileID > itr.maxProfileID) { + rowID, columnID, eof = itr.itr.Next() + if eof || rowID > itr.maxRowID || (rowID == itr.maxRowID && columnID > itr.maxColumnID) { itr.eof = true return 0, 0, true } - return bitmapID, profileID, false + return rowID, columnID, false } -// SliceIterator iterates over a pair of bitmap/profile ID slices. +// SliceIterator iterates over a pair of row/column ID slices. type SliceIterator struct { - bitmapIDs []uint64 - profileIDs []uint64 + rowIDs []uint64 + columnIDs []uint64 i, n int } -// NewSliceIterator returns an iterator to iterate over a set of bitmap/profile ID pairs. +// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs. // Both slices MUST have an equal length. Otherwise the function will panic. -func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator { - if len(profileIDs) != len(bitmapIDs) { - panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(bitmapIDs), len(profileIDs))) +func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator { + if len(columnIDs) != len(rowIDs) { + panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs))) } return &SliceIterator{ - bitmapIDs: bitmapIDs, - profileIDs: profileIDs, + rowIDs: rowIDs, + columnIDs: columnIDs, - n: len(bitmapIDs), + n: len(rowIDs), } } @@ -131,10 +131,10 @@ func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator { // If the pair is not found, the iterator seeks to the next pair. func (itr *SliceIterator) Seek(bseek, pseek uint64) { for i := 0; i < itr.n; i++ { - bitmapID := itr.bitmapIDs[i] - profileID := itr.profileIDs[i] + rowID := itr.rowIDs[i] + columnID := itr.columnIDs[i] - if (bseek == bitmapID && pseek <= profileID) || bseek < bitmapID { + if (bseek == rowID && pseek <= columnID) || bseek < rowID { itr.i = i return } @@ -144,20 +144,20 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) { itr.i = itr.n } -// Next returns the next bitmap/profile ID pair. -func (itr *SliceIterator) Next() (bitmapID, profileID uint64, eof bool) { +// Next returns the next row/column ID pair. +func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) { if itr.i >= itr.n { return 0, 0, true } - bitmapID = itr.bitmapIDs[itr.i] - profileID = itr.profileIDs[itr.i] + rowID = itr.rowIDs[itr.i] + columnID = itr.columnIDs[itr.i] itr.i++ - return bitmapID, profileID, false + return rowID, columnID, false } -// RoaringIterator converts a roaring.Iterator to output profile/bitmap pairs. +// RoaringIterator converts a roaring.Iterator to output column/row pairs. type RoaringIterator struct { itr *roaring.Iterator } @@ -173,8 +173,8 @@ func (itr *RoaringIterator) Seek(bseek, pseek uint64) { itr.itr.Seek((bseek * SliceWidth) + pseek) } -// Next returns the next profile/bitmap ID pair. -func (itr *RoaringIterator) Next() (bitmapID, profileID uint64, eof bool) { +// Next returns the next column/row ID pair. +func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) { v, eof := itr.itr.Next() return v / SliceWidth, v % SliceWidth, eof } diff --git a/pilosa.go b/pilosa.go index b754c0e48..8ffb54155 100644 --- a/pilosa.go +++ b/pilosa.go @@ -35,54 +35,54 @@ var ( // Todo: remove . when frame doesn't require . for topN var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`) -// Profile represents vertical column in a database. -// A profile can have a set of attributes attached to it. -type Profile struct { +// ColumnAttrSet represents a set of attributes for a vertical column in a database. +// Can have a set of attributes attached to it. +type ColumnAttrSet struct { ID uint64 `json:"id"` Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeProfiles converts a into its internal representation. -func encodeProfiles(a []*Profile) []*internal.Profile { - other := make([]*internal.Profile, len(a)) +// encodeColumnAttrSets converts a into its internal representation. +func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = encodeProfile(a[i]) + other[i] = encodeColumnAttrSet(a[i]) } return other } -// decodeProfiles converts a from its internal representation. -func decodeProfiles(a []*internal.Profile) []*Profile { - other := make([]*Profile, len(a)) +// decodeColumnAttrSets converts a from its internal representation. +func decodeColumnAttrSets(a []*internal.ColumnAttrSet) []*ColumnAttrSet { + other := make([]*ColumnAttrSet, len(a)) for i := range a { - other[i] = decodeProfile(a[i]) + other[i] = decodeColumnAttrSet(a[i]) } return other } -// encodeProfile converts p into its internal representation. -func encodeProfile(p *Profile) *internal.Profile { - return &internal.Profile{ - ID: p.ID, - Attrs: encodeAttrs(p.Attrs), +// encodeColumnAttrSet converts set into its internal representation. +func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), } } -// decodeProfile converts b from its internal representation. -func decodeProfile(pb *internal.Profile) *Profile { - p := &Profile{ +// decodeColumnAttrSet converts b from its internal representation. +func decodeColumnAttrSet(pb *internal.ColumnAttrSet) *ColumnAttrSet { + set := &ColumnAttrSet{ ID: pb.ID, } if len(pb.Attrs) > 0 { - p.Attrs = make(map[string]interface{}, len(pb.Attrs)) + set.Attrs = make(map[string]interface{}, len(pb.Attrs)) for _, attr := range pb.Attrs { k, v := decodeAttr(attr) - p.Attrs[k] = v + set.Attrs[k] = v } } - return p + return set } // TimeFormat is the go-style time format used to parse string dates. diff --git a/server/server_test.go b/server/server_test.go index 8cd7c1d24..0f8a2f5d9 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -45,18 +45,18 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { + if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } // Validate data. for frame, frameSet := range SetCommands(cmds).Frames() { - for id, profileIDs := range frameSet { + for id, columnIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": profileIDs, + "bits": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -75,11 +75,11 @@ func TestMain_Set_Quick(t *testing.T) { // Validate data after reopening. for frame, frameSet := range SetCommands(cmds).Frames() { - for id, profileIDs := range frameSet { + for id, columnIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": profileIDs, + "bits": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -102,8 +102,8 @@ func TestMain_Set_Quick(t *testing.T) { } } -// Ensure program can set bitmap attributes and retrieve them. -func TestMain_SetBitmapAttrs(t *testing.T) { +// Ensure program can set row attributes and retrieve them. +func TestMain_SetRowAttrs(t *testing.T) { m := MustRunMain() defer m.Close() @@ -119,36 +119,36 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on different bitmaps in different frames. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { + // Set bits on different rows in different frames. + if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } - // Set bitmap attributes. - if _, err := m.Query("d", "", `SetBitmapAttrs(id=1, frame="x.n", x=100)`); err != nil { + // Set row attributes. + if _, err := m.Query("d", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="x.n", x=-200)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } - // Query bitmap x.n/1. + // Query row x.n/1. if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } - // Query bitmap x.n/2. + // Query row x.n/2. if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { @@ -159,19 +159,19 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } - // Query bitmaps after reopening. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query rows after reopening. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("d", "profiles=true", `Bitmap(id=3, frame="neg")`); err != nil { + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - // Query bitmap x.n/2. + // Query row x.n/2. if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { @@ -179,8 +179,8 @@ func TestMain_SetBitmapAttrs(t *testing.T) { } } -// Ensure program can set profile attributes and retrieve them. -func TestMain_SetProfileAttrs(t *testing.T) { +// Ensure program can set column attributes and retrieve them. +func TestMain_SetColumnAttrs(t *testing.T) { m := MustRunMain() defer m.Close() @@ -192,22 +192,22 @@ func TestMain_SetProfileAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on bitmap. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { + // Set bits on row. + if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=101)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil { t.Fatal(err) } - // Set profile attributes. - if _, err := m.Query("d", "", `SetProfileAttrs(id=100, foo="bar")`); err != nil { + // Set column attributes. + if _, err := m.Query("d", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { t.Fatal(err) } - // Query bitmap. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -215,16 +215,16 @@ func TestMain_SetProfileAttrs(t *testing.T) { t.Fatal(err) } - // Query bitmap after reopening. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row after reopening. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } -// Ensure program can set profile attributes with columnLabel option. -func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) { +// Ensure program can set column attributes with columnLabel option. +func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { m := MustRunMain() defer m.Close() @@ -236,22 +236,22 @@ func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) { t.Fatal(err) } - // Set bits on bitmap. + // Set bits on row. if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil { t.Fatal(err) } - // Set profile attributes. - if _, err := m.Query("d", "", `SetProfileAttrs(col=100, foo="bar")`); err != nil { + // Set column attributes. + if _, err := m.Query("d", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } - // Query bitmap. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -282,18 +282,18 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m0.Query("d", "", ` - SetBit(id=1, frame="f", profileID=100) - SetBit(id=1, frame="f", profileID=1000) - SetBit(id=1, frame="f", profileID=100000) - SetBit(id=1, frame="f", profileID=200000) - SetBit(id=1, frame="f", profileID=400000) - SetBit(id=1, frame="f", profileID=600000) - SetBit(id=1, frame="f", profileID=800000) + SetBit(id=1, frame="f", columnID=100) + SetBit(id=1, frame="f", columnID=1000) + SetBit(id=1, frame="f", columnID=100000) + SetBit(id=1, frame="f", columnID=200000) + SetBit(id=1, frame="f", columnID=400000) + SetBit(id=1, frame="f", columnID=600000) + SetBit(id=1, frame="f", columnID=800000) `); err != nil { t.Fatal(err) } - // Query bitmap on first cluster. + // Query row on first cluster. if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { @@ -316,7 +316,7 @@ func TestMain_FrameRestore(t *testing.T) { t.Fatal(err) } - // Query bitmap on second cluster. + // Query row on second cluster. if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { @@ -441,14 +441,14 @@ func (m *Main) Query(db, rawQuery, query string) (string, error) { // SetCommand represents a command to set a bit. type SetCommand struct { - ID uint64 - Frame string - ProfileID uint64 + ID uint64 + Frame string + ColumnID uint64 } type SetCommands []SetCommand -// Frames returns the set of profile ids for each frame/bitmap. +// Frames returns the set of column ids for each frame/row. func (a SetCommands) Frames() map[string]map[uint64][]uint64 { // Create a set of unique commands. m := make(map[SetCommand]struct{}) @@ -456,16 +456,16 @@ func (a SetCommands) Frames() map[string]map[uint64][]uint64 { m[cmd] = struct{}{} } - // Build unique ids for each frame & bitmap. + // Build unique ids for each frame & row. frames := make(map[string]map[uint64][]uint64) for cmd := range m { if frames[cmd.Frame] == nil { frames[cmd.Frame] = make(map[uint64][]uint64) } - frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ProfileID) + frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID) } - // Sort each set of profile ids. + // Sort each set of column ids. for _, frame := range frames { for id := range frame { sort.Sort(uint64Slice(frame[id])) @@ -480,9 +480,9 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { cmds := make([]SetCommand, rand.Intn(n)) for i := range cmds { cmds[i] = SetCommand{ - ID: uint64(rand.Intn(1000)), - Frame: "x.n", - ProfileID: uint64(rand.Intn(10)), + ID: uint64(rand.Intn(1000)), + Frame: "x.n", + ColumnID: uint64(rand.Intn(10)), } } return cmds diff --git a/view.go b/view.go index c7ec92733..4fa096943 100644 --- a/view.go +++ b/view.go @@ -38,8 +38,8 @@ type View struct { stats StatsClient - BitmapAttrStore *AttrStore - LogOutput io.Writer + RowAttrStore *AttrStore + LogOutput io.Writer } // NewView returns a new instance of View. @@ -124,7 +124,7 @@ func (v *View) openFragments() error { if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) } - frag.BitmapAttrStore = v.BitmapAttrStore + frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag v.stats.Count("maxSlice", 1) @@ -205,7 +205,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { if err := frag.Open(); err != nil { return nil, err } - frag.BitmapAttrStore = v.BitmapAttrStore + frag.RowAttrStore = v.RowAttrStore // Save to lookup. v.fragments[slice] = frag @@ -225,23 +225,23 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { } // SetBit sets a bit within the view. -func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth +func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { + slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetBit(bitmapID, profileID) + return frag.SetBit(rowID, columnID) } // ClearBit clears a bit within the view. -func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth +func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { + slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.ClearBit(bitmapID, profileID) + return frag.ClearBit(rowID, columnID) } // IsInverseView returns true if the view is used for storing an inverted representation. diff --git a/view_test.go b/view_test.go index d10c90936..c531e1fce 100644 --- a/view_test.go +++ b/view_test.go @@ -10,7 +10,7 @@ import ( // View is a test wrapper for pilosa.View. type View struct { *pilosa.View - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore } // NewView returns a new instance of View with a temporary path. @@ -22,10 +22,10 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), - BitmapAttrStore: MustOpenAttrStore(), + View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), + RowAttrStore: MustOpenAttrStore(), } - v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore.AttrStore return v } @@ -41,7 +41,7 @@ func MustOpenView(db, frame, name string) *View { // Close closes the view and removes all underlying data. func (v *View) Close() error { defer os.Remove(v.Path()) - defer v.BitmapAttrStore.Close() + defer v.RowAttrStore.Close() return v.View.Close() } @@ -53,27 +53,27 @@ func (v *View) Reopen() error { } v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore.AttrStore if err := v.Open(); err != nil { return err } return nil } -// MustSetBits sets bits on a bitmap. Panic on error. +// MustSetBits sets bits on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := v.SetBit(bitmapID, profileID); err != nil { +func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := v.SetBit(rowID, columnID); err != nil { panic(err) } } } -// MustClearBits clears bits on a bitmap. Panic on error. -func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := v.ClearBit(bitmapID, profileID); err != nil { +// MustClearBits clears bits on a row. Panic on error. +func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := v.ClearBit(rowID, columnID); err != nil { panic(err) } } From 2cacff8e0937371685dfddcd3aaae30cfcdd54d0 Mon Sep 17 00:00:00 2001 From: Travis Date: Sun, 23 Apr 2017 15:52:51 -0500 Subject: [PATCH 56/63] Rename `Index` to `Holder` Rename `index.go` to `holder.go` and `index_test.go` to `holder_test.go` --- client_test.go | 122 ++++++------- executor.go | 30 +-- executor_test.go | 312 ++++++++++++++++---------------- handler.go | 59 +++--- handler_test.go | 122 ++++++------- index.go => holder.go | 192 ++++++++++---------- index_test.go => holder_test.go | 128 ++++++------- server.go | 66 +++---- server/server.go | 6 +- 9 files changed, 519 insertions(+), 518 deletions(-) rename index.go => holder.go (68%) rename index_test.go => holder_test.go (57%) diff --git a/client_test.go b/client_test.go index a05032d57..858658928 100644 --- a/client_test.go +++ b/client_test.go @@ -13,81 +13,81 @@ import ( "github.com/pilosa/pilosa/pql" ) -func createCluster(c *pilosa.Cluster) ([]*Server, []*Index) { +func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) { numNodes := len(c.Nodes) - idx := make([]*Index, numNodes) + hldr := make([]*Holder, numNodes) server := make([]*Server, numNodes) for i := 0; i < numNodes; i++ { - idx[i] = MustOpenIndex() + hldr[i] = MustOpenHolder() server[i] = NewServer() server[i].Handler.Host = server[i].Host() server[i].Handler.Cluster = c server[i].Handler.Cluster.Nodes[i].Host = server[i].Host() - server[i].Handler.Index = idx[i].Index + server[i].Handler.Holder = hldr[i].Holder } - return server, idx + return server, hldr } // Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { cluster := NewCluster(3) - s, idx := createCluster(cluster) + s, hldr := createCluster(cluster) for i := 0; i < len(cluster.Nodes); i++ { - defer idx[i].Close() + defer hldr[i].Close() defer s[i].Close() } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() - e.Index = idx[0].Index + e.Holder = hldr[0].Holder e.Host = cluster.Nodes[0].Host e.Cluster = cluster return e.Execute(ctx, db, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() - e.Index = idx[1].Index + e.Holder = hldr[1].Holder e.Host = cluster.Nodes[1].Host e.Cluster = cluster return e.Execute(ctx, db, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() - e.Index = idx[2].Index + e.Holder = hldr[2].Holder e.Host = cluster.Nodes[2].Host e.Cluster = cluster return e.Execute(ctx, db, query, slices, opt) } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4) - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4) + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache() - idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache() - idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache() + hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache() + hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache() + hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -112,9 +112,9 @@ func TestClient_MultiNode(t *testing.T) { } // Set max slice to correct value. - idx[0].DB("d").SetRemoteMaxSlice(10) - idx[1].DB("d").SetRemoteMaxSlice(10) - idx[2].DB("d").SetRemoteMaxSlice(10) + hldr[0].DB("d").SetRemoteMaxSlice(10) + hldr[1].DB("d").SetRemoteMaxSlice(10) + hldr[2].DB("d").SetRemoteMaxSlice(10) result, err = client[0].ExecuteQuery(context.Background(), "d", q, true) if err != nil { @@ -157,11 +157,11 @@ func TestClient_MultiNode(t *testing.T) { // Ensure client can bulk import data. func TestClient_Import(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. - f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f.Row(0) s := NewServer() @@ -169,7 +169,7 @@ func TestClient_Import(t *testing.T) { s.Handler.Host = s.Host() s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder // Send import request. c := MustNewClient(s.Host()) @@ -192,10 +192,10 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import data to an inverse frame. func TestClient_ImportInverseEnabled(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) frameOpts := pilosa.FrameOptions{ InverseEnabled: true, } @@ -220,7 +220,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s.Handler.Host = s.Host() s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder // Send import request. c := MustNewClient(s.Host()) @@ -247,20 +247,20 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Ensure client backup and restore a frame. func TestClient_BackupRestore(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) s := NewServer() defer s.Close() s.Handler.Host = s.Host() s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder c := MustNewClient(s.Host()) @@ -271,7 +271,7 @@ func TestClient_BackupRestore(t *testing.T) { } // Restore to a different frame. - if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { @@ -279,38 +279,38 @@ func TestClient_BackupRestore(t *testing.T) { } // Verify data. - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { t.Fatalf("unexpected bits: %+v", a) } } // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := NewServer() defer s.Close() s.Handler.Host = s.Host() s.Handler.Cluster = NewCluster(1) s.Handler.Cluster.Nodes[0].Host = s.Host() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder // Retrieve blocks. c := MustNewClient(s.Host()) @@ -326,7 +326,7 @@ func TestClient_FragmentBlocks(t *testing.T) { } // Verify data matches local blocks. - if a := idx.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { + if a := hldr.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } diff --git a/executor.go b/executor.go index 6bf3395e6..11ffcee47 100644 --- a/executor.go +++ b/executor.go @@ -27,7 +27,7 @@ const ( // Executor recursively executes calls in a PQL query across all slices. type Executor struct { - Index *Index + Holder *Holder // Local hostname & cluster configuration. Host string @@ -60,7 +60,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices if len(slices) == 0 { if needsSlices(q.Calls) { // Round up the number of slices. - maxSlice := e.Index.DB(db).MaxSlice() + maxSlice := e.Holder.DB(db).MaxSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) @@ -160,7 +160,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { - d := e.Index.DB(db) + d := e.Holder.DB(db) if d != nil { columnLabel := d.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { @@ -315,7 +315,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, frame = DefaultFrame } - f := e.Index.Fragment(db, frame, ViewStandard, slice) + f := e.Holder.Fragment(db, frame, ViewStandard, slice) if f == nil { return nil, nil } @@ -362,7 +362,7 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { // Fetch column label from database. - d := e.Index.DB(db) + d := e.Holder.DB(db) if d == nil { return nil, ErrDatabaseNotFound } @@ -373,7 +373,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal if frame == "" { frame = DefaultFrame } - f := e.Index.Frame(db, frame) + f := e.Holder.Frame(db, frame) if f == nil { return nil, ErrFrameNotFound } @@ -400,7 +400,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal } } - frag := e.Index.Fragment(db, frame, view, slice) + frag := e.Holder.Fragment(db, frame, view, slice) if frag == nil { return NewBitmap(), nil } @@ -438,7 +438,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call } // Retrieve base frame. - f := e.Index.Frame(db, frame) + f := e.Holder.Frame(db, frame) if f == nil { return nil, ErrFrameNotFound } @@ -479,7 +479,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call // Union bitmaps across all time-based subframes. bm := &Bitmap{} for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Index.Fragment(db, frame, view, slice) + f := e.Holder.Fragment(db, frame, view, slice) if f == nil { continue } @@ -548,7 +548,7 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, } // Retrieve frame. - d := e.Index.DB(db) + d := e.Holder.DB(db) if d == nil { return false, ErrDatabaseNotFound } @@ -642,7 +642,7 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op } // Retrieve frame. - d := e.Index.DB(db) + d := e.Holder.DB(db) if d == nil { return false, ErrDatabaseNotFound } @@ -747,7 +747,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Cal } // Retrieve frame. - frame := e.Index.Frame(db, frameName) + frame := e.Holder.Frame(db, frameName) if frame == nil { return ErrFrameNotFound } @@ -807,7 +807,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls } // Retrieve frame. - f := e.Index.Frame(db, frame) + f := e.Holder.Frame(db, frame) if f == nil { return nil, ErrFrameNotFound } @@ -846,7 +846,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls // Bulk insert attributes by frame. for name, frameMap := range m { // Retrieve frame. - frame := e.Index.Frame(db, name) + frame := e.Holder.Frame(db, name) if frame == nil { return nil, ErrFrameNotFound } @@ -886,7 +886,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls // executeSetColumnAttrs executes a SetColumnAttrs() call. func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { // Retrieve database. - d := e.Index.DB(db) + d := e.Holder.DB(db) if d == nil { return ErrDatabaseNotFound } diff --git a/executor_test.go b/executor_test.go index 97a89f3ae..0be31c73b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -15,15 +15,15 @@ import ( // Ensure a bitmap query can be executed. func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + hldr := MustOpenHolder() + defer hldr.Close() + db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) f, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ @@ -47,14 +47,14 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { }) t.Run("Column", func(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + hldr := MustOpenHolder() + defer hldr.Close() + db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) if _, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ @@ -80,15 +80,15 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Ensure a difference query can be executed. func TestExecutor_Execute_Difference(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { @@ -98,11 +98,11 @@ func TestExecutor_Execute_Difference(t *testing.T) { // Ensure an empty difference query behaves properly. func TestExecutor_Execute_Empty_Difference(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } @@ -110,17 +110,17 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { // Ensure an intersect query can be executed. func TestExecutor_Execute_Intersect(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { @@ -130,10 +130,10 @@ func TestExecutor_Execute_Intersect(t *testing.T) { // Ensure an empty intersect query behaves properly. func TestExecutor_Execute_Empty_Intersect(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } @@ -141,16 +141,16 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { // Ensure a union query can be executed. func TestExecutor_Execute_Union(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { @@ -160,11 +160,11 @@ func TestExecutor_Execute_Union(t *testing.T) { // Ensure an empty union query behaves properly. func TestExecutor_Execute_Empty_Union(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { @@ -174,13 +174,13 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { @@ -190,11 +190,11 @@ func TestExecutor_Execute_Count(t *testing.T) { // Ensure a set query can be executed. func TestExecutor_Execute_SetBit(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - e := NewExecutor(idx.Index, NewCluster(1)) - f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + e := NewExecutor(hldr.Holder, NewCluster(1)) + f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } @@ -221,11 +221,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) { // Ensure a SetRowAttrs() query can be executed. func TestExecutor_Execute_SetRowAttrs(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Create frames. - db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { @@ -234,7 +234,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } @@ -248,7 +248,7 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Fatal(err) } - f := idx.Frame("d", "f") + f := hldr.Frame("d", "f") if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { @@ -258,22 +258,22 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) // Execute query. - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ @@ -284,19 +284,19 @@ func TestExecutor_Execute_TopN(t *testing.T) { } } func TestExecutor_Execute_TopN_fill(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -308,29 +308,29 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Ensure func TestExecutor_Execute_TopN_fill_small(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -342,26 +342,26 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Ensure a TopN() query with a source bitmap can be executed. func TestExecutor_Execute_TopN_Src(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) // Create an intersecting row. - idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) + hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) // Execute query. - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -376,16 +376,16 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { //Ensure TopN handles Attribute filters func TestExecutor_Execute_TopN_Attr(t *testing.T) { // - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -399,16 +399,16 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { //Ensure TopN handles Attribute filters with source bitmap func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ @@ -421,11 +421,11 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Range(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Create database. - db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) // Create frame. f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) @@ -447,7 +447,7 @@ func TestExecutor_Execute_Range(t *testing.T) { f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row - e := NewExecutor(idx.Index, NewCluster(1)) + e := NewExecutor(hldr.Holder, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { @@ -487,11 +487,11 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // Create local executor data. // The local node owns slice 1. - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) - e := NewExecutor(idx.Index, c) + e := NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) { @@ -514,12 +514,12 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { } // Create local executor data. The local node owns slice 1. - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) - e := NewExecutor(idx.Index, c) + e := NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { @@ -550,21 +550,21 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } // Create local executor data. - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Create frame. - if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, c) + e := NewExecutor(hldr.Holder, c) if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } - // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { + // Verify that one bit is set on both node's holder. + if n := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -595,23 +595,23 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } // Create local executor data. - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Create frame. - if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if f, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) } - e := NewExecutor(idx.Index, c) + e := NewExecutor(hldr.Holder, c) if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } - // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + // Verify that one bit is set on both node's holder. + if n := hldr.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -662,12 +662,12 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { } // Create local executor data on slice 1 & 3. - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) - e := NewExecutor(idx.Index, c) + e := NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ @@ -686,9 +686,9 @@ type Executor struct { // NewExecutor returns a new instance of Executor. // The executor always matches the hostname of the first cluster node. -func NewExecutor(index *pilosa.Index, cluster *pilosa.Cluster) *Executor { +func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { e := &Executor{Executor: pilosa.NewExecutor()} - e.Index = index + e.Holder = holder e.Cluster = cluster e.Host = cluster.Nodes[0].Host return e diff --git a/handler.go b/handler.go index 7040a05ab..e3d4d833b 100644 --- a/handler.go +++ b/handler.go @@ -17,16 +17,17 @@ import ( "strings" "time" + "reflect" + "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "reflect" ) // Handler represents an HTTP handler. type Handler struct { - Index *Index + Holder *Holder Broadcaster Broadcaster // Local hostname & cluster configuration. @@ -107,7 +108,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSchemaResponse{ - DBs: h.Index.Schema(), + DBs: h.Holder.Schema(), }); err != nil { h.logger().Printf("write schema response error: %s", err) } @@ -172,7 +173,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Retrieve column attributes across all calls. - columnAttrSets, err := h.readColumnAttrSets(h.Index.DB(dbName), columnIDs) + columnAttrSets, err := h.readColumnAttrSets(h.Holder.DB(dbName), columnIDs) if err != nil { w.WriteHeader(http.StatusInternalServerError) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) @@ -195,9 +196,9 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { var ms map[string]uint64 if inverse, _ := strconv.ParseBool(r.URL.Query().Get("inverse")); inverse { - ms = h.Index.MaxInverseSlices() + ms = h.Holder.MaxInverseSlices() } else { - ms = h.Index.MaxSlices() + ms = h.Holder.MaxSlices() } if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { pb := &internal.MaxSlicesResponse{ @@ -226,7 +227,7 @@ func (h *Handler) handleGetDBs(w http.ResponseWriter, r *http.Request) { // handleGetDB handles GET /db/ requests. func (h *Handler) handleGetDB(w http.ResponseWriter, r *http.Request) { dbName := mux.Vars(r)["db"] - db := h.Index.DB(dbName) + db := h.Holder.DB(dbName) if db == nil { http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return @@ -311,8 +312,8 @@ type postDBResponse struct{} func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { dbName := mux.Vars(r)["db"] - // Delete database from the index. - if err := h.Index.DeleteDB(dbName); err != nil { + // Delete database from the holder. + if err := h.Holder.DeleteDB(dbName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -350,7 +351,7 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { } // Create database. - _, err = h.Index.CreateDB(dbName, req.Options) + _, err = h.Holder.CreateDB(dbName, req.Options) if err == ErrDatabaseExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -394,7 +395,7 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques } // Retrieve database by name. - database := h.Index.DB(dbName) + database := h.Holder.DB(dbName) if database == nil { http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return @@ -429,8 +430,8 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { return } - // Retrieve database from index. - db := h.Index.DB(dbName) + // Retrieve database from holder. + db := h.Holder.DB(dbName) if db == nil { http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return @@ -492,7 +493,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { } // Find database. - db := h.Index.DB(dbName) + db := h.Holder.DB(dbName) if db == nil { http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return @@ -576,7 +577,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { frameName := mux.Vars(r)["frame"] // Find database. - db := h.Index.DB(dbName) + db := h.Holder.DB(dbName) if db == nil { if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) @@ -628,7 +629,7 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req } // Retrieve database by name. - f := h.Index.Frame(dbName, frameName) + f := h.Holder.Frame(dbName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -658,7 +659,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { frameName := mux.Vars(r)["frame"] // Retrieve views. - f := h.Index.Frame(dbName, frameName) + f := h.Holder.Frame(dbName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -693,8 +694,8 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve database from index. - f := h.Index.Frame(dbName, frameName) + // Retrieve database from holder. + f := h.Holder.Frame(dbName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -891,7 +892,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Find the DB. h.logger().Println("importing:", req.DB, req.Frame, req.Slice) - db := h.Index.DB(req.DB) + db := h.Holder.DB(req.DB) if db == nil { h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error()) http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) @@ -956,7 +957,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Find the fragment. - f := h.Index.Fragment(db, frame, view, slice) + f := h.Holder.Fragment(db, frame, view, slice) if f == nil { return } @@ -1010,8 +1011,8 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) return } - // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) + // Retrieve fragment from holder. + f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1034,7 +1035,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve frame. - f := h.Index.Frame(q.Get("db"), q.Get("frame")) + f := h.Holder.Frame(q.Get("db"), q.Get("frame")) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -1073,8 +1074,8 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ return } - // Retrieve fragment from index. - f := h.Index.Fragment(req.DB, req.Frame, req.View, req.Slice) + // Retrieve fragment from holder. + f := h.Holder.Fragment(req.DB, req.Frame, req.View, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -1109,8 +1110,8 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - // Retrieve fragment from index. - f := h.Index.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) + // Retrieve fragment from holder. + f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1160,7 +1161,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Retrieve frame. - f := h.Index.Frame(dbName, frameName) + f := h.Holder.Frame(dbName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return diff --git a/handler_test.go b/handler_test.go index af4da1dd7..77435fb20 100644 --- a/handler_test.go +++ b/handler_test.go @@ -31,11 +31,11 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can return the schema. func TestHandler_Schema(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) - d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) + d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + d1 := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) @@ -54,7 +54,7 @@ func TestHandler_Schema(t *testing.T) { } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { @@ -66,19 +66,19 @@ func TestHandler_Schema(t *testing.T) { // Ensure the handler can return the maxslice map. func TestHandler_MaxSlices(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { @@ -90,10 +90,10 @@ func TestHandler_MaxSlices(t *testing.T) { // Ensure the handler can return the maxslice map for the inverse views. func TestHandler_MaxSlices_Inverse(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() - f0, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) + f0, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } @@ -105,7 +105,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { t.Fatal(err) } - f1, err := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) + f1, err := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } @@ -118,7 +118,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { @@ -255,11 +255,11 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap with column attributes as JSON. func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { - idx := NewIndex() - defer idx.Close() + hldr := NewHolder() + defer hldr.Close() // Create database and set column attributes. - db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) + db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { @@ -269,7 +269,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -320,11 +320,11 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf. func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { - idx := NewIndex() - defer idx.Close() + hldr := NewHolder() + defer hldr.Close() // Create database and set column attributes. - db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) + db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { @@ -332,7 +332,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -491,15 +491,15 @@ func TestHandler_Query_ErrParse(t *testing.T) { // Ensure the handler can delete a database. func TestHandler_DB_Delete(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() s := NewServer() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder defer s.Close() // Create database. - if _, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil { + if _, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil { t.Fatal(err) } @@ -520,86 +520,86 @@ func TestHandler_DB_Delete(t *testing.T) { } // Verify database is gone. - if idx.DB("d") != nil { + if hldr.DB("d") != nil { t.Fatal("expected nil database") } } // Ensure handler can delete a frame. func TestHandler_DeleteFrame(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + hldr := MustOpenHolder() + defer hldr.Close() + d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/db/d0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if f := idx.DB("d0").Frame("f1"); f != nil { + } else if f := hldr.DB("d0").Frame("f1"); f != nil { t.Fatal("expected nil frame") } } // Ensure handler can set the DB time quantum. func TestHandler_SetDBTimeQuantum(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() - idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + hldr := MustOpenHolder() + defer hldr.Close() + hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if q := idx.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + } else if q := hldr.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { t.Fatalf("unexpected time quantum: %s", q) } } // Ensure handler can set the frame time quantum. func TestHandler_SetFrameTimeQuantum(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() // Create frame. - if _, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } h := NewHandler() - h.Index = idx.Index + h.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if q := idx.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + } else if q := hldr.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { t.Fatalf("unexpected time quantum: %s", q) } } // Ensure the handler can return data in differing blocks for a database. func TestHandler_DB_AttrStore_Diff(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() s := NewServer() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder defer s.Close() // Set attributes on the database. - db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) + db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } @@ -640,15 +640,15 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { // Ensure the handler can return data in differing blocks for a frame. func TestHandler_Frame_AttrStore_Diff(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() s := NewServer() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder defer s.Close() // Set attributes on the database. - d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) f, err := d.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) @@ -690,15 +690,15 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Ensure the handler can backup a fragment and then restore it. func TestHandler_Fragment_BackupRestore(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() + hldr := MustOpenHolder() + defer hldr.Close() s := NewServer() - s.Handler.Index = idx.Index + s.Handler.Holder = hldr.Holder defer s.Close() // Set bits in the index. - f0 := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f0 := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) // Begin backing up from slice d/f/0. @@ -714,7 +714,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Create frame. - if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -729,7 +729,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Verify data is correctly restored. - f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0) + f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { t.Fatal("fragment x/y/standard/0 not created") } else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { diff --git a/index.go b/holder.go similarity index 68% rename from index.go rename to holder.go index 93228bb14..a70062150 100644 --- a/index.go +++ b/holder.go @@ -16,8 +16,8 @@ import ( // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. const DefaultCacheFlushInterval = 1 * time.Minute -// Index represents a container for fragments. -type Index struct { +// Holder represents a container for indexes. +type Holder struct { mu sync.Mutex // Databases by name. @@ -40,9 +40,9 @@ type Index struct { LogOutput io.Writer } -// NewIndex returns a new instance of Index. -func NewIndex() *Index { - return &Index{ +// NewHolder returns a new instance of Holder. +func NewHolder() *Holder { + return &Holder{ dbs: make(map[string]*DB), closing: make(chan struct{}, 0), @@ -54,14 +54,14 @@ func NewIndex() *Index { } } -// Open initializes the root data directory for the index. -func (i *Index) Open() error { - if err := os.MkdirAll(i.Path, 0777); err != nil { +// Open initializes the root data directory for the holder. +func (h *Holder) Open() error { + if err := os.MkdirAll(h.Path, 0777); err != nil { return err } // Open path to read all database directories. - f, err := os.Open(i.Path) + f, err := os.Open(h.Path) if err != nil { return err } @@ -77,68 +77,68 @@ func (i *Index) Open() error { continue } - i.logger().Printf("opening database: %s", filepath.Base(fi.Name())) + h.logger().Printf("opening database: %s", filepath.Base(fi.Name())) - db, err := i.newDB(i.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + db, err := h.newDB(h.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - i.logger().Printf("ERROR opening database: %s, err=%s", fi.Name(), err) + h.logger().Printf("ERROR opening database: %s, err=%s", fi.Name(), err) continue } else if err != nil { return err } if err := db.Open(); err != nil { if err == ErrName { - i.logger().Printf("ERROR opening database: %s, err=%s", db.Name(), err) + h.logger().Printf("ERROR opening database: %s, err=%s", db.Name(), err) continue } return fmt.Errorf("open db: name=%s, err=%s", db.Name(), err) } - i.dbs[db.Name()] = db + h.dbs[db.Name()] = db - i.Stats.Count("dbN", 1) + h.Stats.Count("dbN", 1) } // Periodically flush cache. - i.wg.Add(1) - go func() { defer i.wg.Done(); i.monitorCacheFlush() }() + h.wg.Add(1) + go func() { defer h.wg.Done(); h.monitorCacheFlush() }() return nil } // Close closes all open fragments. -func (i *Index) Close() error { +func (h *Holder) Close() error { // Notify goroutines of closing and wait for completion. - close(i.closing) - i.wg.Wait() + close(h.closing) + h.wg.Wait() - for _, db := range i.dbs { + for _, db := range h.dbs { db.Close() } return nil } // MaxSlices returns MaxSlice map for all databases. -func (i *Index) MaxSlices() map[string]uint64 { +func (h *Holder) MaxSlices() map[string]uint64 { a := make(map[string]uint64) - for _, db := range i.DBs() { + for _, db := range h.DBs() { a[db.Name()] = db.MaxSlice() } return a } // MaxInverseSlices returns MaxInverseSlice map for all databases. -func (i *Index) MaxInverseSlices() map[string]uint64 { +func (h *Holder) MaxInverseSlices() map[string]uint64 { a := make(map[string]uint64) - for _, db := range i.DBs() { + for _, db := range h.DBs() { a[db.Name()] = db.MaxInverseSlice() } return a } // Schema returns schema data for all databases and frames. -func (i *Index) Schema() []*DBInfo { +func (h *Holder) Schema() []*DBInfo { var a []*DBInfo - for _, db := range i.DBs() { + for _, db := range h.DBs() { di := &DBInfo{Name: db.Name()} for _, frame := range db.Frames() { fi := &FrameInfo{Name: frame.Name()} @@ -156,24 +156,24 @@ func (i *Index) Schema() []*DBInfo { } // DBPath returns the path where a given database is stored. -func (i *Index) DBPath(name string) string { return filepath.Join(i.Path, name) } +func (h *Holder) DBPath(name string) string { return filepath.Join(h.Path, name) } // DB returns the database by name. -func (i *Index) DB(name string) *DB { - i.mu.Lock() - defer i.mu.Unlock() - return i.db(name) +func (h *Holder) DB(name string) *DB { + h.mu.Lock() + defer h.mu.Unlock() + return h.db(name) } -func (i *Index) db(name string) *DB { return i.dbs[name] } +func (h *Holder) db(name string) *DB { return h.dbs[name] } -// DBs returns a list of all databases in the index. -func (i *Index) DBs() []*DB { - i.mu.Lock() - defer i.mu.Unlock() +// DBs returns a list of all databases in the holder. +func (h *Holder) DBs() []*DB { + h.mu.Lock() + defer h.mu.Unlock() - a := make([]*DB, 0, len(i.dbs)) - for _, db := range i.dbs { + a := make([]*DB, 0, len(h.dbs)) + for _, db := range h.dbs { a = append(a, db) } sort.Sort(dbSlice(a)) @@ -183,43 +183,43 @@ func (i *Index) DBs() []*DB { // CreateDB creates a database. // An error is returned if the database already exists. -func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { - i.mu.Lock() - defer i.mu.Unlock() +func (h *Holder) CreateDB(name string, opt DBOptions) (*DB, error) { + h.mu.Lock() + defer h.mu.Unlock() // Ensure db doesn't already exist. - if i.dbs[name] != nil { + if h.dbs[name] != nil { return nil, ErrDatabaseExists } - return i.createDB(name, opt) + return h.createDB(name, opt) } // CreateDBIfNotExists returns a database by name. // The database is created if it does not already exist. -func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { - i.mu.Lock() - defer i.mu.Unlock() +func (h *Holder) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { + h.mu.Lock() + defer h.mu.Unlock() // Find database in cache first. - if db := i.dbs[name]; db != nil { + if db := h.dbs[name]; db != nil { return db, nil } - return i.createDB(name, opt) + return h.createDB(name, opt) } -func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { +func (h *Holder) createDB(name string, opt DBOptions) (*DB, error) { if name == "" { return nil, errors.New("database name required") } // Return database if it exists. - if db := i.db(name); db != nil { + if db := h.db(name); db != nil { return db, nil } // Otherwise create a new database. - db, err := i.newDB(i.DBPath(name), name) + db, err := h.newDB(h.DBPath(name), name) if err != nil { return nil, err } @@ -232,31 +232,31 @@ func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { db.SetColumnLabel(opt.ColumnLabel) db.SetTimeQuantum(opt.TimeQuantum) - i.dbs[db.Name()] = db + h.dbs[db.Name()] = db - i.Stats.Count("dbN", 1) + h.Stats.Count("dbN", 1) return db, nil } -func (i *Index) newDB(path, name string) (*DB, error) { +func (h *Holder) newDB(path, name string) (*DB, error) { db, err := NewDB(path, name) if err != nil { return nil, err } - db.LogOutput = i.LogOutput - db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.broadcaster = i.Broadcaster + db.LogOutput = h.LogOutput + db.stats = h.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) + db.broadcaster = h.Broadcaster return db, nil } -// DeleteDB removes a database from the index. -func (i *Index) DeleteDB(name string) error { - i.mu.Lock() - defer i.mu.Unlock() +// DeleteDB removes a database from the holder. +func (h *Holder) DeleteDB(name string) error { + h.mu.Lock() + defer h.mu.Unlock() // Ignore if database doesn't exist. - db := i.db(name) + db := h.db(name) if db == nil { return nil } @@ -267,21 +267,21 @@ func (i *Index) DeleteDB(name string) error { } // Delete database directory. - if err := os.RemoveAll(i.DBPath(name)); err != nil { + if err := os.RemoveAll(h.DBPath(name)); err != nil { return err } // Remove reference. - delete(i.dbs, name) + delete(h.dbs, name) - i.Stats.Count("dbN", -1) + h.Stats.Count("dbN", -1) return nil } // Frame returns the frame for a database and name. -func (i *Index) Frame(db, name string) *Frame { - d := i.DB(db) +func (h *Holder) Frame(db, name string) *Frame { + d := h.DB(db) if d == nil { return nil } @@ -289,8 +289,8 @@ func (i *Index) Frame(db, name string) *Frame { } // View returns the view for a database, frame, and name. -func (i *Index) View(db, frame, name string) *View { - f := i.Frame(db, frame) +func (h *Holder) View(db, frame, name string) *View { + f := h.Frame(db, frame) if f == nil { return nil } @@ -298,8 +298,8 @@ func (i *Index) View(db, frame, name string) *View { } // Fragment returns the fragment for a database, frame & slice. -func (i *Index) Fragment(db, frame, view string, slice uint64) *Fragment { - v := i.View(db, frame, view) +func (h *Holder) Fragment(db, frame, view string, slice uint64) *Fragment { + v := h.View(db, frame, view) if v == nil { return nil } @@ -308,33 +308,33 @@ func (i *Index) Fragment(db, frame, view string, slice uint64) *Fragment { // monitorCacheFlush periodically flushes all fragment caches sequentially. // This is run in a goroutine. -func (i *Index) monitorCacheFlush() { - ticker := time.NewTicker(i.CacheFlushInterval) +func (h *Holder) monitorCacheFlush() { + ticker := time.NewTicker(h.CacheFlushInterval) defer ticker.Stop() for { select { - case <-i.closing: + case <-h.closing: return case <-ticker.C: - i.flushCaches() + h.flushCaches() } } } -func (i *Index) flushCaches() { - for _, db := range i.DBs() { +func (h *Holder) flushCaches() { + for _, db := range h.DBs() { for _, frame := range db.Frames() { for _, view := range frame.Views() { for _, fragment := range view.Fragments() { select { - case <-i.closing: + case <-h.closing: return default: } if err := fragment.FlushCache(); err != nil { - i.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) } } } @@ -342,12 +342,12 @@ func (i *Index) flushCaches() { } } -func (i *Index) logger() *log.Logger { return log.New(i.LogOutput, "", log.LstdFlags) } +func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } -// IndexSyncer is an active anti-entropy tool that compares the local index -// with a remote index based on block checksums and resolves differences. -type IndexSyncer struct { - Index *Index +// HolderSyncer is an active anti-entropy tool that compares the local holder +// with a remote holder based on block checksums and resolves differences. +type HolderSyncer struct { + Holder *Holder Host string Cluster *Cluster @@ -357,7 +357,7 @@ type IndexSyncer struct { } // Returns true if the syncer has been marked to close. -func (s *IndexSyncer) IsClosing() bool { +func (s *HolderSyncer) IsClosing() bool { select { case <-s.Closing: return true @@ -366,10 +366,10 @@ func (s *IndexSyncer) IsClosing() bool { } } -// SyncIndex compares the index on host with the local index and resolves differences. -func (s *IndexSyncer) SyncIndex() error { +// SyncHolder compares the holder on host with the local holder and resolves differences. +func (s *HolderSyncer) SyncHolder() error { // Iterate over schema in sorted order. - for _, di := range s.Index.Schema() { + for _, di := range s.Holder.Schema() { // Verify syncer has not closed. if s.IsClosing() { return nil @@ -397,7 +397,7 @@ func (s *IndexSyncer) SyncIndex() error { return nil } - for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { + for slice := uint64(0); slice <= s.Holder.DB(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { continue @@ -421,9 +421,9 @@ func (s *IndexSyncer) SyncIndex() error { } // syncDatabase synchronizes database attributes with the rest of the cluster. -func (s *IndexSyncer) syncDatabase(db string) error { +func (s *HolderSyncer) syncDatabase(db string) error { // Retrieve database reference. - d := s.Index.DB(db) + d := s.Holder.DB(db) if d == nil { return nil } @@ -466,9 +466,9 @@ func (s *IndexSyncer) syncDatabase(db string) error { } // syncFrame synchronizes frame attributes with the rest of the cluster. -func (s *IndexSyncer) syncFrame(db, name string) error { +func (s *HolderSyncer) syncFrame(db, name string) error { // Retrieve database reference. - f := s.Index.Frame(db, name) + f := s.Holder.Frame(db, name) if f == nil { return nil } @@ -513,9 +513,9 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *IndexSyncer) syncFragment(db, frame, view string, slice uint64) error { +func (s *HolderSyncer) syncFragment(db, frame, view string, slice uint64) error { // Retrieve local frame. - f := s.Index.Frame(db, frame) + f := s.Holder.Frame(db, frame) if f == nil { return ErrFrameNotFound } diff --git a/index_test.go b/holder_test.go similarity index 57% rename from index_test.go rename to holder_test.go index 22efbc718..97a8358e9 100644 --- a/index_test.go +++ b/holder_test.go @@ -12,56 +12,56 @@ import ( "github.com/pilosa/pilosa/pql" ) -// Ensure index can delete a database and its underlying files. -func TestIndex_DeleteDB(t *testing.T) { - idx := MustOpenIndex() - defer idx.Close() +// Ensure holder can delete a database and its underlying files. +func TestHolder_DeleteDB(t *testing.T) { + hldr := MustOpenHolder() + defer hldr.Close() // Write bits to separate databases. - f0 := idx.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0) + f0 := hldr.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } - f1 := idx.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0) + f1 := hldr.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0) if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } // Ensure d0 exists. - if _, err := os.Stat(idx.DBPath("d0")); err != nil { + if _, err := os.Stat(hldr.DBPath("d0")); err != nil { t.Fatal(err) } // Delete d0. - if err := idx.DeleteDB("d0"); err != nil { + if err := hldr.DeleteDB("d0"); err != nil { t.Fatal(err) } // Ensure d0 files are removed & d1 still exists. - if _, err := os.Stat(idx.DBPath("d0")); !os.IsNotExist(err) { + if _, err := os.Stat(hldr.DBPath("d0")); !os.IsNotExist(err) { t.Fatal("expected d0 file deletion") - } else if _, err := os.Stat(idx.DBPath("d1")); err != nil { + } else if _, err := os.Stat(hldr.DBPath("d1")); err != nil { t.Fatal("expected d1 files to still exist", err) } } -// Ensure index can sync with a remote index. -func TestIndexSyncer_SyncIndex(t *testing.T) { +// Ensure holder can sync with a remote holder. +func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := NewCluster(2) - // Create a local index. - idx0 := MustOpenIndex() - defer idx0.Close() + // Create a local holder. + hldr0 := MustOpenHolder() + defer hldr0.Close() - // Create a remote index wrapped by an HTTP - idx1 := MustOpenIndex() - defer idx1.Close() + // Create a remote holder wrapped by an HTTP + hldr1 := MustOpenHolder() + defer hldr1.Close() s := NewServer() defer s.Close() - s.Handler.Index = idx1.Index + s.Handler.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() - e.Index = idx1.Index + e.Holder = hldr1.Holder e.Host = cluster.Nodes[1].Host e.Cluster = cluster return e.Execute(ctx, db, query, slices, opt) @@ -73,14 +73,14 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { cluster.Nodes[1].Host = MustParseURLHost(s.URL) // Create frames on nodes. - for _, idx := range []*Index{idx0, idx1} { - idx.MustCreateFrameIfNotExists("d", "f") - idx.MustCreateFrameIfNotExists("d", "f0") - idx.MustCreateFrameIfNotExists("y", "z") + for _, hldr := range []*Holder{hldr0, hldr1} { + hldr.MustCreateFrameIfNotExists("d", "f") + hldr.MustCreateFrameIfNotExists("d", "f0") + hldr.MustCreateFrameIfNotExists("y", "z") } - // Set data on the local index. - f := idx0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + // Set data on the local holder. + f := hldr0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 20); err != nil { @@ -91,15 +91,15 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1) + f = hldr0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1) if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } - idx0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) + hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) - // Set data on the remote index. - f = idx1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + // Set data on the remote holder. + f = hldr1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) } else if _, err := f.SetBit(3, 10); err != nil { @@ -108,7 +108,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatal(err) } - f = idx1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) + f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { @@ -118,23 +118,23 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } // Set highest slice. - idx0.DB("d").SetRemoteMaxSlice(1) - idx0.DB("y").SetRemoteMaxSlice(3) + hldr0.DB("d").SetRemoteMaxSlice(1) + hldr0.DB("y").SetRemoteMaxSlice(3) // Set up syncer. - syncer := pilosa.IndexSyncer{ - Index: idx0.Index, + syncer := pilosa.HolderSyncer{ + Holder: hldr0.Holder, Host: cluster.Nodes[0].Host, Cluster: cluster, } - if err := syncer.SyncIndex(); err != nil { + if err := syncer.SyncHolder(); err != nil { t.Fatal(err) } // Verify data is the same on both nodes. - for i, idx := range []*Index{idx0, idx1} { - f := idx.Fragment("d", "f", pilosa.ViewStandard, 0) + for i, hldr := range []*Holder{hldr0, hldr1} { + f := hldr.Fragment("d", "f", pilosa.ViewStandard, 0) if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { @@ -147,7 +147,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } - f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1) + f = hldr.Fragment("d", "f0", pilosa.ViewStandard, 1) a := f.Row(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) @@ -155,51 +155,51 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - f = idx.Fragment("y", "z", pilosa.ViewStandard, 3) + f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) } } } -// Index is a test wrapper for pilosa.Index. -type Index struct { - *pilosa.Index +// Holder is a test wrapper for pilosa.Holder. +type Holder struct { + *pilosa.Holder LogOutput bytes.Buffer } -// NewIndex returns a new instance of Index with a temporary path. -func NewIndex() *Index { +// NewHolder returns a new instance of Holder with a temporary path. +func NewHolder() *Holder { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) } - i := &Index{Index: pilosa.NewIndex()} - i.Path = path - i.Index.LogOutput = &i.LogOutput + h := &Holder{Holder: pilosa.NewHolder()} + h.Path = path + h.Holder.LogOutput = &h.LogOutput - return i + return h } -// MustOpenIndex creates and opens an index at a temporary path. Panic on error. -func MustOpenIndex() *Index { - i := NewIndex() - if err := i.Open(); err != nil { +// MustOpenHolder creates and opens a holder at a temporary path. Panic on error. +func MustOpenHolder() *Holder { + h := NewHolder() + if err := h.Open(); err != nil { panic(err) } - return i + return h } -// Close closes the index and removes all underlying data. -func (i *Index) Close() error { - defer os.RemoveAll(i.Path) - return i.Index.Close() +// Close closes the holder and removes all underlying data. +func (h *Holder) Close() error { + defer os.RemoveAll(h.Path) + return h.Holder.Close() } // MustCreateDBIfNotExists returns a given db. Panic on error. -func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB { - d, err := i.Index.CreateDBIfNotExists(db, opt) +func (h *Holder) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB { + d, err := h.Holder.CreateDBIfNotExists(db, opt) if err != nil { panic(err) } @@ -207,8 +207,8 @@ func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB { } // MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { - f, err := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) +func (h *Holder) MustCreateFrameIfNotExists(db, frame string) *Frame { + f, err := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } @@ -216,8 +216,8 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { } // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (i *Index) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment { - d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) +func (h *Holder) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment { + d := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) diff --git a/server.go b/server.go index 567461ccc..ba11f1fae 100644 --- a/server.go +++ b/server.go @@ -24,7 +24,7 @@ const ( DefaultPollingInterval = 60 * time.Second ) -// Server represents an index wrapped by a running HTTP server. +// Server represents a holder wrapped by a running HTTP server. type Server struct { ln net.Listener @@ -33,7 +33,7 @@ type Server struct { closing chan struct{} // Data storage and HTTP interface. - Index *Index + Holder *Holder Handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver @@ -55,7 +55,7 @@ func NewServer() *Server { s := &Server{ closing: make(chan struct{}), - Index: NewIndex(), + Holder: NewHolder(), Handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, @@ -66,7 +66,7 @@ func NewServer() *Server { LogOutput: os.Stderr, } - s.Handler.Index = s.Index + s.Handler.Holder = s.Holder return s } @@ -96,8 +96,8 @@ func (s *Server) Open() error { s.Cluster.Nodes = []*Node{{Host: s.Host}} } - // Open index. - if err := s.Index.Open(); err != nil { + // Open holder. + if err := s.Holder.Open(); err != nil { return err } @@ -112,7 +112,7 @@ func (s *Server) Open() error { // Create executor for executing queries. e := NewExecutor() - e.Index = s.Index + e.Holder = s.Holder e.Host = s.Host e.Cluster = s.Cluster @@ -123,9 +123,9 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Handler.LogOutput = s.LogOutput - // Initialize Index. - s.Index.Broadcaster = s.Broadcaster - s.Index.LogOutput = s.LogOutput + // Initialize Holder. + s.Holder.Broadcaster = s.Broadcaster + s.Holder.LogOutput = s.LogOutput // Serve HTTP. go func() { http.Serve(ln, s.Handler) }() @@ -147,8 +147,8 @@ func (s *Server) Close() error { if s.ln != nil { s.ln.Close() } - if s.Index != nil { - s.Index.Close() + if s.Holder != nil { + s.Holder.Close() } return nil @@ -168,7 +168,7 @@ func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) defer ticker.Stop() - s.logger().Printf("index sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) for { // Wait for tick or a close. @@ -178,23 +178,23 @@ func (s *Server) monitorAntiEntropy() { case <-ticker.C: } - s.logger().Printf("index sync beginning") + s.logger().Printf("holder sync beginning") - // Initialize syncer with local index and remote client. - var syncer IndexSyncer - syncer.Index = s.Index + // Initialize syncer with local holder and remote client. + var syncer HolderSyncer + syncer.Holder = s.Holder syncer.Host = s.Host syncer.Cluster = s.Cluster syncer.Closing = s.closing - // Sync indexes. - if err := syncer.SyncIndex(); err != nil { - s.logger().Printf("index sync error: err=%s", err) + // Sync holders. + if err := syncer.SyncHolder(); err != nil { + s.logger().Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.logger().Printf("index sync complete") + s.logger().Printf("holder sync complete") } } @@ -215,14 +215,14 @@ func (s *Server) monitorMaxSlices() { case <-ticker.C: } - oldmaxslices := s.Index.MaxSlices() + oldmaxslices := s.Holder.MaxSlices() for _, node := range s.Cluster.Nodes { if s.Host != node.Host { maxSlices, _ := checkMaxSlices(node.Host) for db, newmax := range maxSlices { // if we don't know about a db locally, log an error because // db's should be created and synced prior to slice creation - if localdb := s.Index.DB(db); localdb != nil { + if localdb := s.Holder.DB(db); localdb != nil { if newmax > oldmaxslices[db] { oldmaxslices[db] = newmax localdb.SetRemoteMaxSlice(newmax) @@ -240,30 +240,30 @@ func (s *Server) monitorMaxSlices() { func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { case *internal.CreateSliceMessage: - d := s.Index.DB(obj.DB) + d := s.Holder.DB(obj.DB) if d == nil { return fmt.Errorf("Local DB not found: %s", obj.DB) } d.SetRemoteMaxSlice(obj.Slice) case *internal.CreateDBMessage: opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := s.Index.CreateDB(obj.DB, opt) + _, err := s.Holder.CreateDB(obj.DB, opt) if err != nil { return err } case *internal.DeleteDBMessage: - if err := s.Index.DeleteDB(obj.DB); err != nil { + if err := s.Holder.DeleteDB(obj.DB); err != nil { return err } case *internal.CreateFrameMessage: - db := s.Index.DB(obj.DB) + db := s.Holder.DB(obj.DB) opt := FrameOptions{RowLabel: obj.Meta.RowLabel} _, err := db.CreateFrame(obj.Frame, opt) if err != nil { return err } case *internal.DeleteFrameMessage: - db := s.Index.DB(obj.DB) + db := s.Holder.DB(obj.DB) if err := db.DeleteFrame(obj.Frame); err != nil { return err } @@ -273,16 +273,16 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { // Server implements gossip.StateHandler. // LocalState returns the state of the local node as well as the -// index (dbs/frames) according to the local node. +// holder (dbs/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalState() (proto.Message, error) { - if s.Index == nil { - return nil, errors.New("Server.Index is nil.") + if s.Holder == nil { + return nil, errors.New("Server.Holder is nil.") } return &internal.NodeState{ Host: s.Host, State: "OK", // TODO: make this work, pull from s.Cluster.Node - DBs: encodeDBs(s.Index.DBs()), + DBs: encodeDBs(s.Holder.DBs()), }, nil } @@ -300,7 +300,7 @@ func (s *Server) mergeRemoteState(ns *internal.NodeState) error { ColumnLabel: db.Meta.ColumnLabel, TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), } - d, err := s.Index.CreateDBIfNotExists(db.Name, opt) + d, err := s.Holder.CreateDBIfNotExists(db.Name, opt) if err != nil { return err } diff --git a/server/server.go b/server/server.go index ec2b55c7c..153f2dc23 100644 --- a/server/server.go +++ b/server/server.go @@ -115,10 +115,10 @@ func (m *Command) SetupServer() error { m.Server.LogOutput = logFile } - // Configure index. + // Configure holder. fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir) - m.Server.Index.Path = m.Config.DataDir - m.Server.Index.Stats = pilosa.NewExpvarStatsClient() + m.Server.Holder.Path = m.Config.DataDir + m.Server.Holder.Stats = pilosa.NewExpvarStatsClient() var err error m.Server.Host, err = normalizeHost(m.Config.Host) From 2ad322c2c73b4a5f5cabce1e47bbdff25e7f76e6 Mon Sep 17 00:00:00 2001 From: Travis Date: Sun, 23 Apr 2017 21:46:49 -0500 Subject: [PATCH 57/63] Rename `DB` to `Index` Rename `db.go` to `index.go` and `db_test.go` to `index_test.go` --- NOTES | 2 +- README.md | 36 +-- broadcast.go | 20 +- broadcast_test.go | 10 +- client.go | 156 +++++------ client_test.go | 102 +++---- cluster.go | 12 +- cluster_test.go | 4 +- cmd/backup.go | 2 +- cmd/backup_test.go | 4 +- cmd/bench.go | 4 +- cmd/bench_test.go | 4 +- cmd/export.go | 2 +- cmd/export_test.go | 4 +- cmd/import.go | 4 +- cmd/import_test.go | 4 +- cmd/restore.go | 4 +- cmd/restore_test.go | 4 +- ctl/backup.go | 10 +- ctl/bench.go | 14 +- ctl/export.go | 16 +- ctl/import.go | 14 +- ctl/restore.go | 10 +- db.go | 565 --------------------------------------- db_test.go | 179 ------------- executor.go | 188 ++++++------- executor_test.go | 274 ++++++++++--------- fragment.go | 26 +- fragment_test.go | 68 ++--- frame.go | 18 +- frame_test.go | 10 +- handler.go | 310 ++++++++++----------- handler_internal_test.go | 12 +- handler_test.go | 190 ++++++------- holder.go | 222 +++++++-------- holder_test.go | 66 ++--- index.go | 565 +++++++++++++++++++++++++++++++++++++++ index_test.go | 179 +++++++++++++ internal/private.pb.go | 310 ++++++++++----------- internal/private.proto | 26 +- internal/public.pb.go | 89 +++--- internal/public.proto | 2 +- pilosa.go | 12 +- server.go | 60 ++--- server/server_test.go | 91 +++---- view.go | 12 +- view_test.go | 10 +- 47 files changed, 1962 insertions(+), 1964 deletions(-) delete mode 100644 db.go delete mode 100644 db_test.go create mode 100644 index.go create mode 100644 index_test.go diff --git a/NOTES b/NOTES index 6ba447da0..6b8e088ea 100644 --- a/NOTES +++ b/NOTES @@ -1,5 +1,5 @@ - DB Column + Index Column ┌───────────▼────────────────────────────┐ │0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ diff --git a/README.md b/README.md index d8318820b..acab512eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # pilosa -Pilosa is a bitmap index database. +Pilosa is a bitmap index. [![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa) @@ -123,60 +123,60 @@ Return the version of Pilosa: $ curl "http://127.0.0.1:10101/version" ``` -Return a list of all databases and frames in the index: +Return a list of all indexes and frames in the index: ```sh $ curl "http://127.0.0.1:10101/schema" ``` -### Database and Frame Schema +### Index and Frame Schema -Before running a query, the corresponding database and frame must be created. Note that database and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`). +Before running a query, the corresponding index and frame must be created. Note that index and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`). -You can create the database `sample-db` using: +You can create the index `sample-idx` using: ```sh -$ curl -XPOST "http://127.0.0.1:10101/db" \ - -d '{"db": "sample-db"}' +$ curl -XPOST "http://127.0.0.1:10101/index" \ + -d '{"index": "sample-idx"}' ``` -Optionally, you can specify the column label on database creation: +Optionally, you can specify the column label on index creation: ```sh -$ curl -XPOST "http://127.0.0.1:10101/db" \ - -d '{"db": "sample-db", "options": {"columnLabel": "user"}}' +$ curl -XPOST "http://127.0.0.1:10101/index" \ + -d '{"index": "sample-idx", "options": {"columnLabel": "user"}}' ``` The frame `collaboration` may be created using the following call: ```sh $ curl -XPOST "http://127.0.0.1:10101/frame" \ - -d '{"db": "sample-db", "frame": "collaboration"}' + -d '{"index": "sample-idx", "frame": "collaboration"}' ``` It is possible to specify the frame row label on frame creation: ```sh $ curl -XPOST "http://127.0.0.1:10101/frame" \ - -d '{"db": "sample-db", "frame": "collaboration", "options": {"rowLabel": "project"}}' + -d '{"index": "sample-idx", "frame": "collaboration", "options": {"rowLabel": "project"}}' ``` ### Queries Queries to Pilosa require sending a POST request where the query itself is sent as POST data. -You specify the database on which to perform the query with a URL argument `db=database-name`. +You specify the index on which to perform the query with a URL argument `index=index-name`. -In this section, we assume both the database `sample-db` with column label `user` and the frame `collaboration` with row label `project` was created. +In this section, we assume both the index `sample-idx` with column label `user` and the frame `collaboration` with row label `project` was created. -A query sent to database `sample-db` will have the following format: +A query sent to index `sample-idx` will have the following format: ```sh -$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query()' +$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query()' ``` The `Query()` object referenced above should be made up of one or more of the query types listed below. So for example, a SetBit() query would look like this: ```sh -$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'SetBit(project=10, frame="collaboration", user=1)' +$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'SetBit(project=10, frame="collaboration", user=1)' ``` Query results have the format `{"results":[]}`, where `results` is a list of results for each `Query()`. This @@ -184,7 +184,7 @@ means that you can provide multiple `Query()` objects with each HTTP request and the results of all of the queries. ```sh -$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query() Query() Query()' +$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query() Query() Query()' ``` --- diff --git a/broadcast.go b/broadcast.go index a3f3cb6c9..5a1600fbb 100644 --- a/broadcast.go +++ b/broadcast.go @@ -84,8 +84,8 @@ var NopBroadcastReceiver = &nopBroadcastReceiver{} const ( MessageTypeCreateSlice = 1 - MessageTypeCreateDB = 2 - MessageTypeDeleteDB = 3 + MessageTypeCreateIndex = 2 + MessageTypeDeleteIndex = 3 MessageTypeCreateFrame = 4 MessageTypeDeleteFrame = 5 ) @@ -95,10 +95,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) { switch obj := m.(type) { case *internal.CreateSliceMessage: typ = MessageTypeCreateSlice - case *internal.CreateDBMessage: - typ = MessageTypeCreateDB - case *internal.DeleteDBMessage: - typ = MessageTypeDeleteDB + case *internal.CreateIndexMessage: + typ = MessageTypeCreateIndex + case *internal.DeleteIndexMessage: + typ = MessageTypeDeleteIndex case *internal.CreateFrameMessage: typ = MessageTypeCreateFrame case *internal.DeleteFrameMessage: @@ -120,10 +120,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { switch typ { case MessageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeCreateDB: - m = &internal.CreateDBMessage{} - case MessageTypeDeleteDB: - m = &internal.DeleteDBMessage{} + case MessageTypeCreateIndex: + m = &internal.CreateIndexMessage{} + case MessageTypeDeleteIndex: + m = &internal.DeleteIndexMessage{} case MessageTypeCreateFrame: m = &internal.CreateFrameMessage{} case MessageTypeDeleteFrame: diff --git a/broadcast_test.go b/broadcast_test.go index a2842a919..de38239d4 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -13,12 +13,12 @@ import ( func TestMessage_Marshal(t *testing.T) { testMessageMarshal(t, &internal.CreateSliceMessage{ - DB: "d", + Index: "i", Slice: 8, }) - testMessageMarshal(t, &internal.DeleteDBMessage{ - DB: "d", + testMessageMarshal(t, &internal.DeleteIndexMessage{ + Index: "i", }) } @@ -47,8 +47,8 @@ func TestBroadcast_BroadcastReceiver(t *testing.T) { s.BroadcastReceiver = sbr s.BroadcastReceiver.Start(sbh) - msg := &internal.DeleteDBMessage{ - DB: "d", + msg := &internal.DeleteIndexMessage{ + Index: "i", } s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg) diff --git a/client.go b/client.go index 46a27c02f..7606369ae 100644 --- a/client.go +++ b/client.go @@ -45,18 +45,18 @@ func NewClient(host string) (*Client, error) { // Host returns the host the client was initialized with. func (c *Client) Host() string { return c.host } -// MaxSliceByDatabase returns the number of slices on a server by database. -func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByDatabase(ctx, false) +// MaxSliceByIndex returns the number of slices on a server by index. +func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByIndex(ctx, false) } -// MaxInverseSliceByDatabase returns the number of inverse slices on a server by database. -func (c *Client) MaxInverseSliceByDatabase(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByDatabase(ctx, true) +// MaxInverseSliceByIndex returns the number of inverse slices on a server by index. +func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByIndex(ctx, true) } -// maxSliceByDatabase returns the number of slices on a server by database. -func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[string]uint64, error) { +// maxSliceByIndex returns the number of slices on a server by index. +func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { // Execute request against the host. u := url.URL{ Scheme: "http", @@ -90,8 +90,8 @@ func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[stri return rsp.MaxSlices, nil } -// Schema returns all database and frame schema information. -func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) { +// Schema returns all index and frame schema information. +func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) { // Execute request against the host. u := url.URL{ Scheme: "http", @@ -118,13 +118,13 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) { } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, fmt.Errorf("json decode: %s", err) } - return rsp.DBs, nil + return rsp.Indexes, nil } -// CreateDB creates a new database on the server. -func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { +// CreateIndex creates a new index on the server. +func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { // Encode query request. - buf, err := json.Marshal(&postDBRequest{ + buf, err := json.Marshal(&postIndexRequest{ Options: opt, }) if err != nil { @@ -132,7 +132,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { } // Create URL & HTTP request. - u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s", db)} + u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s", index)} req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return err @@ -159,20 +159,20 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { case http.StatusOK: return nil // ok case http.StatusConflict: - return ErrDatabaseExists + return ErrIndexExists default: return errors.New(string(body)) } } // FragmentNodes returns a list of nodes that own a slice. -func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) { +func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { // Execute request against the host. u := url.URL{ Scheme: "http", Host: c.host, Path: "/fragment/nodes", - RawQuery: (url.Values{"db": {db}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(), + RawQuery: (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(), } // Build request. @@ -198,10 +198,10 @@ func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([] return a, nil } -// ExecuteQuery executes query against db on the server. -func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedirect bool) (result interface{}, err error) { - if db == "" { - return nil, ErrDatabaseRequired +// ExecuteQuery executes query against index on the server. +func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRedirect bool) (result interface{}, err error) { + if index == "" { + return nil, ErrIndexRequired } else if query == "" { return nil, ErrQueryRequired } @@ -219,7 +219,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire u := url.URL{ Scheme: "http", Host: c.host, - Path: fmt.Sprintf("/db/%s/query", db), + Path: fmt.Sprintf("/index/%s/query", index), } req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { @@ -254,14 +254,14 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire return qresp, nil } -// ExecutePQL executes query string against db on the server. -func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, error) { +// ExecutePQL executes query string against index on the server. +func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, Path: "/query", RawQuery: url.Values{ - "db": {db}, + "index": {index}, }.Encode(), } @@ -287,20 +287,20 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, } // Import bulk imports bits for a single slice to a host. -func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bits []Bit) error { - if db == "" { - return ErrDatabaseRequired +func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { + if index == "" { + return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := MarshalImportPayload(db, frame, slice, bits) + buf, err := MarshalImportPayload(index, frame, slice, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, db, slice) + nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { return fmt.Errorf("slice nodes: %s", err) } @@ -315,7 +315,7 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit return nil } -func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { +func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. rowIDs := Bits(bits).RowIDs() columnIDs := Bits(bits).ColumnIDs() @@ -323,7 +323,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ - DB: db, + Index: index, Frame: frame, Slice: slice, RowIDs: rowIDs, @@ -374,15 +374,15 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64, w io.Writer) error { - if db == "" { - return ErrDatabaseRequired +func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { + if index == "" { + return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, db, slice) + nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { return fmt.Errorf("slice nodes: %s", err) } @@ -392,7 +392,7 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64, for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, db, frame, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) continue } else { @@ -404,14 +404,14 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64, } // exportNode copies a CSV export from a node to w. -func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string, slice uint64, w io.Writer) error { +func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { // Create URL. u := url.URL{ Scheme: "http", Host: node.Host, Path: "/export", RawQuery: url.Values{ - "db": {db}, + "index": {index}, "frame": {frame}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), @@ -445,9 +445,9 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string } // BackupTo backs up an entire frame from a cluster to w. -func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error { - if db == "" { - return ErrDatabaseRequired +func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error { + if index == "" { + return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } @@ -456,14 +456,14 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri tw := tar.NewWriter(w) // Find the maximum number of slices. - maxSlices, err := c.MaxSliceByDatabase(ctx) + maxSlices, err := c.MaxSliceByIndex(ctx) if err != nil { return fmt.Errorf("slice n: %s", err) } // Backup every slice to the tar file. - for i := uint64(0); i <= maxSlices[db]; i++ { - if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil { + for i := uint64(0); i <= maxSlices[index]; i++ { + if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil { return err } } @@ -477,9 +477,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri } // backupSliceTo backs up a single slice to tw. -func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error { +func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error { // Return error if unable to backup from any slice. - r, err := c.BackupSlice(ctx, db, frame, view, slice) + r, err := c.BackupSlice(ctx, index, frame, view, slice) if err != nil { return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err) } else if r == nil { @@ -515,16 +515,16 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, v // BackupSlice retrieves a streaming backup from a single slice. // This function tries slice owners until one succeeds. -func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) { +func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) { // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, db, slice) + nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { return nil, fmt.Errorf("slice nodes: %s", err) } // Try to backup slice from each one until successful. for _, i := range rand.Perm(len(nodes)) { - r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i]) + r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i]) if err == nil { return r, nil // successfully attached } else if err == ErrFragmentNotFound { @@ -538,13 +538,13 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice return nil, fmt.Errorf("unable to connect to any owner") } -func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { +func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) { u := url.URL{ Scheme: "http", Host: node.Host, Path: "/fragment/data", RawQuery: url.Values{ - "db": {db}, + "index": {index}, "frame": {frame}, "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, @@ -576,9 +576,9 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, sl } // RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error { - if db == "" { - return ErrDatabaseRequired +func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error { + if index == "" { + return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } @@ -608,16 +608,16 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view s } // Restore file to all nodes that own it. - if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil { + if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil { return err } } } // restoreSliceFrom restores a single slice to all owning nodes. -func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error { +func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error { // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, db, slice) + nodes, err := c.FragmentNodes(ctx, index, slice) if err != nil { return fmt.Errorf("slice nodes: %s", err) } @@ -629,7 +629,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi Host: node.Host, Path: "/fragment/data", RawQuery: url.Values{ - "db": {db}, + "index": {index}, "frame": {frame}, "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, @@ -659,9 +659,9 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi } // CreateFrame creates a new frame on the server. -func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error { - if db == "" { - return ErrDatabaseRequired +func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { + if index == "" { + return ErrIndexRequired } // Encode query request. @@ -673,7 +673,7 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt } // Create URL & HTTP request. - u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s/frame/%s", db, frame)} + u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s/frame/%s", index, frame)} req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) if err != nil { return err @@ -707,11 +707,11 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt } // RestoreFrame restores an entire frame from a host in another cluster. -func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error { +func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error { u := url.URL{ Scheme: "http", Host: c.Host(), - Path: fmt.Sprintf("/db/%s/frame/%s/restore", db, frame), + Path: fmt.Sprintf("/index/%s/frame/%s/restore", index, frame), RawQuery: url.Values{ "host": {host}, }.Encode(), @@ -740,12 +740,12 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error } // FrameViews returns a list of view names for a frame. -func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) { +func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) { // Create URL & HTTP request. u := url.URL{ Scheme: "http", Host: c.host, - Path: fmt.Sprintf("/db/%s/frame/%s/views", db, frame), + Path: fmt.Sprintf("/index/%s/frame/%s/views", index, frame), } req, err := http.NewRequest("GET", u.String(), nil) if err != nil { @@ -780,13 +780,13 @@ func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, er // FragmentBlocks returns a list of block checksums for a fragment on a host. // Only returns blocks which contain data. -func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) { +func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) { u := url.URL{ Scheme: "http", Host: c.host, Path: "/fragment/blocks", RawQuery: url.Values{ - "db": {db}, + "index": {index}, "frame": {frame}, "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, @@ -824,9 +824,9 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli } // BlockData returns row/column id pairs for a block. -func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { +func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ - DB: db, + Index: index, Frame: frame, View: view, Slice: slice, @@ -871,15 +871,15 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui } // ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, - Path: fmt.Sprintf("/db/%s/attr/diff", db), + Path: fmt.Sprintf("/index/%s/attr/diff", index), } // Encode request. - buf, err := json.Marshal(postDBAttrDiffRequest{Blocks: blks}) + buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) if err != nil { return nil, err } @@ -906,7 +906,7 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock } // Decode response object. - var rsp postDBAttrDiffResponse + var rsp postIndexAttrDiffResponse if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { return nil, err } @@ -914,11 +914,11 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock } // RowAttrDiff returns data from differing blocks on a remote host. -func (c *Client) RowAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, - Path: fmt.Sprintf("/db/%s/frame/%s/attr/diff", db, frame), + Path: fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame), } // Encode request. diff --git a/client_test.go b/client_test.go index 858658928..d2111144d 100644 --- a/client_test.go +++ b/client_test.go @@ -38,56 +38,56 @@ func TestClient_MultiNode(t *testing.T) { defer s[i].Close() } - s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() e.Holder = hldr[0].Holder e.Host = cluster.Nodes[0].Host e.Cluster = cluster - return e.Execute(ctx, db, query, slices, opt) + return e.Execute(ctx, index, query, slices, opt) } - s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() e.Holder = hldr[1].Holder e.Host = cluster.Nodes[1].Host e.Cluster = cluster - return e.Execute(ctx, db, query, slices, opt) + return e.Execute(ctx, index, query, slices, opt) } - s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() e.Holder = hldr[2].Holder e.Host = cluster.Nodes[2].Host e.Cluster = cluster - return e.Execute(ctx, db, query, slices, opt) + return e.Execute(ctx, index, query, slices, opt) } // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4) - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(100, (SliceWidth*9)+10) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12, (SliceWidth*9)+13, (SliceWidth*9)+14, (SliceWidth*9)+15) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(2, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(3, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4, (SliceWidth*9)+5) + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(22, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+10) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5) - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4) + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6) + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(1, 4) + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11) + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache() - hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache() - hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache() + hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 10).RecalculateCache() + hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).RecalculateCache() // Connect to each node to compare results. client := make([]*Client, 3) @@ -96,9 +96,9 @@ func TestClient_MultiNode(t *testing.T) { client[2] = MustNewClient(s[0].Host()) topN := 4 - q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f.n", topN) + q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN) - result, err := client[0].ExecuteQuery(context.Background(), "d", q, true) + result, err := client[0].ExecuteQuery(context.Background(), "i", q, true) if err != nil { t.Fatal(err) } @@ -106,17 +106,17 @@ func TestClient_MultiNode(t *testing.T) { // Check the results before every node has the correct max slice value. pairs := result.(internal.QueryResponse).Results[0].Pairs for _, pair := range pairs { - if pair.Key == 22 && pair.Count != 5 { + if pair.Key == 22 && pair.Count != 11 { t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair) } } // Set max slice to correct value. - hldr[0].DB("d").SetRemoteMaxSlice(10) - hldr[1].DB("d").SetRemoteMaxSlice(10) - hldr[2].DB("d").SetRemoteMaxSlice(10) + hldr[0].Index("i").SetRemoteMaxSlice(10) + hldr[1].Index("i").SetRemoteMaxSlice(10) + hldr[2].Index("i").SetRemoteMaxSlice(10) - result, err = client[0].ExecuteQuery(context.Background(), "d", q, true) + result, err = client[0].ExecuteQuery(context.Background(), "i", q, true) if err != nil { t.Fatal(err) } @@ -136,11 +136,11 @@ func TestClient_MultiNode(t *testing.T) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } - result1, err := client[1].ExecuteQuery(context.Background(), "d", q, true) + result1, err := client[1].ExecuteQuery(context.Background(), "i", q, true) if err != nil { t.Fatal(err) } - result2, err := client[2].ExecuteQuery(context.Background(), "d", q, true) + result2, err := client[2].ExecuteQuery(context.Background(), "i", q, true) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestClient_Import(t *testing.T) { defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. - f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) f.Row(0) s := NewServer() @@ -173,7 +173,7 @@ func TestClient_Import(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) - if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ + if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, {RowID: 200, ColumnID: 6}, @@ -195,7 +195,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) frameOpts := pilosa.FrameOptions{ InverseEnabled: true, } @@ -224,7 +224,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) - if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ + if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, {RowID: 200, ColumnID: 5}, @@ -250,10 +250,10 @@ func TestClient_BackupRestore(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) s := NewServer() defer s.Close() @@ -266,12 +266,12 @@ func TestClient_BackupRestore(t *testing.T) { // Backup from frame. var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil { + if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil { t.Fatal(err) } // Restore to a different frame. - if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { @@ -299,11 +299,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer hldr.Close() // Set two bits on blocks 0 & 3. - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := NewServer() defer s.Close() @@ -314,7 +314,7 @@ func TestClient_FragmentBlocks(t *testing.T) { // Retrieve blocks. c := MustNewClient(s.Host()) - blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0) + blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { @@ -326,7 +326,7 @@ func TestClient_FragmentBlocks(t *testing.T) { } // Verify data matches local blocks. - if a := hldr.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { + if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } diff --git a/cluster.go b/cluster.go index f92b8c6e9..b9d799aec 100644 --- a/cluster.go +++ b/cluster.go @@ -146,25 +146,25 @@ func (c *Cluster) NodeByHost(host string) *Node { } // Partition returns the partition that a slice belongs to. -func (c *Cluster) Partition(db string, slice uint64) int { +func (c *Cluster) Partition(index string, slice uint64) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], slice) // Hash the bytes and mod by partition count. h := fnv.New64a() - h.Write([]byte(db)) + h.Write([]byte(index)) h.Write(buf[:]) return int(h.Sum64() % uint64(c.PartitionN)) } // FragmentNodes returns a list of nodes that own a fragment. -func (c *Cluster) FragmentNodes(db string, slice uint64) []*Node { - return c.PartitionNodes(c.Partition(db, slice)) +func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { + return c.PartitionNodes(c.Partition(index, slice)) } // OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(host string, db string, slice uint64) bool { - return Nodes(c.FragmentNodes(db, slice)).ContainsHost(host) +func (c *Cluster) OwnsFragment(host string, index string, slice uint64) bool { + return Nodes(c.FragmentNodes(index, slice)).ContainsHost(host) } // PartitionNodes returns a list of nodes that own a partition. diff --git a/cluster_test.go b/cluster_test.go index 9dbaa420d..e561223fb 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -37,11 +37,11 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(db string, slice uint64, partitionN int) bool { + if err := quick.Check(func(index string, slice uint64, partitionN int) bool { c := pilosa.NewCluster() c.PartitionN = partitionN - partitionID := c.Partition(db, slice) + partitionID := c.Partition(index, slice) if partitionID < 0 || partitionID >= partitionN { t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) } diff --git a/cmd/backup.go b/cmd/backup.go index 24904cbb5..d25b968f4 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -29,7 +29,7 @@ Backs up the view from across the cluster into a single file. } flags := backupCmd.Flags() flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.") + flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup into.") flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.") flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.") flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") diff --git a/cmd/backup_test.go b/cmd/backup_test.go index cacd6510b..4e8f8fe43 100644 --- a/cmd/backup_test.go +++ b/cmd/backup_test.go @@ -22,13 +22,13 @@ func TestBackupConfig(t *testing.T) { args: []string{"backup", "--output-file", "/somefile"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` -database = "mydb" +index = "myindex" frame = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Backuper.Host, "localhost:12345") - v.Check(cmd.Backuper.Database, "mydb") + v.Check(cmd.Backuper.Index, "myindex") v.Check(cmd.Backuper.Frame, "f1") v.Check(cmd.Backuper.Path, "/somefile") return v.Error() diff --git a/cmd/bench.go b/cmd/bench.go index ec7be3760..2dd0d86fc 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -18,7 +18,7 @@ func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { Use: "bench", Short: "Benchmark operations.", Long: ` -Executes a benchmark for a given operation against the database. +Executes a benchmark for a given operation against the index. `, RunE: func(cmd *cobra.Command, args []string) error { if err := Bencher.Run(context.Background()); err != nil { @@ -29,7 +29,7 @@ Executes a benchmark for a given operation against the database. } flags := benchCmd.Flags() flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Bencher.Database, "database", "d", "", "Pilosa database to benchmark.") + flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") diff --git a/cmd/bench_test.go b/cmd/bench_test.go index 2b5e2c872..db47829ea 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -22,13 +22,13 @@ func TestBenchConfig(t *testing.T) { args: []string{"bench", "--operation", "set-bit"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` -database = "mydb" +index = "myindex" frame = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Bencher.Host, "localhost:12345") - v.Check(cmd.Bencher.Database, "mydb") + v.Check(cmd.Bencher.Index, "myindex") v.Check(cmd.Bencher.Frame, "f1") v.Check(cmd.Bencher.Op, "set-bit") v.Check(cmd.Bencher.N, 0) diff --git a/cmd/export.go b/cmd/export.go index e4511d40f..186759909 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -37,7 +37,7 @@ The file does not contain any headers. flags := exportCmd.Flags() flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Exporter.Database, "database", "d", "", "Pilosa database to export into.") + flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") diff --git a/cmd/export_test.go b/cmd/export_test.go index d546a839a..45ddc58c6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -22,13 +22,13 @@ func TestExportConfig(t *testing.T) { args: []string{"export", "--output-file", "/somefile"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` -database = "mydb" +index = "myindex" frame = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Exporter.Host, "localhost:12345") - v.Check(cmd.Exporter.Database, "mydb") + v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() diff --git a/cmd/import.go b/cmd/import.go index d8bc932e8..43b26e7f8 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -16,7 +16,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's database and frame. The bits + Long: `Bulk imports one or more CSV files to a host's index and frame. The bits of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -36,7 +36,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. } flags := importCmd.Flags() flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Importer.Database, "database", "d", "", "Pilosa database to import into.") + flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") diff --git a/cmd/import_test.go b/cmd/import_test.go index ac98539a4..0251d1894 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -22,13 +22,13 @@ func TestImportConfig(t *testing.T) { args: []string{"import"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` -database = "mydb" +index = "myindex" frame = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Importer.Host, "localhost:12345") - v.Check(cmd.Importer.Database, "mydb") + v.Check(cmd.Importer.Index, "myindex") v.Check(cmd.Importer.Frame, "f1") return v.Error() }, diff --git a/cmd/restore.go b/cmd/restore.go index 5179cbc0d..38a1a3145 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -30,10 +30,10 @@ Restores a view to the cluster from a backup file. } flags := restoreCmd.Flags() flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.") + flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.") flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.") flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.") - flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.") + flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.") return restoreCmd } diff --git a/cmd/restore_test.go b/cmd/restore_test.go index ef8a90d32..6f8b5c15b 100644 --- a/cmd/restore_test.go +++ b/cmd/restore_test.go @@ -22,13 +22,13 @@ func TestRestoreConfig(t *testing.T) { args: []string{"restore", "--input-file", "/somefile"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` -database = "mydb" +index = "myindex" frame = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Restorer.Host, "localhost:12345") - v.Check(cmd.Restorer.Database, "mydb") + v.Check(cmd.Restorer.Index, "myindex") v.Check(cmd.Restorer.Frame, "f1") v.Check(cmd.Restorer.Path, "/somefile") return v.Error() diff --git a/ctl/backup.go b/ctl/backup.go index 6eeed5d4c..369455e7c 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -14,10 +14,10 @@ type BackupCommand struct { // Destination host and port. Host string - // Name of the database, frame, view to backup. - Database string - Frame string - View string + // Name of the index, frame, view to backup. + Index string + Frame string + View string // Output file to write to. Path string @@ -54,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error { defer f.Close() // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { + if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { return err } diff --git a/ctl/bench.go b/ctl/bench.go index 167017f2f..b63d9165f 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -11,14 +11,14 @@ import ( "github.com/pilosa/pilosa" ) -// BenchCommand represents a command for benchmarking database operations. +// BenchCommand represents a command for benchmarking index operations. type BenchCommand struct { // Destination host and port. Host string - // Name of the database & frame to execute against. - Database string - Frame string + // Name of the index & frame to execute against. + Index string + Frame string // Type of operation and number to execute. Op string @@ -57,8 +57,8 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error { if cmd.N == 0 { return errors.New("operation count required") - } else if cmd.Database == "" { - return pilosa.ErrDatabaseRequired + } else if cmd.Index == "" { + return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired } @@ -75,7 +75,7 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID) - if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil { + if _, err := client.ExecuteQuery(ctx, cmd.Index, q, true); err != nil { return err } } diff --git a/ctl/export.go b/ctl/export.go index f4b0efd0a..6bbf1ac0e 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -14,9 +14,9 @@ type ExportCommand struct { // Remote host and port. Host string - // Name of the database & frame to export from. - Database string - Frame string + // Name of the index & frame to export from. + Index string + Frame string // Filename to export to. Path string @@ -37,8 +37,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - if cmd.Database == "" { - return pilosa.ErrDatabaseRequired + if cmd.Index == "" { + return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired } @@ -63,15 +63,15 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - maxSlices, err := client.MaxSliceByDatabase(ctx) + maxSlices, err := client.MaxSliceByIndex(ctx) if err != nil { return err } // Export each slice. - for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ { + for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { return err } } diff --git a/ctl/import.go b/ctl/import.go index 1ba8677cb..cefe2e45b 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -19,9 +19,9 @@ type ImportCommand struct { // Destination host and port. Host string `json:"host"` - // Name of the database & frame to import into. - Database string `json:"db"` - Frame string `json:"frame"` + // Name of the index & frame to import into. + Index string `json:"index"` + Frame string `json:"frame"` // Filenames to import from. Paths []string `json:"paths"` @@ -54,9 +54,9 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - // Database and frame are validated early before the files are parsed. - if cmd.Database == "" { - return pilosa.ErrDatabaseRequired + // Index and frame are validated early before the files are parsed. + if cmd.Index == "" { + return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired } else if len(cmd.Paths) == 0 { @@ -176,7 +176,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err // Parse path into bits. for slice, bits := range bitsBySlice { logger.Printf("importing slice: %d, n=%d", slice, len(bits)) - if err := cmd.Client.Import(ctx, cmd.Database, cmd.Frame, slice, bits); err != nil { + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil { return err } } diff --git a/ctl/restore.go b/ctl/restore.go index 56dfb3080..ecfcf5f5a 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -14,10 +14,10 @@ type RestoreCommand struct { // Destination host and port. Host string - // Name of the database & frame to backup. - Database string - Frame string - View string + // Name of the index & frame to backup. + Index string + Frame string + View string // Import file to read from. Path string @@ -54,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error { defer f.Close() // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil { + if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { return err } diff --git a/db.go b/db.go deleted file mode 100644 index 049e81261..000000000 --- a/db.go +++ /dev/null @@ -1,565 +0,0 @@ -package pilosa - -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "sync" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" -) - -// Default database settings. -const ( - DefaultColumnLabel = "columnID" -) - -// DB represents a container for frames. -type DB struct { - mu sync.Mutex - path string - name string - - // Default time quantum for all frames in database. - // This can be overridden by individual frames. - timeQuantum TimeQuantum - - // Label used for referring to columns in database. - columnLabel string - - // Frames by name. - frames map[string]*Frame - - // Max Slice on any node in the cluster, according to this node - remoteMaxSlice uint64 - remoteMaxInverseSlice uint64 - - // Column attribute storage and cache - columnAttrStore *AttrStore - - broadcaster Broadcaster - stats StatsClient - - LogOutput io.Writer -} - -// NewDB returns a new instance of DB. -func NewDB(path, name string) (*DB, error) { - err := ValidateName(name) - if err != nil { - return nil, err - } - - return &DB{ - path: path, - name: name, - frames: make(map[string]*Frame), - - remoteMaxSlice: 0, - remoteMaxInverseSlice: 0, - - columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), - - columnLabel: DefaultColumnLabel, - - stats: NopStatsClient, - LogOutput: ioutil.Discard, - }, nil -} - -// Name returns name of the database. -func (db *DB) Name() string { return db.name } - -// Path returns the path the database was initialized with. -func (db *DB) Path() string { return db.path } - -// ColumnAttrStore returns the storage for column attributes. -func (db *DB) ColumnAttrStore() *AttrStore { return db.columnAttrStore } - -// SetColumnLabel sets the column label. Persists to meta file on update. -func (db *DB) SetColumnLabel(v string) error { - db.mu.Lock() - defer db.mu.Unlock() - - // Ignore if no change occurred. - if v == "" || db.columnLabel == v { - return nil - } - - // Make sure columnLabel is valid name - err := ValidateName(v) - if err != nil { - return err - } - - // Persist meta data to disk on change. - db.columnLabel = v - if err := db.saveMeta(); err != nil { - return err - } - - return nil -} - -// ColumnLabel returns the column label. -func (db *DB) ColumnLabel() string { - db.mu.Lock() - v := db.columnLabel - db.mu.Unlock() - return v -} - -// Open opens and initializes the database. -func (db *DB) Open() error { - // Ensure the path exists. - if err := os.MkdirAll(db.path, 0777); err != nil { - return err - } - - // Read meta file. - if err := db.loadMeta(); err != nil { - return err - } - - if err := db.openFrames(); err != nil { - return err - } - - if err := db.columnAttrStore.Open(); err != nil { - return err - } - - return nil -} - -// openFrames opens and initializes the frames inside the database. -func (db *DB) openFrames() error { - f, err := os.Open(db.path) - if err != nil { - return err - } - defer f.Close() - - fis, err := f.Readdir(0) - if err != nil { - return err - } - - for _, fi := range fis { - if !fi.IsDir() { - continue - } - - fr, err := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) - if err != nil { - return ErrName - } - if err := fr.Open(); err != nil { - return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) - } - db.frames[fr.Name()] = fr - - db.stats.Count("frameN", 1) - } - return nil -} - -// loadMeta reads meta data for the database, if any. -func (db *DB) loadMeta() error { - var pb internal.DBMeta - - // Read data from meta file. - buf, err := ioutil.ReadFile(filepath.Join(db.path, ".meta")) - if os.IsNotExist(err) { - db.timeQuantum = "" - db.columnLabel = DefaultColumnLabel - return nil - } else if err != nil { - return err - } else { - if err := proto.Unmarshal(buf, &pb); err != nil { - return err - } - } - - // Copy metadata fields. - db.timeQuantum = TimeQuantum(pb.TimeQuantum) - db.columnLabel = pb.ColumnLabel - - return nil -} - -// saveMeta writes meta data for the database. -func (db *DB) saveMeta() error { - // Marshal metadata. - buf, err := proto.Marshal(&internal.DBMeta{ - TimeQuantum: string(db.timeQuantum), - ColumnLabel: db.columnLabel, - }) - if err != nil { - return err - } - - // Write to meta file. - if err := ioutil.WriteFile(filepath.Join(db.path, ".meta"), buf, 0666); err != nil { - return err - } - - return nil -} - -// Close closes the database and its frames. -func (db *DB) Close() error { - db.mu.Lock() - defer db.mu.Unlock() - - // Close the attribute store. - if db.columnAttrStore != nil { - db.columnAttrStore.Close() - } - - // Close all frames. - for _, f := range db.frames { - f.Close() - } - db.frames = make(map[string]*Frame) - - return nil -} - -// MaxSlice returns the max slice in the database according to this node. -func (db *DB) MaxSlice() uint64 { - if db == nil { - return 0 - } - db.mu.Lock() - defer db.mu.Unlock() - - max := db.remoteMaxSlice - for _, f := range db.frames { - if slice := f.MaxSlice(); slice > max { - max = slice - } - } - return max -} - -func (db *DB) SetRemoteMaxSlice(newmax uint64) { - db.mu.Lock() - defer db.mu.Unlock() - db.remoteMaxSlice = newmax -} - -// MaxInverseSlice returns the max inverse slice in the database according to this node. -func (db *DB) MaxInverseSlice() uint64 { - if db == nil { - return 0 - } - db.mu.Lock() - defer db.mu.Unlock() - - max := db.remoteMaxInverseSlice - for _, f := range db.frames { - if slice := f.MaxInverseSlice(); slice > max { - max = slice - } - } - return max -} - -func (db *DB) SetRemoteMaxInverseSlice(v uint64) { - db.mu.Lock() - defer db.mu.Unlock() - db.remoteMaxInverseSlice = v -} - -// TimeQuantum returns the default time quantum for the database. -func (db *DB) TimeQuantum() TimeQuantum { - db.mu.Lock() - defer db.mu.Unlock() - return db.timeQuantum -} - -// SetTimeQuantum sets the default time quantum for the database. -func (db *DB) SetTimeQuantum(q TimeQuantum) error { - db.mu.Lock() - defer db.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on database. - db.timeQuantum = q - - // Perist meta data to disk. - if err := db.saveMeta(); err != nil { - return err - } - - return nil -} - -// FramePath returns the path to a frame in the database. -func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) } - -// Frame returns a frame in the database by name. -func (db *DB) Frame(name string) *Frame { - db.mu.Lock() - defer db.mu.Unlock() - return db.frame(name) -} - -func (db *DB) frame(name string) *Frame { return db.frames[name] } - -// Frames returns a list of all frames in the database. -func (db *DB) Frames() []*Frame { - db.mu.Lock() - defer db.mu.Unlock() - - a := make([]*Frame, 0, len(db.frames)) - for _, f := range db.frames { - a = append(a, f) - } - sort.Sort(frameSlice(a)) - - return a -} - -// CreateFrame creates a frame. -func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) { - db.mu.Lock() - defer db.mu.Unlock() - - // Ensure frame doesn't already exist. - if db.frames[name] != nil { - return nil, ErrFrameExists - } - return db.createFrame(name, opt) -} - -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { - db.mu.Lock() - defer db.mu.Unlock() - - // Find frame in cache first. - if f := db.frames[name]; f != nil { - return f, nil - } - - return db.createFrame(name, opt) -} - -func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { - if name == "" { - return nil, errors.New("frame name required") - } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { - return nil, ErrInvalidCacheType - } - - // Initialize frame. - f, err := db.newFrame(db.FramePath(name), name) - if err != nil { - return nil, err - } - - // Open frame. - if err := f.Open(); err != nil { - return nil, err - } - - // Default the time quantum to what is set on the DB. - if err := f.SetTimeQuantum(db.timeQuantum); err != nil { - f.Close() - return nil, err - } - - // Set cache type. - if opt.CacheType == "" { - opt.CacheType = DefaultCacheType - } - f.cacheType = opt.CacheType - - // Set options. - if opt.RowLabel != "" { - f.rowLabel = opt.RowLabel - } - if opt.CacheSize != 0 { - f.cacheSize = opt.CacheSize - } - - f.inverseEnabled = opt.InverseEnabled - if err := f.saveMeta(); err != nil { - f.Close() - return nil, err - } - - // Add to database's frame lookup. - db.frames[name] = f - - db.stats.Count("frameN", 1) - - return f, nil -} - -func (db *DB) newFrame(path, name string) (*Frame, error) { - f, err := NewFrame(path, db.name, name) - if err != nil { - return nil, err - } - f.LogOutput = db.LogOutput - f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - f.broadcaster = db.broadcaster - return f, nil -} - -// DeleteFrame removes a frame from the database. -func (db *DB) DeleteFrame(name string) error { - db.mu.Lock() - defer db.mu.Unlock() - - // Ignore if frame doesn't exist. - f := db.frame(name) - if f == nil { - return nil - } - - // Close frame. - if err := f.Close(); err != nil { - return err - } - - // Delete frame directory. - if err := os.RemoveAll(db.FramePath(name)); err != nil { - return err - } - - // Remove reference. - delete(db.frames, name) - - db.stats.Count("frameN", -1) - - return nil -} - -type dbSlice []*DB - -func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p dbSlice) Len() int { return len(p) } -func (p dbSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } - -// DBInfo represents schema information for a database. -type DBInfo struct { - Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` -} - -type dbInfoSlice []*DBInfo - -func (p dbInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p dbInfoSlice) Len() int { return len(p) } -func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } - -// MergeSchemas combines databases and frames from a and b into one schema. -func MergeSchemas(a, b []*DBInfo) []*DBInfo { - // Generate a map from both schemas. - m := make(map[string]map[string]map[string]struct{}) - for _, dbs := range [][]*DBInfo{a, b} { - for _, db := range dbs { - if m[db.Name] == nil { - m[db.Name] = make(map[string]map[string]struct{}) - } - for _, frame := range db.Frames { - if m[db.Name][frame.Name] == nil { - m[db.Name][frame.Name] = make(map[string]struct{}) - } - for _, view := range frame.Views { - m[db.Name][frame.Name][view.Name] = struct{}{} - } - } - } - } - - // Generate new schema from map. - dbs := make([]*DBInfo, 0, len(m)) - for db, frames := range m { - di := &DBInfo{Name: db} - for frame, views := range frames { - fi := &FrameInfo{Name: frame} - for view := range views { - fi.Views = append(fi.Views, &ViewInfo{Name: view}) - } - sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) - } - sort.Sort(frameInfoSlice(di.Frames)) - dbs = append(dbs, di) - } - sort.Sort(dbInfoSlice(dbs)) - - return dbs -} - -// encodeDBs converts a into its internal representation. -func encodeDBs(a []*DB) []*internal.DB { - other := make([]*internal.DB, len(a)) - for i := range a { - other[i] = encodeDB(a[i]) - } - return other -} - -// encodeDB converts d into its internal representation. -func encodeDB(d *DB) *internal.DB { - return &internal.DB{ - Name: d.name, - Meta: &internal.DBMeta{ - ColumnLabel: d.columnLabel, - TimeQuantum: string(d.timeQuantum), - }, - MaxSlice: d.remoteMaxSlice, - Frames: encodeFrames(d.Frames()), - } -} - -// DBOptions represents options to set when initializing a db. -type DBOptions struct { - ColumnLabel string `json:"columnLabel,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` -} - -// Encode converts o into its internal representation. -func (o *DBOptions) Encode() *internal.DBMeta { - return &internal.DBMeta{ - ColumnLabel: o.ColumnLabel, - TimeQuantum: string(o.TimeQuantum), - } -} - -// hasTime returns true if a contains a non-nil time. -func hasTime(a []*time.Time) bool { - for _, t := range a { - if t != nil { - return true - } - } - return false -} - -type importKey struct { - View string - Slice uint64 -} - -type importData struct { - RowIDs []uint64 - ColumnIDs []uint64 -} diff --git a/db_test.go b/db_test.go deleted file mode 100644 index f7cfc906a..000000000 --- a/db_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package pilosa_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pilosa/pilosa" -) - -// Ensure database can open and retrieve a frame. -func TestDB_CreateFrameIfNotExists(t *testing.T) { - db := MustOpenDB() - defer db.Close() - - // Create frame. - f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if f == nil { - t.Fatal("expected frame") - } - - // Retrieve existing frame. - other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if f.Frame != other.Frame { - t.Fatal("frame mismatch") - } - - if f.Frame != db.Frame("f") { - t.Fatal("frame mismatch") - } -} - -// Ensure database defaults the time quantum on new frames. -func TestDB_CreateFrame_TimeQuantum(t *testing.T) { - db := MustOpenDB() - defer db.Close() - - // Set database time quantum. - if err := db.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - - // Create frame. - f, err := db.CreateFrame("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { - t.Fatalf("unexpected frame time quantum: %s", q) - } -} - -// Ensure database can delete a frame. -func TestDB_DeleteFrame(t *testing.T) { - db := MustOpenDB() - defer db.Close() - - // Create frame. - if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Delete frame & verify it's gone. - if err := db.DeleteFrame("f"); err != nil { - t.Fatal(err) - } else if db.Frame("f") != nil { - t.Fatal("expected nil frame") - } - - // Delete again to make sure it doesn't error. - if err := db.DeleteFrame("f"); err != nil { - t.Fatal(err) - } -} - -// Ensure database can set the default time quantum. -func TestDB_SetTimeQuantum(t *testing.T) { - db := MustOpenDB() - defer db.Close() - - // Set & retrieve time quantum. - if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload database and verify that it is persisted. - if err := db.Reopen(); err != nil { - t.Fatal(err) - } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - -// DB represents a test wrapper for pilosa.DB. -type DB struct { - *pilosa.DB -} - -// NewDB returns a new instance of DB d. -func NewDB() *DB { - path, err := ioutil.TempDir("", "pilosa-db-") - if err != nil { - panic(err) - } - db, err := pilosa.NewDB(path, "d") - if err != nil { - panic(err) - } - return &DB{DB: db} -} - -// MustOpenDB returns a new, opened database at a temporary path. Panic on error. -func MustOpenDB() *DB { - db := NewDB() - if err := db.Open(); err != nil { - panic(err) - } - return db -} - -// Close closes the database and removes the underlying data. -func (db *DB) Close() error { - defer os.RemoveAll(db.Path()) - return db.DB.Close() -} - -// Reopen closes the database and reopens it. -func (db *DB) Reopen() error { - var err error - if err := db.DB.Close(); err != nil { - return err - } - - path, name := db.Path(), db.Name() - db.DB, err = pilosa.NewDB(path, name) - if err != nil { - return err - } - - if err := db.Open(); err != nil { - return err - } - return nil -} - -// CreateFrame creates a frame with the given options. -func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := db.DB.CreateFrame(name, opt) - if err != nil { - return nil, err - } - return &Frame{Frame: f}, nil -} - -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := db.DB.CreateFrameIfNotExists(name, opt) - if err != nil { - return nil, err - } - return &Frame{Frame: f}, nil -} - -// Ensure database can delete a frame. -func TestDB_InvalidName(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-db-") - if err != nil { - panic(err) - } - db, err := pilosa.NewDB(path, "ABC") - if db != nil { - t.Fatalf("unexpected db name %s", db) - } -} diff --git a/executor.go b/executor.go index 11ffcee47..8c7fef54a 100644 --- a/executor.go +++ b/executor.go @@ -45,10 +45,10 @@ func NewExecutor() *Executor { } // Execute executes a PQL query. -func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { - // Verify that a database is set. - if db == "" { - return nil, ErrDatabaseRequired +func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { + // Verify that an index is set. + if index == "" { + return nil, ErrIndexRequired } // Default options. @@ -60,7 +60,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices if len(slices) == 0 { if needsSlices(q.Calls) { // Round up the number of slices. - maxSlice := e.Holder.DB(db).MaxSlice() + maxSlice := e.Holder.Index(index).MaxSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) @@ -72,13 +72,13 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices // Optimize handling for bulk attribute insertion. if hasOnlySetRowAttrs(q.Calls) { - return e.executeBulkSetRowAttrs(ctx, db, q.Calls, opt) + return e.executeBulkSetRowAttrs(ctx, index, q.Calls, opt) } // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - v, err := e.executeCall(ctx, db, call, slices, opt) + v, err := e.executeCall(ctx, index, call, slices, opt) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices } // executeCall executes a call. -func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { +func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { if err := e.validateCallArgs(c); err != nil { return nil, err @@ -97,19 +97,19 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic // Special handling for mutation and top-n calls. switch c.Name { case "ClearBit": - return e.executeClearBit(ctx, db, c, opt) + return e.executeClearBit(ctx, index, c, opt) case "Count": - return e.executeCount(ctx, db, c, slices, opt) + return e.executeCount(ctx, index, c, slices, opt) case "SetBit": - return e.executeSetBit(ctx, db, c, opt) + return e.executeSetBit(ctx, index, c, opt) case "SetRowAttrs": - return nil, e.executeSetRowAttrs(ctx, db, c, opt) + return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": - return nil, e.executeSetColumnAttrs(ctx, db, c, opt) + return nil, e.executeSetColumnAttrs(ctx, index, c, opt) case "TopN": - return e.executeTopN(ctx, db, c, slices, opt) + return e.executeTopN(ctx, index, c, slices, opt) default: - return e.executeBitmapCall(ctx, db, c, slices, opt) + return e.executeBitmapCall(ctx, index, c, slices, opt) } } @@ -133,10 +133,10 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { +func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeBitmapCallSlice(ctx, db, c, slice) + return e.executeBitmapCallSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -149,7 +149,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call return other } - other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -160,7 +160,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { - d := e.Holder.DB(db) + d := e.Holder.Index(index) if d != nil { columnLabel := d.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { @@ -193,18 +193,18 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call } // executeBitmapCallSlice executes a bitmap call for a single slice. -func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { +func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { switch c.Name { case "Bitmap": - return e.executeBitmapSlice(ctx, db, c, slice) + return e.executeBitmapSlice(ctx, index, c, slice) case "Difference": - return e.executeDifferenceSlice(ctx, db, c, slice) + return e.executeDifferenceSlice(ctx, index, c, slice) case "Intersect": - return e.executeIntersectSlice(ctx, db, c, slice) + return e.executeIntersectSlice(ctx, index, c, slice) case "Range": - return e.executeRangeSlice(ctx, db, c, slice) + return e.executeRangeSlice(ctx, index, c, slice) case "Union": - return e.executeUnionSlice(ctx, db, c, slice) + return e.executeUnionSlice(ctx, index, c, slice) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -213,7 +213,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -224,7 +224,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic } // Execute original query. - pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt) + pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt) if err != nil { return nil, err } @@ -241,7 +241,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids - trimmedList, err := e.executeTopNSlices(ctx, db, other, slices, opt) + trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt) if err != nil { return nil, err } @@ -252,10 +252,10 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic return trimmedList, nil } -func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { +func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeTopNSlice(ctx, db, c, slice) + return e.executeTopNSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -264,7 +264,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call return Pairs(other).Add(v.([]Pair)) } - other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn) + other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { return nil, err } @@ -277,7 +277,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call } // executeTopNSlice executes a TopN call for a single slice. -func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) { +func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) n, _, err := c.UintArg("n") if err != nil { @@ -301,7 +301,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, // Retrieve bitmap used to intersect. var src *Bitmap if len(c.Children) == 1 { - bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice) + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { return nil, err } @@ -315,7 +315,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, frame = DefaultFrame } - f := e.Holder.Fragment(db, frame, ViewStandard, slice) + f := e.Holder.Fragment(index, frame, ViewStandard, slice) if f == nil { return nil, nil } @@ -339,13 +339,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, } // executeDifferenceSlice executes a difference() call for a local slice. -func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { +func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { var other *Bitmap if len(c.Children) == 0 { return nil, fmt.Errorf("empty Difference query is currently not supported") } for i, input := range c.Children { - bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) + bm, err := e.executeBitmapCallSlice(ctx, index, input, slice) if err != nil { return nil, err } @@ -360,11 +360,11 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql return other, nil } -func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { - // Fetch column label from database. - d := e.Holder.DB(db) +func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + // Fetch column label from index. + d := e.Holder.Index(index) if d == nil { - return nil, ErrDatabaseNotFound + return nil, ErrIndexNotFound } columnLabel := d.ColumnLabel() @@ -373,7 +373,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal if frame == "" { frame = DefaultFrame } - f := e.Holder.Frame(db, frame) + f := e.Holder.Frame(index, frame) if f == nil { return nil, ErrFrameNotFound } @@ -400,7 +400,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal } } - frag := e.Holder.Fragment(db, frame, view, slice) + frag := e.Holder.Fragment(index, frame, view, slice) if frag == nil { return NewBitmap(), nil } @@ -408,13 +408,13 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal } // executeIntersectSlice executes a intersect() call for a local slice. -func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { +func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { var other *Bitmap if len(c.Children) == 0 { return nil, fmt.Errorf("empty Intersect query is currently not supported") } for i, input := range c.Children { - bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) + bm, err := e.executeBitmapCallSlice(ctx, index, input, slice) if err != nil { return nil, err } @@ -430,7 +430,7 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql. } // executeRangeSlice executes a range() call for a local slice. -func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { +func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { // Parse frame, use default if unset. frame, _ := c.Args["frame"].(string) if frame == "" { @@ -438,7 +438,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call } // Retrieve base frame. - f := e.Holder.Frame(db, frame) + f := e.Holder.Frame(index, frame) if f == nil { return nil, ErrFrameNotFound } @@ -479,7 +479,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call // Union bitmaps across all time-based subframes. bm := &Bitmap{} for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(db, frame, view, slice) + f := e.Holder.Fragment(index, frame, view, slice) if f == nil { continue } @@ -489,10 +489,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call } // executeUnionSlice executes a union() call for a local slice. -func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { +func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { other := NewBitmap() for i, input := range c.Children { - bm, err := e.executeBitmapCallSlice(ctx, db, input, slice) + bm, err := e.executeBitmapCallSlice(ctx, index, input, slice) if err != nil { return nil, err } @@ -508,7 +508,7 @@ func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call } // executeCount executes a count() call. -func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { +func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { if len(c.Children) == 0 { return 0, errors.New("Count() requires an input bitmap") } else if len(c.Children) > 1 { @@ -517,7 +517,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice) + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { return 0, err } @@ -530,7 +530,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli return other + v.(uint64) } - result, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn) + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { return 0, err } @@ -540,7 +540,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli } // executeClearBit executes a ClearBit() call. -func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) { +func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { @@ -548,9 +548,9 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, } // Retrieve frame. - d := e.Holder.DB(db) + d := e.Holder.Index(index) if d == nil { - return false, ErrDatabaseNotFound + return false, ErrIndexNotFound } f := d.Frame(frame) if f == nil { @@ -579,19 +579,19 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, // Clear bits for each view. switch view { case ViewStandard: - return e.executeClearBitView(ctx, db, c, f, view, colID, rowID, opt) + return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt) case ViewInverse: - return e.executeClearBitView(ctx, db, c, f, view, rowID, colID, opt) + return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt) case "": var ret bool - if changed, err := e.executeClearBitView(ctx, db, c, f, ViewStandard, colID, rowID, opt); err != nil { + if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil { return ret, err } else if changed { ret = true } if f.InverseEnabled() { - if changed, err := e.executeClearBitView(ctx, db, c, f, ViewInverse, rowID, colID, opt); err != nil { + if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil { return ret, err } else if changed { ret = true @@ -604,10 +604,10 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, } // executeClearBitView executes a ClearBit() call for a single view. -func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { +func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(db, slice) { + for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. if node.Host == e.Host { val, err := f.ClearBit(view, rowID, colID, nil) @@ -624,7 +624,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -634,7 +634,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) { +func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { @@ -642,9 +642,9 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op } // Retrieve frame. - d := e.Holder.DB(db) + d := e.Holder.Index(index) if d == nil { - return false, ErrDatabaseNotFound + return false, ErrIndexNotFound } f := d.Frame(frame) if f == nil { @@ -683,19 +683,19 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op // Set bits for each view. switch view { case ViewStandard: - return e.executeSetBitView(ctx, db, c, f, view, colID, rowID, timestamp, opt) + return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt) case ViewInverse: - return e.executeSetBitView(ctx, db, c, f, view, rowID, colID, timestamp, opt) + return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt) case "": var ret bool - if changed, err := e.executeSetBitView(ctx, db, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil { + if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil { return ret, err } else if changed { ret = true } if f.InverseEnabled() { - if changed, err := e.executeSetBitView(ctx, db, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil { + if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil { return ret, err } else if changed { ret = true @@ -708,11 +708,11 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op } // executeSetBitView executes a SetBit() call for a specific view. -func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { +func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(db, slice) { + for _, node := range e.Cluster.FragmentNodes(index, slice) { // Update locally if host matches. if node.Host == e.Host { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -730,7 +730,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call } // Forward call to remote node otherwise. - if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { + if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil { return false, err } else { ret = res[0].(bool) @@ -740,14 +740,14 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call } // executeSetRowAttrs executes a SetRowAttrs() call. -func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { +func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { frameName, ok := c.Args["frame"].(string) if !ok { return errors.New("SetRowAttrs() frame required") } // Retrieve frame. - frame := e.Holder.Frame(db, frameName) + frame := e.Holder.Frame(index, frameName) if frame == nil { return ErrFrameNotFound } @@ -781,7 +781,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Cal resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -797,7 +797,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Cal } // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. -func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { +func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { // Collect attributes by frame/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { @@ -807,7 +807,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls } // Retrieve frame. - f := e.Holder.Frame(db, frame) + f := e.Holder.Frame(index, frame) if f == nil { return nil, ErrFrameNotFound } @@ -846,7 +846,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls // Bulk insert attributes by frame. for name, frameMap := range m { // Retrieve frame. - frame := e.Holder.Frame(db, name) + frame := e.Holder.Frame(index, name) if frame == nil { return nil, ErrFrameNotFound } @@ -867,7 +867,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, db, &pql.Query{Calls: calls}, nil, opt) + _, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt) resp <- err }(node) } @@ -884,11 +884,11 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls } // executeSetColumnAttrs executes a SetColumnAttrs() call. -func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { - // Retrieve database. - d := e.Holder.DB(db) +func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { + // Retrieve index. + d := e.Holder.Index(index) if d == nil { - return ErrDatabaseNotFound + return ErrIndexNotFound } var colName string @@ -925,7 +925,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql. resp := make(chan error, len(nodes)) for _, node := range nodes { go func(node *Node) { - _, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) + _, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt) resp <- err }(node) } @@ -941,7 +941,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql. } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { +func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ Query: q.String(), @@ -957,7 +957,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query req, err := http.NewRequest("POST", (&url.URL{ Scheme: "http", Host: node.Host, - Path: fmt.Sprintf("/db/%s/query", db), + Path: fmt.Sprintf("/index/%s/query", index), }).String(), bytes.NewReader(buf)) if err != nil { return nil, err @@ -1027,12 +1027,12 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query // slicesByNode returns a mapping of nodes to slices. // Returns errSliceUnavailable if a slice cannot be allocated to a node. -func (e *Executor) slicesByNode(nodes []*Node, db string, slices []uint64) (map[*Node][]uint64, error) { +func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) { m := make(map[*Node][]uint64) loop: for _, slice := range slices { - for _, node := range e.Cluster.FragmentNodes(db, slice) { + for _, node := range e.Cluster.FragmentNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop @@ -1047,7 +1047,7 @@ loop: // // If a mapping of slices to a node fails then the slices are resplit across // secondary nodes and retried. This continues to occur until all nodes are exhausted. -func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { +func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) { ch := make(chan mapResponse, 0) // Wrap context with a cancel to kill goroutines on exit. @@ -1066,7 +1066,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c } // Start mapping across all primary owners. - if err := e.mapper(ctx, ch, nodes, db, slices, c, opt, mapFn, reduceFn); err != nil { + if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil { return nil, err } @@ -1085,7 +1085,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c nodes = Nodes(nodes).Filter(resp.node) // Begin mapper against secondary nodes. - if err := e.mapper(ctx, ch, nodes, db, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable { + if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable { return nil, resp.err } else if err != nil { return nil, err @@ -1105,9 +1105,9 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c } } -func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { +func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error { // Group slices together by nodes. - m, err := e.slicesByNode(nodes, db, slices) + m, err := e.slicesByNode(nodes, index, slices) if err != nil { return err } @@ -1122,7 +1122,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) + results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] } diff --git a/executor_test.go b/executor_test.go index 0be31c73b..8fed950b4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -17,8 +17,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Row", func(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - f, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } @@ -26,7 +26,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits. - if _, err := e.Execute(context.Background(), "d", MustParse(``+ + if _, err := e.Execute(context.Background(), "i", MustParse(``+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), @@ -37,7 +37,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) @@ -49,26 +49,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Run("Column", func(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - if _, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits. - if _, err := e.Execute(context.Background(), "d", MustParse(``+ + if _, err := e.Execute(context.Background(), "i", MustParse(``+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } - if err := db.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -82,14 +82,14 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { t.Fatalf("unexpected bits: %+v", bits) @@ -100,10 +100,10 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } @@ -112,16 +112,16 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -134,7 +134,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { defer hldr.Close() e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect()`), nil, nil); err == nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } @@ -143,15 +143,15 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { t.Fatalf("unexpected bits: %+v", bits) @@ -162,10 +162,10 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected bits: %+v", bits) @@ -176,12 +176,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -194,12 +194,12 @@ func TestExecutor_Execute_SetBit(t *testing.T) { defer hldr.Close() e := NewExecutor(hldr.Holder, NewCluster(1)) - f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -210,7 +210,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -225,30 +225,30 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { defer hldr.Close() // Create frames. - db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) - if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(hldr.Holder, NewCluster(1)) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } - f := hldr.Frame("d", "f") + f := hldr.Frame("i", "f") if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { @@ -262,19 +262,19 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0) // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: 0, Count: 5}, @@ -288,16 +288,16 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, @@ -311,27 +311,27 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -346,23 +346,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer hldr.Close() // Set bits for rows 0, 10, & 20 across two slices. - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) // Create an intersecting row. - hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -378,15 +378,15 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -401,15 +401,15 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -424,11 +424,11 @@ func TestExecutor_Execute_Range(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - // Create database. - db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + // Create index. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) // Create frame. - f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { @@ -448,7 +448,7 @@ func TestExecutor_Execute_Range(t *testing.T) { f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row e := NewExecutor(hldr.Holder, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) @@ -465,14 +465,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != `d` { - t.Fatalf("unexpected db: %s", db) + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "i" { + t.Fatalf("unexpected index: %s", index) } else if query.String() != `Bitmap(frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) - // NOTE: while the following is technically incorrect (it should be {0, 2}) because the calling node doesn't know about slice 2 yet, - // we are ok with this and assuming that the calling node will become aware of slice 2 via inter-node messaging - } else if !reflect.DeepEqual(slices, []uint64{0}) { + } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) } @@ -489,12 +487,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) { + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected bits: %+v", bits) } } @@ -509,18 +507,18 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(10)}, nil } // Create local executor data. The local node owns slice 1. hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -539,9 +537,9 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != `d` { - t.Fatalf("unexpected db: %s", db) + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != `i` { + t.Fatalf("unexpected index: %s", index) } else if query.String() != `SetBit(columnID=2, frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) } @@ -554,17 +552,17 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { defer hldr.Close() // Create frame. - if _, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { + if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -584,9 +582,9 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != `d` { - t.Fatalf("unexpected db: %s", db) + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != `i` { + t.Fatalf("unexpected index: %s", index) } else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } @@ -599,19 +597,19 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { defer hldr.Close() // Create frame. - if f, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) } e := NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -630,10 +628,10 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Mock secondary server's executor to verify arguments and return a bitmap. var remoteExecN int - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != `d` { - t.Fatalf("unexpected db: %s", db) - } else if !reflect.DeepEqual(slices, []uint64{0, 2}) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "i" { + t.Fatalf("unexpected index: %s", index) + } else if !reflect.DeepEqual(slices, []uint64{1, 3}) { t.Fatalf("unexpected slices: %+v", slices) } @@ -661,14 +659,14 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { }}, nil } - // Create local executor data on slice 1 & 3. + // Create local executor data on slice 2 & 4. hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) e := NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, diff --git a/fragment.go b/fragment.go index 9f68df30e..ce5109684 100644 --- a/fragment.go +++ b/fragment.go @@ -50,12 +50,12 @@ const ( DefaultFragmentMaxOpN = 2000 ) -// Fragment represents the intersection of a frame and slice in a database. +// Fragment represents the intersection of a frame and slice in an index. type Fragment struct { mu sync.Mutex // Composite identifiers - db string + index string frame string view string slice uint64 @@ -94,10 +94,10 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment. -func NewFragment(path, db, frame, view string, slice uint64) *Fragment { +func NewFragment(path, index, frame, view string, slice uint64) *Fragment { return &Fragment{ path: path, - db: db, + index: index, frame: frame, view: view, slice: slice, @@ -117,8 +117,8 @@ func (f *Fragment) Path() string { return f.path } // CachePath returns the path to the fragment's cache data. func (f *Fragment) CachePath() string { return f.path + CacheExt } -// DB returns the database the fragment was initialized with. -func (f *Fragment) DB() string { return f.db } +// Index returns the index that the fragment was initialized with. +func (f *Fragment) Index() string { return f.index } // Frame returns the frame the fragment was initialized with. func (f *Fragment) Frame() string { return f.frame } @@ -1002,8 +1002,8 @@ func track(start time.Time, name string, logger *log.Logger) { func (f *Fragment) snapshot() error { logger := f.logger() - logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice) - defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger) + logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice), logger) // Create a temporary file to snapshot to. snapshotPath := f.path + SnapshotExt @@ -1307,7 +1307,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. - nodes := s.Cluster.FragmentNodes(s.Fragment.DB(), s.Fragment.Slice()) + nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice()) if len(nodes) == 1 { return nil } @@ -1327,7 +1327,7 @@ func (s *FragmentSyncer) SyncFragment() error { if err != nil { return err } - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) + blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice()) if err != nil && err != ErrFragmentNotFound { return err } @@ -1392,7 +1392,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet var clients []*Client - for _, node := range s.Cluster.FragmentNodes(f.DB(), f.Slice()) { + for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { if s.Host == node.Host { continue } @@ -1409,7 +1409,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), ViewStandard, f.Slice(), id) if err != nil { return err } @@ -1457,7 +1457,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Execute query. - _, err := clients[i].ExecuteQuery(context.Background(), f.DB(), buf.String(), false) + _, err := clients[i].ExecuteQuery(context.Background(), f.Index(), buf.String(), false) if err != nil { return err } diff --git a/fragment_test.go b/fragment_test.go index fd5dc95ac..f4d3fc247 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -54,7 +54,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -81,7 +81,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set and then clear bits on the fragment. @@ -110,7 +110,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -139,7 +139,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the rows 100, 101, & 102. @@ -161,7 +161,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the rows 100, 101, & 102. @@ -191,7 +191,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input row. @@ -221,7 +221,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Create an intersecting input row. @@ -258,7 +258,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on various rows. @@ -282,12 +282,12 @@ func TestFragment_TopN_CacheSize(t *testing.T) { slice := uint64(0) cacheSize := uint32(3) - // Create DB. - db := MustOpenDB() - defer db.Close() + // Create Index. + index := MustOpenIndex() + defer index.Close() // Create frame. - frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } @@ -346,7 +346,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve checksum and set bits. @@ -365,7 +365,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Retrieve initial checksum. @@ -403,7 +403,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on a different block. @@ -421,7 +421,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() // Set bits on the fragment. @@ -453,11 +453,11 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - db := MustOpenDB() - defer db.Close() + index := MustOpenIndex() + defer index.Close() // Create frame. - frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { t.Fatal(err) } @@ -488,13 +488,13 @@ func TestFragment_RankCache_Persistence(t *testing.T) { t.Fatalf("unexpected cache len: %d", cache.Len()) } - // Reopen the database. - if err := db.Reopen(); err != nil { + // Reopen the index. + if err := index.Reopen(); err != nil { t.Fatal(err) } // Re-fetch fragment. - f = db.Frame("f").View(pilosa.ViewStandard).Fragment(0) + f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0) // Re-verify correct cache type and size. if cache, ok := f.Cache().(*pilosa.RankCache); !ok { @@ -506,7 +506,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f0 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f0.Close() // Set and then clear bits on the fragment. @@ -531,7 +531,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f1 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -564,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0) + f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -580,7 +580,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() f.MaxOpN = math.MaxInt32 @@ -617,7 +617,7 @@ type Fragment struct { } // NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(db, frame, view string, slice uint64) *Fragment { +func NewFragment(index, frame, view string, slice uint64) *Fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") if err != nil { panic(err) @@ -625,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), RowAttrStore: MustOpenAttrStore(), } f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore @@ -633,8 +633,8 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { } // MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(db, frame, view string, slice uint64) *Fragment { - f := NewFragment(db, frame, view, slice) +func MustOpenFragment(index, frame, view string, slice uint64) *Fragment { + f := NewFragment(index, frame, view, slice) if err := f.Open(); err != nil { panic(err) } @@ -656,7 +656,7 @@ func (f *Fragment) Reopen() error { return err } - f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) + f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { return err @@ -720,7 +720,7 @@ func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { } func TestFragment_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) @@ -742,7 +742,7 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) + f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0) defer f.Close() src := pilosa.NewBitmap(1, 2, 3) diff --git a/frame.go b/frame.go index df4c8182f..14514cd01 100644 --- a/frame.go +++ b/frame.go @@ -29,7 +29,7 @@ const ( type Frame struct { mu sync.Mutex path string - db string + index string name string timeQuantum TimeQuantum @@ -53,16 +53,16 @@ type Frame struct { } // NewFrame returns a new instance of frame. -func NewFrame(path, db, name string) (*Frame, error) { +func NewFrame(path, index, name string) (*Frame, error) { err := ValidateName(name) if err != nil { return nil, err } return &Frame{ - path: path, - db: db, - name: name, + path: path, + index: index, + name: name, views: make(map[string]*View), rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), @@ -81,8 +81,8 @@ func NewFrame(path, db, name string) (*Frame, error) { // Name returns the name the frame was initialized with. func (f *Frame) Name() string { return f.name } -// DB returns the database name the frame was initialized with. -func (f *Frame) DB() string { return f.db } +// Index returns the index name the frame was initialized with. +func (f *Frame) Index() string { return f.index } // Path returns the path the frame was initialized with. func (f *Frame) Path() string { return f.path } @@ -418,7 +418,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { } func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.db, f.name, name, f.cacheSize) + view := NewView(path, f.index, f.name, name, f.cacheSize) view.cacheType = f.cacheType view.LogOutput = f.LogOutput view.RowAttrStore = f.rowAttrStore @@ -515,7 +515,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // Determine quantum if timestamps are set. q := f.TimeQuantum() if hasTime(timestamps) && q == "" { - return errors.New("time quantum not set in either database or frame") + return errors.New("time quantum not set in either index or frame") } // Split import data by fragment. diff --git a/frame_test.go b/frame_test.go index 0d10841db..669ffcc40 100644 --- a/frame_test.go +++ b/frame_test.go @@ -60,7 +60,7 @@ func TestFrame_NameRestriction(t *testing.T) { if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "d", ".meta") + frame, err := pilosa.NewFrame(path, "i", ".meta") if frame != nil { t.Fatalf("unexpected frame name %s", err) } @@ -77,7 +77,7 @@ func NewFrame() *Frame { if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "d", "f") + frame, err := pilosa.NewFrame(path, "i", "f") if err != nil { panic(err) } @@ -99,15 +99,15 @@ func (f *Frame) Close() error { return f.Frame.Close() } -// Reopen closes the database and reopens it. +// Reopen closes the index and reopens it. func (f *Frame) Reopen() error { var err error if err := f.Frame.Close(); err != nil { return err } - path, db, name := f.Path(), f.DB(), f.Name() - f.Frame, err = pilosa.NewFrame(path, db, name) + path, index, name := f.Path(), f.Index(), f.Name() + f.Frame, err = pilosa.NewFrame(path, index, name) if err != nil { return err } diff --git a/handler.go b/handler.go index e3d4d833b..904db8ce5 100644 --- a/handler.go +++ b/handler.go @@ -38,7 +38,7 @@ type Handler struct { // The execution engine for running queries. Executor interface { - Execute(context context.Context, db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } // The version to report on the /version endpoint. @@ -59,20 +59,20 @@ func NewHandler() *Handler { func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/db", handler.handleGetDBs).Methods("GET") - router.HandleFunc("/db/{db}", handler.handleGetDB).Methods("GET") - router.HandleFunc("/db/{db}", handler.handlePostDB).Methods("POST") - router.HandleFunc("/db/{db}", handler.handleDeleteDB).Methods("DELETE") - router.HandleFunc("/db/{db}/attr/diff", handler.handlePostDBAttrDiff).Methods("POST") - //router.HandleFunc("/db/{db}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. - router.HandleFunc("/db/{db}/frame/{frame}", handler.handlePostFrame).Methods("POST") - router.HandleFunc("/db/{db}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") - router.HandleFunc("/db/{db}/query", handler.handlePostQuery).Methods("POST") - router.HandleFunc("/db/{db}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/db/{db}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") - router.HandleFunc("/db/{db}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") - router.HandleFunc("/db/{db}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") - router.HandleFunc("/db/{db}/time-quantum", handler.handlePatchDBTimeQuantum).Methods("PATCH") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") + router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") + router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET") @@ -91,7 +91,7 @@ func NewRouter(handler *Handler) *mux.Router { // Ideally this would be automatic, as described in this (wontfix) ticket: // https://github.com/gorilla/mux/issues/6 // For now we just do it for the most commonly used handler, /query - router.HandleFunc("/db/{db}/query", handler.methodNotAllowedHandler).Methods("GET") + router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") return router } @@ -108,7 +108,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSchemaResponse{ - DBs: h.Holder.Schema(), + Indexes: h.Holder.Schema(), }); err != nil { h.logger().Printf("write schema response error: %s", err) } @@ -124,7 +124,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { } type getSchemaResponse struct { - DBs []*DBInfo `json:"dbs"` + Indexes []*IndexInfo `json:"indexes"` } type getStatusResponse struct { @@ -133,7 +133,7 @@ type getStatusResponse struct { // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] // Parse incoming request. req, err := h.readQueryRequest(r) @@ -157,7 +157,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Execute the query. - results, err := h.Executor.Execute(r.Context(), dbName, q, req.Slices, opt) + results, err := h.Executor.Execute(r.Context(), indexName, q, req.Slices, opt) resp := &QueryResponse{Results: results, Err: err} // Fill column attributes if requested. @@ -173,7 +173,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Retrieve column attributes across all calls. - columnAttrSets, err := h.readColumnAttrSets(h.Holder.DB(dbName), columnIDs) + columnAttrSets, err := h.readColumnAttrSets(h.Holder.Index(indexName), columnIDs) if err != nil { w.WriteHeader(http.StatusInternalServerError) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) @@ -219,40 +219,40 @@ type sliceMaxResponse struct { MaxSlices map[string]uint64 `json:"maxSlices"` } -// handleGetDBs handles GET /db request. -func (h *Handler) handleGetDBs(w http.ResponseWriter, r *http.Request) { +// handleGetIndexes handles GET /index request. +func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { h.handleGetSchema(w, r) } -// handleGetDB handles GET /db/ requests. -func (h *Handler) handleGetDB(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] - db := h.Holder.DB(dbName) - if db == nil { - http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) +// handleGetIndex handles GET /index/ requests. +func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } - if err := json.NewEncoder(w).Encode(getDBResponse{ - map[string]string{"name": db.Name()}, + if err := json.NewEncoder(w).Encode(getIndexResponse{ + map[string]string{"name": index.Name()}, }); err != nil { h.logger().Printf("write response error: %s", err) } } -type getDBResponse struct { - DB map[string]string `json:"db"` +type getIndexResponse struct { + Index map[string]string `json:"index"` } -type postDBRequest struct { - Options DBOptions `json:"options"` +type postIndexRequest struct { + Options IndexOptions `json:"options"` } -//_postDBRequest is necessary to avoid recursion while decoding. -type _postDBRequest postDBRequest +//_postIndexRequest is necessary to avoid recursion while decoding. +type _postIndexRequest postIndexRequest -// Custom Unmarshal JSON to validate request body when creating a new database -func (p *postDBRequest) UnmarshalJSON(b []byte) error { +// Custom Unmarshal JSON to validate request body when creating a new index. +func (p *postIndexRequest) UnmarshalJSON(b []byte) error { // m is an overflow map used to capture additional, unexpected keys. m := make(map[string]interface{}) @@ -260,13 +260,13 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error { return err } - validDBOptions := getValidOptions(DBOptions{}) - err := validateOptions(m, validDBOptions) + validIndexOptions := getValidOptions(IndexOptions{}) + err := validateOptions(m, validIndexOptions) if err != nil { return err } // Unmarshal expected values. - var _p _postDBRequest + var _p _postIndexRequest if err := json.Unmarshal(b, &_p); err != nil { return err } @@ -277,7 +277,7 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error { } // Raise errors for any unknown key -func validateOptions(data map[string]interface{}, validDBOptions []string) error { +func validateOptions(data map[string]interface{}, validIndexOptions []string) error { for k, v := range data { switch k { case "options": @@ -286,7 +286,7 @@ func validateOptions(data map[string]interface{}, validDBOptions []string) error return errors.New("options is not map[string]interface{}") } for kk, vv := range options { - if !foundItem(validDBOptions, kk) { + if !foundItem(validIndexOptions, kk) { return fmt.Errorf("Unknown key: %v:%v", kk, vv) } } @@ -306,53 +306,53 @@ func foundItem(items []string, item string) bool { return false } -type postDBResponse struct{} +type postIndexResponse struct{} -// handleDeleteDB handles DELETE /db request. -func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] +// handleDeleteIndex handles DELETE /index request. +func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] - // Delete database from the holder. - if err := h.Holder.DeleteDB(dbName); err != nil { + // Delete index from the holder. + if err := h.Holder.DeleteIndex(indexName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete database message to all nodes. + // Send the delete index message to all nodes. err := h.Broadcaster.SendSync( - &internal.DeleteDBMessage{ - DB: dbName, + &internal.DeleteIndexMessage{ + Index: indexName, }) if err != nil { - h.logger().Printf("problem sending DeleteDB message: %s", err) + h.logger().Printf("problem sending DeleteIndex message: %s", err) } // Encode response. - if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type deleteDBResponse struct{} +type deleteIndexResponse struct{} -// handlePostDB handles POST /db request. -func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] +// handlePostIndex handles POST /index request. +func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] // Decode request. - var req postDBRequest + var req postIndexRequest err := json.NewDecoder(r.Body).Decode(&req) if err == io.EOF { - // If no data was provided (EOF), we still create the database + // If no data was provided (EOF), we still create the index // with default values. } else if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - // Create database. - _, err = h.Holder.CreateDB(dbName, req.Options) - if err == ErrDatabaseExists { + // Create index. + _, err = h.Holder.CreateIndex(indexName, req.Options) + if err == ErrIndexExists { http.Error(w, err.Error(), http.StatusConflict) return } else if err != nil { @@ -360,28 +360,28 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { return } - // Send the create database message to all nodes. + // Send the create index message to all nodes. err = h.Broadcaster.SendSync( - &internal.CreateDBMessage{ - DB: dbName, - Meta: req.Options.Encode(), + &internal.CreateIndexMessage{ + Index: indexName, + Meta: req.Options.Encode(), }) if err != nil { - h.logger().Printf("problem sending CreateDB message: %s", err) + h.logger().Printf("problem sending CreateIndex message: %s", err) } // Encode response. - if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -// handlePatchDBTimeQuantum handles PATCH /db/time_quantum request. -func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] +// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. +func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] // Decode request. - var req patchDBTimeQuantumRequest + var req patchIndexTimeQuantumRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -394,51 +394,51 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques return } - // Retrieve database by name. - database := h.Holder.DB(dbName) - if database == nil { - http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) + // Retrieve index by name. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } - // Set default time quantum on database. - if err := database.SetTimeQuantum(tq); err != nil { + // Set default time quantum on index. + if err := index.SetTimeQuantum(tq); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Encode response. - if err := json.NewEncoder(w).Encode(patchDBTimeQuantumResponse{}); err != nil { + if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type patchDBTimeQuantumRequest struct { +type patchIndexTimeQuantumRequest struct { TimeQuantum string `json:"timeQuantum"` } -type patchDBTimeQuantumResponse struct{} +type patchIndexTimeQuantumResponse struct{} -// handlePostDBAttrDiff handles POST /db/attr/diff requests. -func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] +// handlePostIndexAttrDiff handles POST /index/attr/diff requests. +func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] // Decode request. - var req postDBAttrDiffRequest + var req postIndexAttrDiffRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - // Retrieve database from holder. - db := h.Holder.DB(dbName) - if db == nil { - http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) + // Retrieve index from holder. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } // Retrieve local blocks. - blks, err := db.ColumnAttrStore().Blocks() + blks, err := index.ColumnAttrStore().Blocks() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -448,37 +448,37 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { attrs := make(map[uint64]map[string]interface{}) for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { // Retrieve block data. - m, err := db.ColumnAttrStore().BlockData(blockID) + m, err := index.ColumnAttrStore().BlockData(blockID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Copy to database-wide struct. + // Copy to index-wide struct. for k, v := range m { attrs[k] = v } } // Encode response. - if err := json.NewEncoder(w).Encode(postDBAttrDiffResponse{ + if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ Attrs: attrs, }); err != nil { h.logger().Printf("response encoding error: %s", err) } } -type postDBAttrDiffRequest struct { +type postIndexAttrDiffRequest struct { Blocks []AttrBlock `json:"blocks"` } -type postDBAttrDiffResponse struct { +type postIndexAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } // handlePostFrame handles POST /frame request. func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] // Decode request. @@ -492,15 +492,15 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { return } - // Find database. - db := h.Holder.DB(dbName) - if db == nil { - http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } // Create frame. - _, err = db.CreateFrame(frameName, req.Options) + _, err = index.CreateFrame(frameName, req.Options) if err == ErrFrameExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -512,7 +512,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { // Send the create frame message to all nodes. err = h.Broadcaster.SendSync( &internal.CreateFrameMessage{ - DB: dbName, + Index: indexName, Frame: frameName, Meta: req.Options.Encode(), }) @@ -573,20 +573,20 @@ type postFrameResponse struct{} // handleDeleteFrame handles DELETE /frame request. func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Find database. - db := h.Holder.DB(dbName) - if db == nil { - if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { + // Find index. + index := h.Holder.Index(indexName) + if index == nil { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { h.logger().Printf("response encoding error: %s", err) } return } - // Delete frame from the database. - if err := db.DeleteFrame(frameName); err != nil { + // Delete frame from the index. + if err := index.DeleteFrame(frameName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -594,7 +594,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { // Send the delete frame message to all nodes. err := h.Broadcaster.SendSync( &internal.DeleteFrameMessage{ - DB: dbName, + Index: indexName, Frame: frameName, }) if err != nil { @@ -611,7 +611,7 @@ type deleteFrameResponse struct{} // handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] // Decode request. @@ -628,14 +628,14 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req return } - // Retrieve database by name. - f := h.Holder.Frame(dbName, frameName) + // Retrieve index by name. + f := h.Holder.Frame(indexName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } - // Set default time quantum on database. + // Set default time quantum on index. if err := f.SetTimeQuantum(tq); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -655,11 +655,11 @@ type patchFrameTimeQuantumResponse struct{} // handleGetFrameViews handles GET /frame/views request. func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] // Retrieve views. - f := h.Holder.Frame(dbName, frameName) + f := h.Holder.Frame(indexName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -684,7 +684,7 @@ type getFrameViewsResponse struct { // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] // Decode request. @@ -694,8 +694,8 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve database from holder. - f := h.Holder.Frame(dbName, frameName) + // Retrieve index from holder. + f := h.Holder.Frame(indexName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -718,7 +718,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Copy to database-wide struct. + // Copy to index-wide struct. for k, v := range m { attrs[k] = v } @@ -741,15 +741,15 @@ type postFrameAttrDiffResponse struct { } // readColumnAttrSets returns a list of column attribute objects by id. -func (h *Handler) readColumnAttrSets(db *DB, ids []uint64) ([]*ColumnAttrSet, error) { - if db == nil { +func (h *Handler) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { return nil, nil } a := make([]*ColumnAttrSet, 0, len(ids)) for _, id := range ids { // Read attributes for column. Skip column if empty. - attrs, err := db.ColumnAttrStore().Attrs(id) + attrs, err := index.ColumnAttrStore().Attrs(id) if err != nil { return nil, err } else if len(attrs) == 0 { @@ -884,25 +884,25 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Host, req.DB, req.Slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.DB, req.Slice) + if !h.Cluster.OwnsFragment(h.Host, req.Index, req.Slice) { + mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.Index, req.Slice) http.Error(w, mesg, http.StatusPreconditionFailed) return } - // Find the DB. - h.logger().Println("importing:", req.DB, req.Frame, req.Slice) - db := h.Holder.DB(req.DB) - if db == nil { - h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error()) - http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) + // Find the Index. + h.logger().Println("importing:", req.Index, req.Frame, req.Slice) + index := h.Holder.Index(req.Index) + if index == nil { + h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) + http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) return } // Retrieve frame. - f := db.Frame(req.Frame) + f := index.Frame(req.Frame) if f == nil { - h.logger().Printf("frame error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrFrameNotFound.Error()) + h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -910,7 +910,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Import into fragment. err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ColumnIDs), err) + h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -941,7 +941,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // Parse query parameters. q := r.URL.Query() - db, frame, view := q.Get("db"), q.Get("frame"), q.Get("view") + index, frame, view := q.Get("index"), q.Get("frame"), q.Get("view") slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) if err != nil { @@ -950,14 +950,14 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Host, db, slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, db, slice) + if !h.Cluster.OwnsFragment(h.Host, index, slice) { + mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, index, slice) http.Error(w, mesg, http.StatusPreconditionFailed) return } // Find the fragment. - f := h.Holder.Fragment(db, frame, view, slice) + f := h.Holder.Fragment(index, frame, view, slice) if f == nil { return } @@ -983,7 +983,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { // handleGetFragmentNodes handles /fragment/nodes requests. func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - db := q.Get("db") + index := q.Get("index") // Read slice parameter. slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) @@ -993,7 +993,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.Cluster.FragmentNodes(db, slice) + nodes := h.Cluster.FragmentNodes(index, slice) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1012,7 +1012,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) + f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1035,7 +1035,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve frame. - f := h.Holder.Frame(q.Get("db"), q.Get("frame")) + f := h.Holder.Frame(q.Get("index"), q.Get("frame")) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return @@ -1075,7 +1075,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from holder. - f := h.Holder.Fragment(req.DB, req.Frame, req.View, req.Slice) + f := h.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -1111,7 +1111,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request } // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice) + f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) if f == nil { http.Error(w, "fragment not found", http.StatusNotFound) return @@ -1134,7 +1134,7 @@ type getFragmentBlocksResponse struct { // handlePostFrameRestore handles POST /frame/restore requests. func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) { - dbName := mux.Vars(r)["db"] + indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] q := r.URL.Query() @@ -1154,30 +1154,30 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByDatabase(r.Context()) + maxSlices, err := client.MaxSliceByIndex(r.Context()) if err != nil { http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) return } // Retrieve frame. - f := h.Holder.Frame(dbName, frameName) + f := h.Holder.Frame(indexName, frameName) if f == nil { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } // Retrieve list of all views. - views, err := client.FrameViews(r.Context(), dbName, frameName) + views, err := client.FrameViews(r.Context(), indexName, frameName) if err != nil { http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) return } // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[dbName]; slice++ { + for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.Host, dbName, slice) { + if !h.Cluster.OwnsFragment(h.Host, indexName, slice) { continue } @@ -1198,7 +1198,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), dbName, frameName, view, slice) + rd, err := client.BackupSlice(r.Context(), indexName, frameName, view, slice) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -1262,8 +1262,8 @@ func (h *Handler) logger() *log.Logger { // QueryRequest represent a request to process a query. type QueryRequest struct { - // Database to execute query against. - DB string + // Index to execute query against. + Index string // The query string to parse and execute. Query string diff --git a/handler_internal_test.go b/handler_internal_test.go index 92221f1d7..f749c23aa 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -6,21 +6,21 @@ import ( "testing" ) -// Test custom UnmarshalJSON for postDBRequest object -func TestPostDBRequestUnmarshalJSON(t *testing.T) { +// Test custom UnmarshalJSON for postIndexRequest object +func TestPostIndexRequestUnmarshalJSON(t *testing.T) { tests := []struct { json string - expected postDBRequest + expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postDBRequest{Options: DBOptions{}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, - {json: `{"options": {"columnLabel": "test"}}`, expected: postDBRequest{Options: DBOptions{ColumnLabel: "test"}}}, + {json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}}, {json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"}, } for _, test := range tests { - actual := &postDBRequest{} + actual := &postIndexRequest{} err := json.Unmarshal([]byte(test.json), actual) if err != nil { diff --git a/handler_test.go b/handler_test.go index 77435fb20..fbf7041e8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -34,22 +34,22 @@ func TestHandler_Schema(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) - d1 := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -69,13 +69,13 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := NewHandler() h.Holder = hldr.Holder @@ -83,7 +83,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -93,7 +93,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - f0, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) + f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } @@ -105,7 +105,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { t.Fatal(err) } - f1, err := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) + f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) if err != nil { t.Fatal(err) } @@ -123,7 +123,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -131,9 +131,9 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != "db0" { - t.Fatalf("unexpected db: %s", db) + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "idx0" { + t.Fatalf("unexpected index: %s", index) } else if query.String() != `Count(Bitmap(id=100))` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { @@ -143,7 +143,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -154,9 +154,9 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - if db != "db0" { - t.Fatalf("unexpected db: %s", db) + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + if index != "idx0" { + t.Fatalf("unexpected index: %s", index) } else if query.String() != `Count(Bitmap(id=100))` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{0, 1}) { @@ -175,7 +175,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { } // Generate protobuf request. - req := MustNewHTTPRequest("POST", "/db/db0/query", bytes.NewReader(reqBody)) + req := MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody)) req.Header.Set("Content-Type", "application/x-protobuf") w := httptest.NewRecorder() @@ -188,7 +188,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Ensure the handler returns an error when parsing bad arguments. func TestHandler_Query_Args_Err(t *testing.T) { w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) + NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { @@ -199,12 +199,12 @@ func TestHandler_Query_Args_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { @@ -215,12 +215,12 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Count(Bitmap(id=100))")) + r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -238,14 +238,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return []interface{}{bm}, nil } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" { @@ -258,26 +258,26 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { hldr := NewHolder() defer hldr.Close() - // Create database and set column attributes. - db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) + // Create index and set column attributes. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { + } else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) } h := NewHandler() h.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return []interface{}{bm}, nil } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { @@ -288,14 +288,14 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return []interface{}{bm}, nil } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)")) + r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -323,17 +323,17 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { hldr := NewHolder() defer hldr.Close() - // Create database and set column attributes. - db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) + // Create index and set column attributes. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { + } else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) } h := NewHandler() h.Holder = hldr.Holder - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return []interface{}{bm}, nil @@ -349,7 +349,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/db/d/query", bytes.NewReader(buf)) + r := MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf)) r.Header.Set("Content-Type", "application/x-protobuf") r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) @@ -387,7 +387,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, {ID: 3, Count: 4}, @@ -395,7 +395,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { @@ -406,7 +406,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, {ID: 3, Count: 4}, @@ -414,7 +414,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusOK { @@ -432,12 +432,12 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`Bitmap(id=100)`))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) if w.Code != http.StatusInternalServerError { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { @@ -448,12 +448,12 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() - r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) if w.Code != http.StatusInternalServerError { @@ -471,7 +471,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { // Ensure the handler returns "method not allowed" for non-POST queries. func TestHandler_Query_MethodNotAllowed(t *testing.T) { w := httptest.NewRecorder() - NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil)) + NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } @@ -481,7 +481,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { func TestHandler_Query_ErrParse(t *testing.T) { h := NewHandler() w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("bad_fn("))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { @@ -489,8 +489,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { } } -// Ensure the handler can delete a database. -func TestHandler_DB_Delete(t *testing.T) { +// Ensure the handler can delete an index. +func TestHandler_Index_Delete(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -498,13 +498,13 @@ func TestHandler_DB_Delete(t *testing.T) { s.Handler.Holder = hldr.Holder defer s.Close() - // Create database. - if _, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil { + // Create index. + if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } - // Send request to delete database. - resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/db/d", strings.NewReader(""))) + // Send request to delete index. + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if err != nil { t.Fatal(err) } @@ -519,9 +519,9 @@ func TestHandler_DB_Delete(t *testing.T) { t.Fatalf("unexpected response body: %s", buf) } - // Verify database is gone. - if hldr.DB("d") != nil { - t.Fatal("expected nil database") + // Verify index is gone. + if hldr.Index("i") != nil { + t.Fatal("expected nil index") } } @@ -529,39 +529,39 @@ func TestHandler_DB_Delete(t *testing.T) { func TestHandler_DeleteFrame(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) - if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) + if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } h := NewHandler() h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/db/d0/frame/f1", strings.NewReader(""))) + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if f := hldr.DB("d0").Frame("f1"); f != nil { + } else if f := hldr.Index("i0").Frame("f1"); f != nil { t.Fatal("expected nil frame") } } -// Ensure handler can set the DB time quantum. -func TestHandler_SetDBTimeQuantum(t *testing.T) { +// Ensure handler can set the Index time quantum. +func TestHandler_SetIndexTimeQuantum(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := NewHandler() h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if q := hldr.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + } else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { t.Fatalf("unexpected time quantum: %s", q) } } @@ -572,25 +572,25 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { defer hldr.Close() // Create frame. - if _, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } h := NewHandler() h.Holder = hldr.Holder w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if q := hldr.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + } else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { t.Fatalf("unexpected time quantum: %s", q) } } -// Ensure the handler can return data in differing blocks for a database. -func TestHandler_DB_AttrStore_Diff(t *testing.T) { +// Ensure the handler can return data in differing blocks for an index. +func TestHandler_Index_AttrStore_Diff(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() @@ -598,21 +598,21 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { s.Handler.Holder = hldr.Holder defer s.Close() - // Set attributes on the database. - db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}) + // Set attributes on the index. + index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := db.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } // Retrieve block checksums. - blks, err := db.ColumnAttrStore().Blocks() + blks, err := index.ColumnAttrStore().Blocks() if err != nil { t.Fatal(err) } @@ -623,7 +623,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { // Send block checksums to determine diff. resp, err := http.Post( - s.URL+"/db/d/attr/diff", + s.URL+"/index/i/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`), ) @@ -647,8 +647,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { s.Handler.Holder = hldr.Holder defer s.Close() - // Set attributes on the database. - d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + // Set attributes on the index. + d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) f, err := d.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) @@ -673,7 +673,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Send block checksums to determine diff. resp, err := http.Post( - s.URL+"/db/d/frame/meta/attr/diff", + s.URL+"/index/i/frame/meta/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`), ) @@ -698,11 +698,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer s.Close() // Set bits in the index. - f0 := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) // Begin backing up from slice d/f/0. - resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0") + resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") if err != nil { t.Fatal(err) } @@ -714,12 +714,12 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { } // Create frame. - if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { + if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Restore backup to slice x/y/0. - if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { + if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { t.Fatal(err) } else if resp.StatusCode != http.StatusOK { resp.Body.Close() @@ -759,7 +759,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { h.Cluster.ReplicaN = 2 w := httptest.NewRecorder() - r := MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) + r := MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -802,13 +802,13 @@ func NewHandler() *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) + ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - return c.ExecuteFn(ctx, db, query, slices, opt) +func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + return c.ExecuteFn(ctx, index, query, slices, opt) } // Server represents a test wrapper for httptest.Server. diff --git a/holder.go b/holder.go index a70062150..054cffe57 100644 --- a/holder.go +++ b/holder.go @@ -20,8 +20,8 @@ const DefaultCacheFlushInterval = 1 * time.Minute type Holder struct { mu sync.Mutex - // Databases by name. - dbs map[string]*DB + // Indexes by name. + indexes map[string]*Index Broadcaster Broadcaster // Close management @@ -43,7 +43,7 @@ type Holder struct { // NewHolder returns a new instance of Holder. func NewHolder() *Holder { return &Holder{ - dbs: make(map[string]*DB), + indexes: make(map[string]*Index), closing: make(chan struct{}, 0), Stats: NopStatsClient, @@ -60,7 +60,7 @@ func (h *Holder) Open() error { return err } - // Open path to read all database directories. + // Open path to read all index directories. f, err := os.Open(h.Path) if err != nil { return err @@ -77,25 +77,25 @@ func (h *Holder) Open() error { continue } - h.logger().Printf("opening database: %s", filepath.Base(fi.Name())) + h.logger().Printf("opening index: %s", filepath.Base(fi.Name())) - db, err := h.newDB(h.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err == ErrName { - h.logger().Printf("ERROR opening database: %s, err=%s", fi.Name(), err) + h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err) continue } else if err != nil { return err } - if err := db.Open(); err != nil { + if err := index.Open(); err != nil { if err == ErrName { - h.logger().Printf("ERROR opening database: %s, err=%s", db.Name(), err) + h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue } - return fmt.Errorf("open db: name=%s, err=%s", db.Name(), err) + return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) } - h.dbs[db.Name()] = db + h.indexes[index.Name()] = index - h.Stats.Count("dbN", 1) + h.Stats.Count("indexN", 1) } // Periodically flush cache. @@ -111,36 +111,36 @@ func (h *Holder) Close() error { close(h.closing) h.wg.Wait() - for _, db := range h.dbs { - db.Close() + for _, index := range h.indexes { + index.Close() } return nil } -// MaxSlices returns MaxSlice map for all databases. +// MaxSlices returns MaxSlice map for all indexes. func (h *Holder) MaxSlices() map[string]uint64 { a := make(map[string]uint64) - for _, db := range h.DBs() { - a[db.Name()] = db.MaxSlice() + for _, index := range h.Indexes() { + a[index.Name()] = index.MaxSlice() } return a } -// MaxInverseSlices returns MaxInverseSlice map for all databases. +// MaxInverseSlices returns MaxInverseSlice map for all indexes. func (h *Holder) MaxInverseSlices() map[string]uint64 { a := make(map[string]uint64) - for _, db := range h.DBs() { - a[db.Name()] = db.MaxInverseSlice() + for _, index := range h.Indexes() { + a[index.Name()] = index.MaxInverseSlice() } return a } -// Schema returns schema data for all databases and frames. -func (h *Holder) Schema() []*DBInfo { - var a []*DBInfo - for _, db := range h.DBs() { - di := &DBInfo{Name: db.Name()} - for _, frame := range db.Frames() { +// Schema returns schema data for all indexes and frames. +func (h *Holder) Schema() []*IndexInfo { + var a []*IndexInfo + for _, index := range h.Indexes() { + di := &IndexInfo{Name: index.Name()} + for _, frame := range index.Frames() { fi := &FrameInfo{Name: frame.Name()} for _, view := range frame.Views() { fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) @@ -151,155 +151,155 @@ func (h *Holder) Schema() []*DBInfo { sort.Sort(frameInfoSlice(di.Frames)) a = append(a, di) } - sort.Sort(dbInfoSlice(a)) + sort.Sort(indexInfoSlice(a)) return a } -// DBPath returns the path where a given database is stored. -func (h *Holder) DBPath(name string) string { return filepath.Join(h.Path, name) } +// IndexPath returns the path where a given index is stored. +func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } -// DB returns the database by name. -func (h *Holder) DB(name string) *DB { +// Index returns the index by name. +func (h *Holder) Index(name string) *Index { h.mu.Lock() defer h.mu.Unlock() - return h.db(name) + return h.index(name) } -func (h *Holder) db(name string) *DB { return h.dbs[name] } +func (h *Holder) index(name string) *Index { return h.indexes[name] } -// DBs returns a list of all databases in the holder. -func (h *Holder) DBs() []*DB { +// Indexes returns a list of all indexes in the holder. +func (h *Holder) Indexes() []*Index { h.mu.Lock() defer h.mu.Unlock() - a := make([]*DB, 0, len(h.dbs)) - for _, db := range h.dbs { - a = append(a, db) + a := make([]*Index, 0, len(h.indexes)) + for _, index := range h.indexes { + a = append(a, index) } - sort.Sort(dbSlice(a)) + sort.Sort(indexSlice(a)) return a } -// CreateDB creates a database. -// An error is returned if the database already exists. -func (h *Holder) CreateDB(name string, opt DBOptions) (*DB, error) { +// CreateIndex creates an index. +// An error is returned if the index already exists. +func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { h.mu.Lock() defer h.mu.Unlock() - // Ensure db doesn't already exist. - if h.dbs[name] != nil { - return nil, ErrDatabaseExists + // Ensure index doesn't already exist. + if h.indexes[name] != nil { + return nil, ErrIndexExists } - return h.createDB(name, opt) + return h.createIndex(name, opt) } -// CreateDBIfNotExists returns a database by name. -// The database is created if it does not already exist. -func (h *Holder) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { +// CreateIndexIfNotExists returns an index by name. +// The index is created if it does not already exist. +func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error) { h.mu.Lock() defer h.mu.Unlock() - // Find database in cache first. - if db := h.dbs[name]; db != nil { - return db, nil + // Find index in cache first. + if index := h.indexes[name]; index != nil { + return index, nil } - return h.createDB(name, opt) + return h.createIndex(name, opt) } -func (h *Holder) createDB(name string, opt DBOptions) (*DB, error) { +func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { if name == "" { - return nil, errors.New("database name required") + return nil, errors.New("index name required") } - // Return database if it exists. - if db := h.db(name); db != nil { - return db, nil + // Return index if it exists. + if index := h.index(name); index != nil { + return index, nil } - // Otherwise create a new database. - db, err := h.newDB(h.DBPath(name), name) + // Otherwise create a new index. + index, err := h.newIndex(h.IndexPath(name), name) if err != nil { return nil, err } - if err := db.Open(); err != nil { + if err := index.Open(); err != nil { return nil, err } // Update options. - db.SetColumnLabel(opt.ColumnLabel) - db.SetTimeQuantum(opt.TimeQuantum) + index.SetColumnLabel(opt.ColumnLabel) + index.SetTimeQuantum(opt.TimeQuantum) - h.dbs[db.Name()] = db + h.indexes[index.Name()] = index - h.Stats.Count("dbN", 1) + h.Stats.Count("indexN", 1) - return db, nil + return index, nil } -func (h *Holder) newDB(path, name string) (*DB, error) { - db, err := NewDB(path, name) +func (h *Holder) newIndex(path, name string) (*Index, error) { + index, err := NewIndex(path, name) if err != nil { return nil, err } - db.LogOutput = h.LogOutput - db.stats = h.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - db.broadcaster = h.Broadcaster - return db, nil + index.LogOutput = h.LogOutput + index.stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) + index.broadcaster = h.Broadcaster + return index, nil } -// DeleteDB removes a database from the holder. -func (h *Holder) DeleteDB(name string) error { +// DeleteIndex removes an index from the holder. +func (h *Holder) DeleteIndex(name string) error { h.mu.Lock() defer h.mu.Unlock() - // Ignore if database doesn't exist. - db := h.db(name) - if db == nil { + // Ignore if index doesn't exist. + index := h.index(name) + if index == nil { return nil } - // Close database. - if err := db.Close(); err != nil { + // Close index. + if err := index.Close(); err != nil { return err } - // Delete database directory. - if err := os.RemoveAll(h.DBPath(name)); err != nil { + // Delete index directory. + if err := os.RemoveAll(h.IndexPath(name)); err != nil { return err } // Remove reference. - delete(h.dbs, name) + delete(h.indexes, name) - h.Stats.Count("dbN", -1) + h.Stats.Count("indexN", -1) return nil } -// Frame returns the frame for a database and name. -func (h *Holder) Frame(db, name string) *Frame { - d := h.DB(db) +// Frame returns the frame for an index and name. +func (h *Holder) Frame(index, name string) *Frame { + d := h.Index(index) if d == nil { return nil } return d.Frame(name) } -// View returns the view for a database, frame, and name. -func (h *Holder) View(db, frame, name string) *View { - f := h.Frame(db, frame) +// View returns the view for an index, frame, and name. +func (h *Holder) View(index, frame, name string) *View { + f := h.Frame(index, frame) if f == nil { return nil } return f.View(name) } -// Fragment returns the fragment for a database, frame & slice. -func (h *Holder) Fragment(db, frame, view string, slice uint64) *Fragment { - v := h.View(db, frame, view) +// Fragment returns the fragment for an index, frame & slice. +func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment { + v := h.View(index, frame, view) if v == nil { return nil } @@ -323,8 +323,8 @@ func (h *Holder) monitorCacheFlush() { } func (h *Holder) flushCaches() { - for _, db := range h.DBs() { - for _, frame := range db.Frames() { + for _, index := range h.Indexes() { + for _, frame := range index.Frames() { for _, view := range frame.Views() { for _, fragment := range view.Fragments() { select { @@ -375,9 +375,9 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - // Sync database column attributes. - if err := s.syncDatabase(di.Name); err != nil { - return fmt.Errorf("db sync error: db=%s, err=%s", di.Name, err) + // Sync index column attributes. + if err := s.syncIndex(di.Name); err != nil { + return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err) } for _, fi := range di.Frames { @@ -388,7 +388,7 @@ func (s *HolderSyncer) SyncHolder() error { // Sync frame row attributes. if err := s.syncFrame(di.Name, fi.Name); err != nil { - return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) + return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err) } for _, vi := range fi.Views { @@ -397,7 +397,7 @@ func (s *HolderSyncer) SyncHolder() error { return nil } - for slice := uint64(0); slice <= s.Holder.DB(di.Name).MaxSlice(); slice++ { + for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { continue @@ -410,7 +410,7 @@ func (s *HolderSyncer) SyncHolder() error { // Sync fragment if own it. if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) } } } @@ -420,10 +420,10 @@ func (s *HolderSyncer) SyncHolder() error { return nil } -// syncDatabase synchronizes database attributes with the rest of the cluster. -func (s *HolderSyncer) syncDatabase(db string) error { - // Retrieve database reference. - d := s.Holder.DB(db) +// syncIndex synchronizes index attributes with the rest of the cluster. +func (s *HolderSyncer) syncIndex(index string) error { + // Retrieve index reference. + d := s.Holder.Index(index) if d == nil { return nil } @@ -443,7 +443,7 @@ func (s *HolderSyncer) syncDatabase(db string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ColumnAttrDiff(context.Background(), db, blks) + m, err := client.ColumnAttrDiff(context.Background(), index, blks) if err != nil { return err } else if len(m) == 0 { @@ -466,9 +466,9 @@ func (s *HolderSyncer) syncDatabase(db string) error { } // syncFrame synchronizes frame attributes with the rest of the cluster. -func (s *HolderSyncer) syncFrame(db, name string) error { - // Retrieve database reference. - f := s.Holder.Frame(db, name) +func (s *HolderSyncer) syncFrame(index, name string) error { + // Retrieve index reference. + f := s.Holder.Frame(index, name) if f == nil { return nil } @@ -488,7 +488,7 @@ func (s *HolderSyncer) syncFrame(db, name string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.RowAttrDiff(context.Background(), db, name, blks) + m, err := client.RowAttrDiff(context.Background(), index, name, blks) if err == ErrFrameNotFound { continue // frame not created remotely yet, skip } else if err != nil { @@ -513,9 +513,9 @@ func (s *HolderSyncer) syncFrame(db, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(db, frame, view string, slice uint64) error { +func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error { // Retrieve local frame. - f := s.Holder.Frame(db, frame) + f := s.Holder.Frame(index, frame) if f == nil { return ErrFrameNotFound } diff --git a/holder_test.go b/holder_test.go index 97a8358e9..7452123b4 100644 --- a/holder_test.go +++ b/holder_test.go @@ -12,36 +12,36 @@ import ( "github.com/pilosa/pilosa/pql" ) -// Ensure holder can delete a database and its underlying files. -func TestHolder_DeleteDB(t *testing.T) { +// Ensure holder can delete an index and its underlying files. +func TestHolder_DeleteIndex(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - // Write bits to separate databases. - f0 := hldr.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0) + // Write bits to separate indexes. + f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } - f1 := hldr.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0) + f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0) if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } - // Ensure d0 exists. - if _, err := os.Stat(hldr.DBPath("d0")); err != nil { + // Ensure i0 exists. + if _, err := os.Stat(hldr.IndexPath("i0")); err != nil { t.Fatal(err) } - // Delete d0. - if err := hldr.DeleteDB("d0"); err != nil { + // Delete i0. + if err := hldr.DeleteIndex("i0"); err != nil { t.Fatal(err) } - // Ensure d0 files are removed & d1 still exists. - if _, err := os.Stat(hldr.DBPath("d0")); !os.IsNotExist(err) { - t.Fatal("expected d0 file deletion") - } else if _, err := os.Stat(hldr.DBPath("d1")); err != nil { - t.Fatal("expected d1 files to still exist", err) + // Ensure i0 files are removed & i1 still exists. + if _, err := os.Stat(hldr.IndexPath("i0")); !os.IsNotExist(err) { + t.Fatal("expected i0 file deletion") + } else if _, err := os.Stat(hldr.IndexPath("i1")); err != nil { + t.Fatal("expected i1 files to still exist", err) } } @@ -59,12 +59,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { s := NewServer() defer s.Close() s.Handler.Holder = hldr1.Holder - s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { + s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor() e.Holder = hldr1.Holder e.Host = cluster.Nodes[1].Host e.Cluster = cluster - return e.Execute(ctx, db, query, slices, opt) + return e.Execute(ctx, index, query, slices, opt) } // Mock 2-node, fully replicated cluster. @@ -74,13 +74,13 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Create frames on nodes. for _, hldr := range []*Holder{hldr0, hldr1} { - hldr.MustCreateFrameIfNotExists("d", "f") - hldr.MustCreateFrameIfNotExists("d", "f0") + hldr.MustCreateFrameIfNotExists("i", "f") + hldr.MustCreateFrameIfNotExists("i", "f0") hldr.MustCreateFrameIfNotExists("y", "z") } // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 20); err != nil { @@ -91,7 +91,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { t.Fatal(err) } - f = hldr0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1) + f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) // Set data on the remote holder. - f = hldr1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) + f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) } else if _, err := f.SetBit(3, 10); err != nil { @@ -118,8 +118,8 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { } // Set highest slice. - hldr0.DB("d").SetRemoteMaxSlice(1) - hldr0.DB("y").SetRemoteMaxSlice(3) + hldr0.Index("i").SetRemoteMaxSlice(1) + hldr0.Index("y").SetRemoteMaxSlice(3) // Set up syncer. syncer := pilosa.HolderSyncer{ @@ -134,7 +134,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*Holder{hldr0, hldr1} { - f := hldr.Fragment("d", "f", pilosa.ViewStandard, 0) + f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { @@ -147,7 +147,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } - f = hldr.Fragment("d", "f0", pilosa.ViewStandard, 1) + f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) a := f.Row(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) @@ -197,18 +197,18 @@ func (h *Holder) Close() error { return h.Holder.Close() } -// MustCreateDBIfNotExists returns a given db. Panic on error. -func (h *Holder) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB { - d, err := h.Holder.CreateDBIfNotExists(db, opt) +// MustCreateIndexIfNotExists returns a given index. Panic on error. +func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { + d, err := h.Holder.CreateIndexIfNotExists(index, opt) if err != nil { panic(err) } - return &DB{DB: d} + return &Index{Index: d} } // MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(db, frame string) *Frame { - f, err := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) +func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } @@ -216,8 +216,8 @@ func (h *Holder) MustCreateFrameIfNotExists(db, frame string) *Frame { } // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment { - d := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) +func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { + d := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) diff --git a/index.go b/index.go new file mode 100644 index 000000000..3fa8ac028 --- /dev/null +++ b/index.go @@ -0,0 +1,565 @@ +package pilosa + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +// Default index settings. +const ( + DefaultColumnLabel = "columnID" +) + +// Index represents a container for frames. +type Index struct { + mu sync.Mutex + path string + name string + + // Default time quantum for all frames in index. + // This can be overridden by individual frames. + timeQuantum TimeQuantum + + // Label used for referring to columns in index. + columnLabel string + + // Frames by name. + frames map[string]*Frame + + // Max Slice on any node in the cluster, according to this node + remoteMaxSlice uint64 + remoteMaxInverseSlice uint64 + + // Column attribute storage and cache + columnAttrStore *AttrStore + + broadcaster Broadcaster + stats StatsClient + + LogOutput io.Writer +} + +// NewIndex returns a new instance of Index. +func NewIndex(path, name string) (*Index, error) { + err := ValidateName(name) + if err != nil { + return nil, err + } + + return &Index{ + path: path, + name: name, + frames: make(map[string]*Frame), + + remoteMaxSlice: 0, + remoteMaxInverseSlice: 0, + + columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), + + columnLabel: DefaultColumnLabel, + + stats: NopStatsClient, + LogOutput: ioutil.Discard, + }, nil +} + +// Name returns name of the index. +func (i *Index) Name() string { return i.name } + +// Path returns the path the index was initialized with. +func (i *Index) Path() string { return i.path } + +// ColumnAttrStore returns the storage for column attributes. +func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore } + +// SetColumnLabel sets the column label. Persists to meta file on update. +func (i *Index) SetColumnLabel(v string) error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ignore if no change occurred. + if v == "" || i.columnLabel == v { + return nil + } + + // Make sure columnLabel is valid name + err := ValidateName(v) + if err != nil { + return err + } + + // Persist meta data to disk on change. + i.columnLabel = v + if err := i.saveMeta(); err != nil { + return err + } + + return nil +} + +// ColumnLabel returns the column label. +func (i *Index) ColumnLabel() string { + i.mu.Lock() + v := i.columnLabel + i.mu.Unlock() + return v +} + +// Open opens and initializes the index. +func (i *Index) Open() error { + // Ensure the path exists. + if err := os.MkdirAll(i.path, 0777); err != nil { + return err + } + + // Read meta file. + if err := i.loadMeta(); err != nil { + return err + } + + if err := i.openFrames(); err != nil { + return err + } + + if err := i.columnAttrStore.Open(); err != nil { + return err + } + + return nil +} + +// openFrames opens and initializes the frames inside the index. +func (i *Index) openFrames() error { + f, err := os.Open(i.path) + if err != nil { + return err + } + defer f.Close() + + fis, err := f.Readdir(0) + if err != nil { + return err + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + + fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if err != nil { + return ErrName + } + if err := fr.Open(); err != nil { + return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) + } + i.frames[fr.Name()] = fr + + i.stats.Count("frameN", 1) + } + return nil +} + +// loadMeta reads meta data for the index, if any. +func (i *Index) loadMeta() error { + var pb internal.IndexMeta + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) + if os.IsNotExist(err) { + i.timeQuantum = "" + i.columnLabel = DefaultColumnLabel + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + i.timeQuantum = TimeQuantum(pb.TimeQuantum) + i.columnLabel = pb.ColumnLabel + + return nil +} + +// saveMeta writes meta data for the index. +func (i *Index) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.IndexMeta{ + TimeQuantum: string(i.timeQuantum), + ColumnLabel: i.columnLabel, + }) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(i.path, ".meta"), buf, 0666); err != nil { + return err + } + + return nil +} + +// Close closes the index and its frames. +func (i *Index) Close() error { + i.mu.Lock() + defer i.mu.Unlock() + + // Close the attribute store. + if i.columnAttrStore != nil { + i.columnAttrStore.Close() + } + + // Close all frames. + for _, f := range i.frames { + f.Close() + } + i.frames = make(map[string]*Frame) + + return nil +} + +// MaxSlice returns the max slice in the index according to this node. +func (i *Index) MaxSlice() uint64 { + if i == nil { + return 0 + } + i.mu.Lock() + defer i.mu.Unlock() + + max := i.remoteMaxSlice + for _, f := range i.frames { + if slice := f.MaxSlice(); slice > max { + max = slice + } + } + return max +} + +func (i *Index) SetRemoteMaxSlice(newmax uint64) { + i.mu.Lock() + defer i.mu.Unlock() + i.remoteMaxSlice = newmax +} + +// MaxInverseSlice returns the max inverse slice in the index according to this node. +func (i *Index) MaxInverseSlice() uint64 { + if i == nil { + return 0 + } + i.mu.Lock() + defer i.mu.Unlock() + + max := i.remoteMaxInverseSlice + for _, f := range i.frames { + if slice := f.MaxInverseSlice(); slice > max { + max = slice + } + } + return max +} + +func (i *Index) SetRemoteMaxInverseSlice(v uint64) { + i.mu.Lock() + defer i.mu.Unlock() + i.remoteMaxInverseSlice = v +} + +// TimeQuantum returns the default time quantum for the index. +func (i *Index) TimeQuantum() TimeQuantum { + i.mu.Lock() + defer i.mu.Unlock() + return i.timeQuantum +} + +// SetTimeQuantum sets the default time quantum for the index. +func (i *Index) SetTimeQuantum(q TimeQuantum) error { + i.mu.Lock() + defer i.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on index. + i.timeQuantum = q + + // Perist meta data to disk. + if err := i.saveMeta(); err != nil { + return err + } + + return nil +} + +// FramePath returns the path to a frame in the index. +func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } + +// Frame returns a frame in the index by name. +func (i *Index) Frame(name string) *Frame { + i.mu.Lock() + defer i.mu.Unlock() + return i.frame(name) +} + +func (i *Index) frame(name string) *Frame { return i.frames[name] } + +// Frames returns a list of all frames in the index. +func (i *Index) Frames() []*Frame { + i.mu.Lock() + defer i.mu.Unlock() + + a := make([]*Frame, 0, len(i.frames)) + for _, f := range i.frames { + a = append(a, f) + } + sort.Sort(frameSlice(a)) + + return a +} + +// CreateFrame creates a frame. +func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Ensure frame doesn't already exist. + if i.frames[name] != nil { + return nil, ErrFrameExists + } + return i.createFrame(name, opt) +} + +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Find frame in cache first. + if f := i.frames[name]; f != nil { + return f, nil + } + + return i.createFrame(name, opt) +} + +func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { + if name == "" { + return nil, errors.New("frame name required") + } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { + return nil, ErrInvalidCacheType + } + + // Initialize frame. + f, err := i.newFrame(i.FramePath(name), name) + if err != nil { + return nil, err + } + + // Open frame. + if err := f.Open(); err != nil { + return nil, err + } + + // Default the time quantum to what is set on the Index. + if err := f.SetTimeQuantum(i.timeQuantum); err != nil { + f.Close() + return nil, err + } + + // Set cache type. + if opt.CacheType == "" { + opt.CacheType = DefaultCacheType + } + f.cacheType = opt.CacheType + + // Set options. + if opt.RowLabel != "" { + f.rowLabel = opt.RowLabel + } + if opt.CacheSize != 0 { + f.cacheSize = opt.CacheSize + } + + f.inverseEnabled = opt.InverseEnabled + if err := f.saveMeta(); err != nil { + f.Close() + return nil, err + } + + // Add to index's frame lookup. + i.frames[name] = f + + i.stats.Count("frameN", 1) + + return f, nil +} + +func (i *Index) newFrame(path, name string) (*Frame, error) { + f, err := NewFrame(path, i.name, name) + if err != nil { + return nil, err + } + f.LogOutput = i.LogOutput + f.stats = i.stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.broadcaster = i.broadcaster + return f, nil +} + +// DeleteFrame removes a frame from the index. +func (i *Index) DeleteFrame(name string) error { + i.mu.Lock() + defer i.mu.Unlock() + + // Ignore if frame doesn't exist. + f := i.frame(name) + if f == nil { + return nil + } + + // Close frame. + if err := f.Close(); err != nil { + return err + } + + // Delete frame directory. + if err := os.RemoveAll(i.FramePath(name)); err != nil { + return err + } + + // Remove reference. + delete(i.frames, name) + + i.stats.Count("frameN", -1) + + return nil +} + +type indexSlice []*Index + +func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p indexSlice) Len() int { return len(p) } +func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } + +// IndexInfo represents schema information for an index. +type IndexInfo struct { + Name string `json:"name"` + Frames []*FrameInfo `json:"frames"` +} + +type indexInfoSlice []*IndexInfo + +func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p indexInfoSlice) Len() int { return len(p) } +func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + +// MergeSchemas combines indexes and frames from a and b into one schema. +func MergeSchemas(a, b []*IndexInfo) []*IndexInfo { + // Generate a map from both schemas. + m := make(map[string]map[string]map[string]struct{}) + for _, idxs := range [][]*IndexInfo{a, b} { + for _, idx := range idxs { + if m[idx.Name] == nil { + m[idx.Name] = make(map[string]map[string]struct{}) + } + for _, frame := range idx.Frames { + if m[idx.Name][frame.Name] == nil { + m[idx.Name][frame.Name] = make(map[string]struct{}) + } + for _, view := range frame.Views { + m[idx.Name][frame.Name][view.Name] = struct{}{} + } + } + } + } + + // Generate new schema from map. + idxs := make([]*IndexInfo, 0, len(m)) + for idx, frames := range m { + di := &IndexInfo{Name: idx} + for frame, views := range frames { + fi := &FrameInfo{Name: frame} + for view := range views { + fi.Views = append(fi.Views, &ViewInfo{Name: view}) + } + sort.Sort(viewInfoSlice(fi.Views)) + di.Frames = append(di.Frames, fi) + } + sort.Sort(frameInfoSlice(di.Frames)) + idxs = append(idxs, di) + } + sort.Sort(indexInfoSlice(idxs)) + + return idxs +} + +// encodeIndexes converts a into its internal representation. +func encodeIndexes(a []*Index) []*internal.Index { + other := make([]*internal.Index, len(a)) + for i := range a { + other[i] = encodeIndex(a[i]) + } + return other +} + +// encodeIndex converts d into its internal representation. +func encodeIndex(d *Index) *internal.Index { + return &internal.Index{ + Name: d.name, + Meta: &internal.IndexMeta{ + ColumnLabel: d.columnLabel, + TimeQuantum: string(d.timeQuantum), + }, + MaxSlice: d.remoteMaxSlice, + Frames: encodeFrames(d.Frames()), + } +} + +// IndexOptions represents options to set when initializing an index. +type IndexOptions struct { + ColumnLabel string `json:"columnLabel,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` +} + +// Encode converts o into its internal representation. +func (o *IndexOptions) Encode() *internal.IndexMeta { + return &internal.IndexMeta{ + ColumnLabel: o.ColumnLabel, + TimeQuantum: string(o.TimeQuantum), + } +} + +// hasTime returns true if a contains a non-nil time. +func hasTime(a []*time.Time) bool { + for _, t := range a { + if t != nil { + return true + } + } + return false +} + +type importKey struct { + View string + Slice uint64 +} + +type importData struct { + RowIDs []uint64 + ColumnIDs []uint64 +} diff --git a/index_test.go b/index_test.go new file mode 100644 index 000000000..67b5b8b2c --- /dev/null +++ b/index_test.go @@ -0,0 +1,179 @@ +package pilosa_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/pilosa/pilosa" +) + +// Ensure index can open and retrieve a frame. +func TestIndex_CreateFrameIfNotExists(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create frame. + f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if f == nil { + t.Fatal("expected frame") + } + + // Retrieve existing frame. + other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if f.Frame != other.Frame { + t.Fatal("frame mismatch") + } + + if f.Frame != index.Frame("f") { + t.Fatal("frame mismatch") + } +} + +// Ensure index defaults the time quantum on new frames. +func TestIndex_CreateFrame_TimeQuantum(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set index time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { + t.Fatal(err) + } + + // Create frame. + f, err := index.CreateFrame("f", pilosa.FrameOptions{}) + if err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { + t.Fatalf("unexpected frame time quantum: %s", q) + } +} + +// Ensure index can delete a frame. +func TestIndex_DeleteFrame(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Create frame. + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Delete frame & verify it's gone. + if err := index.DeleteFrame("f"); err != nil { + t.Fatal(err) + } else if index.Frame("f") != nil { + t.Fatal("expected nil frame") + } + + // Delete again to make sure it doesn't error. + if err := index.DeleteFrame("f"); err != nil { + t.Fatal(err) + } +} + +// Ensure index can set the default time quantum. +func TestIndex_SetTimeQuantum(t *testing.T) { + index := MustOpenIndex() + defer index.Close() + + // Set & retrieve time quantum. + if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload index and verify that it is persisted. + if err := index.Reopen(); err != nil { + t.Fatal(err) + } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + +// Index represents a test wrapper for pilosa.Index. +type Index struct { + *pilosa.Index +} + +// NewIndex returns a new instance of Index. +func NewIndex() *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := pilosa.NewIndex(path, "i") + if err != nil { + panic(err) + } + return &Index{Index: index} +} + +// MustOpenIndex returns a new, opened index at a temporary path. Panic on error. +func MustOpenIndex() *Index { + index := NewIndex() + if err := index.Open(); err != nil { + panic(err) + } + return index +} + +// Close closes the index and removes the underlying data. +func (i *Index) Close() error { + defer os.RemoveAll(i.Path()) + return i.Index.Close() +} + +// Reopen closes the index and reopens it. +func (i *Index) Reopen() error { + var err error + if err := i.Index.Close(); err != nil { + return err + } + + path, name := i.Path(), i.Name() + i.Index, err = pilosa.NewIndex(path, name) + if err != nil { + return err + } + + if err := i.Open(); err != nil { + return err + } + return nil +} + +// CreateFrame creates a frame with the given options. +func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := i.Index.CreateFrame(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil +} + +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { + f, err := i.Index.CreateFrameIfNotExists(name, opt) + if err != nil { + return nil, err + } + return &Frame{Frame: f}, nil +} + +// Ensure index can delete a frame. +func TestIndex_InvalidName(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := pilosa.NewIndex(path, "ABC") + if index != nil { + t.Fatalf("unexpected index name %s", index) + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index f82ba23f8..0e86dd7b6 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -9,7 +9,7 @@ private.proto It has these top-level messages: - DBMeta + IndexMeta FrameMeta ImportResponse BlockDataRequest @@ -17,12 +17,12 @@ Cache MaxSlicesResponse CreateSliceMessage - DeleteDBMessage - CreateDBMessage + DeleteIndexMessage + CreateIndexMessage CreateFrameMessage DeleteFrameMessage Frame - DB + Index NodeState */ package internal @@ -44,15 +44,15 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type DBMeta struct { +type IndexMeta struct { ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } -func (m *DBMeta) Reset() { *m = DBMeta{} } -func (m *DBMeta) String() string { return proto.CompactTextString(m) } -func (*DBMeta) ProtoMessage() {} -func (*DBMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } +func (m *IndexMeta) Reset() { *m = IndexMeta{} } +func (m *IndexMeta) String() string { return proto.CompactTextString(m) } +func (*IndexMeta) ProtoMessage() {} +func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } type FrameMeta struct { RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` @@ -77,7 +77,7 @@ func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} } type BlockDataRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` @@ -125,7 +125,7 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { } type CreateSliceMessage struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` } @@ -134,26 +134,26 @@ func (m *CreateSliceMessage) String() string { return proto.CompactTe func (*CreateSliceMessage) ProtoMessage() {} func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} } -type DeleteDBMessage struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` +type DeleteIndexMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } -func (m *DeleteDBMessage) Reset() { *m = DeleteDBMessage{} } -func (m *DeleteDBMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteDBMessage) ProtoMessage() {} -func (*DeleteDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } +func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } +func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteIndexMessage) ProtoMessage() {} +func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} } -type CreateDBMessage struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` - Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` +type CreateIndexMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` } -func (m *CreateDBMessage) Reset() { *m = CreateDBMessage{} } -func (m *CreateDBMessage) String() string { return proto.CompactTextString(m) } -func (*CreateDBMessage) ProtoMessage() {} -func (*CreateDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } +func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } +func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } +func (*CreateIndexMessage) ProtoMessage() {} +func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} } -func (m *CreateDBMessage) GetMeta() *DBMeta { +func (m *CreateIndexMessage) GetMeta() *IndexMeta { if m != nil { return m.Meta } @@ -161,7 +161,7 @@ func (m *CreateDBMessage) GetMeta() *DBMeta { } type CreateFrameMessage struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } @@ -179,7 +179,7 @@ func (m *CreateFrameMessage) GetMeta() *FrameMeta { } type DeleteFrameMessage struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` } @@ -205,26 +205,26 @@ func (m *Frame) GetMeta() *FrameMeta { return nil } -type DB struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` +type Index struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"` + Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` } -func (m *DB) Reset() { *m = DB{} } -func (m *DB) String() string { return proto.CompactTextString(m) } -func (*DB) ProtoMessage() {} -func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (m *Index) Reset() { *m = Index{} } +func (m *Index) String() string { return proto.CompactTextString(m) } +func (*Index) ProtoMessage() {} +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } -func (m *DB) GetMeta() *DBMeta { +func (m *Index) GetMeta() *IndexMeta { if m != nil { return m.Meta } return nil } -func (m *DB) GetFrames() []*Frame { +func (m *Index) GetFrames() []*Frame { if m != nil { return m.Frames } @@ -232,9 +232,9 @@ func (m *DB) GetFrames() []*Frame { } type NodeState struct { - Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` - State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` - DBs []*DB `protobuf:"bytes,3,rep,name=DBs" json:"DBs,omitempty"` + Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` + State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` + Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` } func (m *NodeState) Reset() { *m = NodeState{} } @@ -242,15 +242,15 @@ func (m *NodeState) String() string { return proto.CompactTextString( func (*NodeState) ProtoMessage() {} func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } -func (m *NodeState) GetDBs() []*DB { +func (m *NodeState) GetIndexes() []*Index { if m != nil { - return m.DBs + return m.Indexes } return nil } func init() { - proto.RegisterType((*DBMeta)(nil), "internal.DBMeta") + proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") @@ -258,15 +258,15 @@ func init() { proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") - proto.RegisterType((*DeleteDBMessage)(nil), "internal.DeleteDBMessage") - proto.RegisterType((*CreateDBMessage)(nil), "internal.CreateDBMessage") + proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") + proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") - proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*NodeState)(nil), "internal.NodeState") } -func (m *DBMeta) Marshal() (dAtA []byte, err error) { +func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -276,7 +276,7 @@ func (m *DBMeta) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) { +func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -386,11 +386,11 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Frame) > 0 { dAtA[i] = 0x12 @@ -553,11 +553,11 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if m.Slice != 0 { dAtA[i] = 0x10 @@ -567,7 +567,7 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) { +func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -577,21 +577,21 @@ func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DeleteDBMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } return i, nil } -func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) { +func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -601,16 +601,16 @@ func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *CreateDBMessage) MarshalTo(dAtA []byte) (int, error) { +func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if m.Meta != nil { dAtA[i] = 0x12 @@ -640,11 +640,11 @@ func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Frame) > 0 { dAtA[i] = 0x12 @@ -680,11 +680,11 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Frame) > 0 { dAtA[i] = 0x12 @@ -729,7 +729,7 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *DB) Marshal() (dAtA []byte, err error) { +func (m *Index) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -739,7 +739,7 @@ func (m *DB) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *DB) MarshalTo(dAtA []byte) (int, error) { +func (m *Index) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -807,8 +807,8 @@ func (m *NodeState) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.State))) i += copy(dAtA[i:], m.State) } - if len(m.DBs) > 0 { - for _, msg := range m.DBs { + if len(m.Indexes) > 0 { + for _, msg := range m.Indexes { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -849,7 +849,7 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *DBMeta) Size() (n int) { +func (m *IndexMeta) Size() (n int) { var l int _ = l l = len(m.ColumnLabel) @@ -900,7 +900,7 @@ func (m *ImportResponse) Size() (n int) { func (m *BlockDataRequest) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -971,7 +971,7 @@ func (m *MaxSlicesResponse) Size() (n int) { func (m *CreateSliceMessage) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -981,20 +981,20 @@ func (m *CreateSliceMessage) Size() (n int) { return n } -func (m *DeleteDBMessage) Size() (n int) { +func (m *DeleteIndexMessage) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } return n } -func (m *CreateDBMessage) Size() (n int) { +func (m *CreateIndexMessage) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -1008,7 +1008,7 @@ func (m *CreateDBMessage) Size() (n int) { func (m *CreateFrameMessage) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -1026,7 +1026,7 @@ func (m *CreateFrameMessage) Size() (n int) { func (m *DeleteFrameMessage) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -1051,7 +1051,7 @@ func (m *Frame) Size() (n int) { return n } -func (m *DB) Size() (n int) { +func (m *Index) Size() (n int) { var l int _ = l l = len(m.Name) @@ -1085,8 +1085,8 @@ func (m *NodeState) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.DBs) > 0 { - for _, e := range m.DBs { + if len(m.Indexes) > 0 { + for _, e := range m.Indexes { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -1107,7 +1107,7 @@ func sovPrivate(x uint64) (n int) { func sozPrivate(x uint64) (n int) { return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *DBMeta) Unmarshal(dAtA []byte) error { +func (m *IndexMeta) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1130,10 +1130,10 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DBMeta: wiretype end group for non-group") + return fmt.Errorf("proto: IndexMeta: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DBMeta: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1501,7 +1501,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -1526,7 +1526,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -2118,7 +2118,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2143,7 +2143,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 0 { @@ -2185,7 +2185,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { +func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2208,15 +2208,15 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DeleteDBMessage: wiretype end group for non-group") + return fmt.Errorf("proto: DeleteIndexMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: DeleteIndexMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2241,7 +2241,7 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -2264,7 +2264,7 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { +func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2287,15 +2287,15 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: CreateDBMessage: wiretype end group for non-group") + return fmt.Errorf("proto: CreateIndexMessage: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: CreateDBMessage: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CreateIndexMessage: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2320,7 +2320,7 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -2349,7 +2349,7 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &DBMeta{} + m.Meta = &IndexMeta{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -2407,7 +2407,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2432,7 +2432,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -2548,7 +2548,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2573,7 +2573,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -2737,7 +2737,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } return nil } -func (m *DB) Unmarshal(dAtA []byte) error { +func (m *Index) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2760,10 +2760,10 @@ func (m *DB) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: DB: wiretype end group for non-group") + return fmt.Errorf("proto: Index: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Index: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -2822,7 +2822,7 @@ func (m *DB) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &DBMeta{} + m.Meta = &IndexMeta{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -2988,7 +2988,7 @@ func (m *NodeState) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DBs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3012,8 +3012,8 @@ func (m *NodeState) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DBs = append(m.DBs, &DB{}) - if err := m.DBs[len(m.DBs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Indexes = append(m.Indexes, &Index{}) + if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3146,43 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 596 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0x13, 0x41, - 0x14, 0x76, 0x7f, 0x68, 0xe8, 0x41, 0x4a, 0x19, 0x8d, 0x59, 0x89, 0x69, 0xea, 0xc4, 0x08, 0xf1, - 0x82, 0x0b, 0xbc, 0x31, 0xc4, 0xab, 0x65, 0x51, 0x9a, 0x00, 0x89, 0x03, 0x7a, 0x3f, 0x94, 0x13, - 0xdd, 0xb0, 0xdd, 0xad, 0xbb, 0x53, 0xa0, 0xde, 0xfa, 0x12, 0x26, 0x3e, 0x83, 0xef, 0xe1, 0xa5, - 0x8f, 0x60, 0xea, 0x8b, 0x98, 0x39, 0x33, 0xfb, 0x63, 0x29, 0x51, 0xef, 0xe6, 0x7c, 0xe7, 0xef, - 0x9b, 0x6f, 0xbf, 0x59, 0x58, 0x1d, 0xe7, 0xf1, 0xa5, 0x54, 0xb8, 0x3d, 0xce, 0x33, 0x95, 0xb1, - 0xe5, 0x38, 0x55, 0x98, 0xa7, 0x32, 0xe1, 0x87, 0xd0, 0x8a, 0xc2, 0x23, 0x54, 0x92, 0xf5, 0x61, - 0x65, 0x2f, 0x4b, 0x26, 0xa3, 0xf4, 0x50, 0x9e, 0x61, 0x12, 0x38, 0x7d, 0x67, 0xab, 0x2d, 0x9a, - 0x90, 0xae, 0x38, 0x8d, 0x47, 0xf8, 0x66, 0x22, 0x53, 0x35, 0x19, 0x05, 0xae, 0xa9, 0x68, 0x40, - 0xfc, 0x9b, 0x03, 0xed, 0x57, 0xb9, 0x1c, 0x21, 0x4d, 0xdc, 0x80, 0x65, 0x91, 0x5d, 0x35, 0xc7, - 0x55, 0x31, 0x7b, 0x0a, 0x9d, 0x41, 0x7a, 0x89, 0x79, 0x81, 0xfb, 0xa9, 0x3c, 0x4b, 0xf0, 0x9c, - 0xc6, 0x2d, 0x8b, 0x39, 0x94, 0x3d, 0x82, 0xf6, 0x9e, 0x1c, 0x7e, 0xc0, 0xd3, 0xe9, 0x18, 0x03, - 0x8f, 0x86, 0xd4, 0x40, 0x95, 0x3d, 0x89, 0x3f, 0x61, 0xe0, 0xf7, 0x9d, 0xad, 0x55, 0x51, 0x03, - 0xf3, 0x7c, 0x97, 0x6e, 0xf2, 0xe5, 0xd0, 0x19, 0x8c, 0xc6, 0x59, 0xae, 0x04, 0x16, 0xe3, 0x2c, - 0x2d, 0x90, 0x75, 0xc1, 0xdb, 0xcf, 0x73, 0x4b, 0x57, 0x1f, 0xf9, 0x35, 0x74, 0xc3, 0x24, 0x1b, - 0x5e, 0x44, 0x52, 0x49, 0x81, 0x1f, 0x27, 0x58, 0x28, 0xd6, 0x01, 0x37, 0x0a, 0x6d, 0x91, 0x1b, - 0x85, 0xec, 0x3e, 0x2c, 0xd1, 0xb5, 0xad, 0x26, 0x26, 0xd0, 0x28, 0x75, 0x12, 0x6f, 0x5f, 0x98, - 0x40, 0xa3, 0x27, 0x49, 0x3c, 0x34, 0x7c, 0x7d, 0x61, 0x02, 0xc6, 0xc0, 0x7f, 0x17, 0xe3, 0x95, - 0x25, 0x49, 0x67, 0x3e, 0x80, 0xf5, 0xc6, 0x66, 0x4b, 0xf0, 0x01, 0xb4, 0x44, 0x76, 0x35, 0x88, - 0x8a, 0xc0, 0xe9, 0x7b, 0x5b, 0xbe, 0xb0, 0x11, 0x49, 0x41, 0xdf, 0x4a, 0xa7, 0x5c, 0x4a, 0xd5, - 0x00, 0x7f, 0x08, 0x4b, 0xa4, 0x8b, 0xbe, 0x5f, 0xdd, 0xab, 0x8f, 0xfc, 0xab, 0x03, 0xeb, 0x47, - 0xf2, 0x9a, 0x68, 0x14, 0xd5, 0x9a, 0x03, 0x68, 0x57, 0x20, 0x55, 0xaf, 0xec, 0x3c, 0xdb, 0x2e, - 0x5d, 0xb3, 0x7d, 0xa3, 0xbe, 0x46, 0xf6, 0x53, 0x95, 0x4f, 0x45, 0xdd, 0xbc, 0xf1, 0x12, 0x3a, - 0x7f, 0x26, 0x35, 0x87, 0x0b, 0x9c, 0x96, 0x1a, 0x5f, 0xe0, 0x54, 0x6b, 0x72, 0x29, 0x93, 0x89, - 0xd1, 0xcf, 0x17, 0x26, 0xd8, 0x75, 0x5f, 0x38, 0x7c, 0x17, 0xd8, 0x5e, 0x8e, 0x52, 0x21, 0x0d, - 0x38, 0xc2, 0xa2, 0x90, 0xef, 0x71, 0x91, 0xfe, 0x46, 0x53, 0xb7, 0xa1, 0x29, 0x7f, 0x0c, 0x6b, - 0x11, 0x26, 0xa8, 0x50, 0x3b, 0x7c, 0x61, 0x23, 0x7f, 0x0d, 0x6b, 0x66, 0xfc, 0xad, 0x25, 0xec, - 0x09, 0xf8, 0xda, 0xcd, 0x34, 0x7a, 0x65, 0xa7, 0x5b, 0x8b, 0x60, 0xde, 0x8d, 0xa0, 0x2c, 0x1f, - 0x96, 0x3c, 0xad, 0xfd, 0x6f, 0xe5, 0xb9, 0xc0, 0x27, 0x9b, 0x76, 0x83, 0x47, 0x1b, 0xee, 0xd5, - 0x1b, 0xaa, 0xa7, 0x64, 0x97, 0xec, 0x02, 0x33, 0x17, 0xfa, 0xff, 0x25, 0x3c, 0xb2, 0xa8, 0x76, - 0xda, 0xb1, 0xce, 0x9a, 0x06, 0x3a, 0x57, 0x0c, 0xdc, 0xbf, 0x31, 0xf8, 0xec, 0xe8, 0x65, 0x0b, - 0x67, 0xfc, 0x93, 0x4e, 0xfa, 0x9f, 0x50, 0xba, 0xc1, 0x3e, 0x8b, 0x2a, 0x66, 0x9b, 0xd0, 0xa2, - 0x7d, 0x45, 0xe0, 0x93, 0xe1, 0xd6, 0xe6, 0x78, 0x08, 0x9b, 0xe6, 0x6f, 0xa1, 0x7d, 0x9c, 0x9d, - 0xe3, 0x89, 0x92, 0x8a, 0xee, 0x73, 0x90, 0x15, 0xaa, 0xe4, 0xa2, 0xcf, 0xe4, 0x07, 0x9d, 0x2c, - 0x25, 0x30, 0x95, 0x3d, 0xf0, 0xa2, 0xb0, 0x08, 0x3c, 0x1a, 0x7e, 0xb7, 0x49, 0x50, 0xe8, 0x44, - 0xd8, 0xfd, 0x3e, 0xeb, 0x39, 0x3f, 0x66, 0x3d, 0xe7, 0xe7, 0xac, 0xe7, 0x7c, 0xf9, 0xd5, 0xbb, - 0x73, 0xd6, 0xa2, 0xdf, 0xe5, 0xf3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcf, 0x20, 0xf1, - 0x3f, 0x05, 0x00, 0x00, + // 594 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x4c, + 0x10, 0xfe, 0x9d, 0xb8, 0xfd, 0xe3, 0x89, 0x1a, 0xd2, 0x05, 0x21, 0x53, 0xa1, 0x28, 0xda, 0x03, + 0x0d, 0x3d, 0xe4, 0x50, 0x2e, 0x08, 0x71, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, + 0x66, 0x93, 0x8c, 0xc0, 0x8a, 0x63, 0x07, 0x7b, 0x93, 0x34, 0x1c, 0xb8, 0xf3, 0x06, 0x48, 0x3c, + 0x03, 0xef, 0xc1, 0x91, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x8e, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc, + 0x76, 0xbe, 0x99, 0x9d, 0xef, 0x9b, 0xcf, 0xb3, 0x86, 0xbd, 0x79, 0xe4, 0x2d, 0xa5, 0xc2, 0xf6, + 0x3c, 0x0a, 0x55, 0xc8, 0x2a, 0x5e, 0xa0, 0x30, 0x0a, 0xa4, 0xcf, 0x5f, 0x80, 0xd3, 0x0f, 0x26, + 0x78, 0x39, 0x40, 0x25, 0x59, 0x13, 0xaa, 0x9d, 0xd0, 0x5f, 0xcc, 0x82, 0xe7, 0x72, 0x84, 0xbe, + 0x6b, 0x35, 0xad, 0x96, 0x23, 0xb6, 0x21, 0x5d, 0x71, 0xe1, 0xcd, 0xf0, 0xe5, 0x42, 0x06, 0x6a, + 0x31, 0x73, 0x4b, 0x49, 0xc5, 0x16, 0xc4, 0xbf, 0x58, 0xe0, 0x3c, 0x8b, 0xe4, 0x0c, 0xa9, 0xe3, + 0x01, 0x54, 0x44, 0xb8, 0xda, 0x6e, 0x97, 0xc5, 0xec, 0x01, 0xd4, 0xfa, 0xc1, 0x12, 0xa3, 0x18, + 0x7b, 0x81, 0x1c, 0xf9, 0x38, 0xa1, 0x76, 0x15, 0x51, 0x40, 0xd9, 0x7d, 0x70, 0x3a, 0x72, 0xfc, + 0x16, 0x2f, 0xd6, 0x73, 0x74, 0xcb, 0xd4, 0x24, 0x07, 0xb2, 0xec, 0xd0, 0x7b, 0x8f, 0xae, 0xdd, + 0xb4, 0x5a, 0x7b, 0x22, 0x07, 0x8a, 0x7a, 0x77, 0xae, 0xea, 0xe5, 0x50, 0xeb, 0xcf, 0xe6, 0x61, + 0xa4, 0x04, 0xc6, 0xf3, 0x30, 0x88, 0x91, 0xd5, 0xa1, 0xdc, 0x8b, 0x22, 0x23, 0x57, 0x1f, 0xf9, + 0x07, 0xa8, 0x9f, 0xfa, 0xe1, 0x78, 0xda, 0x95, 0x4a, 0x0a, 0x7c, 0xb7, 0xc0, 0x58, 0xb1, 0x3b, + 0xb0, 0x43, 0xc6, 0x99, 0xba, 0x24, 0xd0, 0x28, 0x0d, 0x6f, 0x9c, 0x49, 0x02, 0x8d, 0xd2, 0x7d, + 0x52, 0x6f, 0x8b, 0x24, 0xd0, 0xe8, 0xd0, 0xf7, 0xc6, 0x89, 0x6a, 0x5b, 0x24, 0x01, 0x63, 0x60, + 0xbf, 0xf2, 0x70, 0x65, 0xa4, 0xd2, 0x99, 0xf7, 0x61, 0x7f, 0x8b, 0xdf, 0xc8, 0xbc, 0x0b, 0xbb, + 0x22, 0x5c, 0xf5, 0xbb, 0xb1, 0x6b, 0x35, 0xcb, 0x2d, 0x5b, 0x98, 0x88, 0x0c, 0xa1, 0x2f, 0xa6, + 0x53, 0x25, 0x4a, 0xe5, 0x00, 0xbf, 0x07, 0x3b, 0xe4, 0x8e, 0x9e, 0x32, 0xbf, 0xab, 0x8f, 0xfc, + 0xb3, 0x05, 0xfb, 0x03, 0x79, 0x49, 0x32, 0xe2, 0x8c, 0xe6, 0x0c, 0x9c, 0x0c, 0xa4, 0xea, 0xea, + 0xf1, 0x51, 0x3b, 0x5d, 0x9f, 0xf6, 0x95, 0xfa, 0x1c, 0xe9, 0x05, 0x2a, 0x5a, 0x8b, 0xfc, 0xf2, + 0xc1, 0x53, 0xa8, 0xfd, 0x9a, 0xd4, 0x1a, 0xa6, 0xb8, 0x4e, 0x9d, 0x9e, 0xe2, 0x5a, 0x7b, 0xb2, + 0x94, 0xfe, 0x22, 0xf1, 0xcf, 0x16, 0x49, 0xf0, 0xa4, 0xf4, 0xd8, 0xe2, 0x27, 0xc0, 0x3a, 0x11, + 0x4a, 0x85, 0xd4, 0x60, 0x80, 0x71, 0x2c, 0xdf, 0xe0, 0xf5, 0x5f, 0x21, 0x71, 0xb6, 0xb4, 0xe5, + 0x2c, 0x3f, 0x02, 0xd6, 0x45, 0x1f, 0x15, 0x9a, 0x85, 0xff, 0x43, 0x07, 0x3e, 0x4c, 0xd9, 0x6e, + 0xae, 0x65, 0x87, 0x60, 0xeb, 0x5d, 0x27, 0xb2, 0xea, 0xf1, 0xed, 0xdc, 0x9c, 0xec, 0x61, 0x09, + 0x2a, 0xe0, 0x5e, 0xda, 0xd4, 0xbc, 0x8f, 0x1b, 0x46, 0xf8, 0xcd, 0x22, 0xa5, 0x54, 0xe5, 0x22, + 0x55, 0xf6, 0xe2, 0x0c, 0xd5, 0x49, 0x3a, 0xeb, 0xbf, 0x52, 0xf1, 0xae, 0x41, 0xf5, 0x42, 0x9e, + 0xeb, 0x6c, 0x72, 0x87, 0xce, 0xd7, 0x8f, 0x5c, 0xd4, 0xf1, 0xd1, 0x32, 0x94, 0x7f, 0xd7, 0xa6, + 0xe0, 0x9c, 0xfe, 0x8d, 0xa4, 0xab, 0x63, 0xde, 0x50, 0x16, 0xb3, 0x43, 0xd8, 0x25, 0xd6, 0xd8, + 0xb5, 0x69, 0x3b, 0x6f, 0x15, 0xd4, 0x08, 0x93, 0xe6, 0xaf, 0xc1, 0x39, 0x0f, 0x27, 0x38, 0x54, + 0x52, 0xd1, 0x54, 0x67, 0x61, 0xac, 0x52, 0x39, 0xfa, 0x4c, 0x6b, 0xa3, 0x93, 0xa9, 0x11, 0x49, + 0xe5, 0x43, 0xf8, 0x9f, 0xe4, 0x60, 0xec, 0x96, 0x8b, 0x04, 0x94, 0x10, 0x69, 0xfe, 0xb4, 0xfe, + 0x75, 0xd3, 0xb0, 0xbe, 0x6d, 0x1a, 0xd6, 0xf7, 0x4d, 0xc3, 0xfa, 0xf4, 0xa3, 0xf1, 0xdf, 0x68, + 0x97, 0xfe, 0xb7, 0x8f, 0x7e, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x68, 0xdc, 0x63, 0x80, 0x05, + 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 6294b1091..b791efb6c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package internal; -message DBMeta { +message IndexMeta { string ColumnLabel = 1; string TimeQuantum = 2; } @@ -20,7 +20,7 @@ message ImportResponse { } message BlockDataRequest { - string DB = 1; + string Index = 1; string Frame = 2; string View = 5; uint64 Slice = 4; @@ -41,27 +41,27 @@ message MaxSlicesResponse { } message CreateSliceMessage { - string DB = 1; + string Index = 1; uint64 Slice = 2; } -message DeleteDBMessage { - string DB = 1; +message DeleteIndexMessage { + string Index = 1; } -message CreateDBMessage { - string DB = 1; - DBMeta Meta = 2; +message CreateIndexMessage { + string Index = 1; + IndexMeta Meta = 2; } message CreateFrameMessage { - string DB = 1; + string Index = 1; string Frame = 2; FrameMeta Meta = 3; } message DeleteFrameMessage { - string DB = 1; + string Index = 1; string Frame = 2; } @@ -70,9 +70,9 @@ message Frame { FrameMeta Meta = 2; } -message DB { +message Index { string Name = 1; - DBMeta Meta = 2; + IndexMeta Meta = 2; uint64 MaxSlice = 3; repeated Frame Frames = 4; } @@ -80,5 +80,5 @@ message DB { message NodeState { string Host = 1; string State = 2; - repeated DB DBs = 3; + repeated Index Indexes = 3; } diff --git a/internal/public.pb.go b/internal/public.pb.go index 24fb7eb85..9eb005c17 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -189,7 +189,7 @@ func (m *QueryResult) GetPairs() []*Pair { } type ImportRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` @@ -627,11 +627,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.DB) > 0 { + if len(m.Index) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.DB))) - i += copy(dAtA[i:], m.DB) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) } if len(m.Frame) > 0 { dAtA[i] = 0x12 @@ -899,7 +899,7 @@ func (m *QueryResult) Size() (n int) { func (m *ImportRequest) Size() (n int) { var l int _ = l - l = len(m.DB) + l = len(m.Index) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } @@ -2185,7 +2185,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2210,7 +2210,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.DB = string(dAtA[iNdEx:postIndex]) + m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { @@ -2575,42 +2575,41 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 579 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0x66, 0x6d, 0x27, 0x4d, 0x26, 0x6d, 0x14, 0xad, 0xf8, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x83, - 0x4f, 0xa9, 0x54, 0x1e, 0x00, 0xe1, 0x24, 0x95, 0x22, 0x44, 0x45, 0x27, 0x85, 0xbb, 0x5b, 0x56, - 0xc5, 0x92, 0xff, 0x58, 0xaf, 0x85, 0xf2, 0x00, 0xdc, 0x91, 0xb8, 0x70, 0xe5, 0xc6, 0xa3, 0x70, - 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0xbb, 0xde, 0xd8, 0xe5, 0x80, 0xb8, 0xed, 0xf7, 0xcd, 0xce, - 0x7a, 0xbe, 0xf9, 0x66, 0x0c, 0xc7, 0x55, 0x73, 0x9d, 0xa5, 0x37, 0x8b, 0x4a, 0x96, 0xaa, 0xe4, - 0xa3, 0xb4, 0x50, 0x42, 0x16, 0x49, 0x16, 0xc6, 0x30, 0x8c, 0x53, 0x95, 0x27, 0x15, 0xe7, 0xe0, - 0xc5, 0xa9, 0xaa, 0x7d, 0x16, 0xb8, 0x91, 0x87, 0xfa, 0xcc, 0x9f, 0xc2, 0xe0, 0x85, 0x52, 0xb2, - 0xf6, 0x9d, 0xc0, 0x8d, 0x26, 0x67, 0xd3, 0x85, 0xcd, 0x5b, 0x10, 0x8d, 0x26, 0x18, 0x2e, 0xc0, - 0x7b, 0x9d, 0xa4, 0x92, 0xcf, 0xc0, 0x7d, 0x29, 0x76, 0x3e, 0x0b, 0x58, 0xe4, 0x21, 0x1d, 0xf9, - 0x7d, 0x18, 0x2c, 0xcb, 0xa6, 0x50, 0xbe, 0xa3, 0x39, 0x03, 0xc2, 0x37, 0xe0, 0xc6, 0xa9, 0xa2, - 0x20, 0x96, 0x1f, 0x37, 0xab, 0x36, 0xc1, 0x00, 0xfe, 0x18, 0x46, 0xcb, 0x32, 0x6b, 0xf2, 0x62, - 0xb3, 0x6a, 0xb3, 0x0e, 0x98, 0x3f, 0x81, 0xf1, 0x55, 0x9a, 0x8b, 0x5a, 0x25, 0x79, 0xe5, 0xbb, - 0x01, 0x8b, 0x5c, 0xec, 0x88, 0x70, 0x0d, 0x27, 0xe6, 0x26, 0x55, 0xb5, 0x15, 0x8a, 0x4f, 0xc1, - 0x39, 0xbc, 0xee, 0x6c, 0x56, 0xff, 0xa9, 0xe6, 0x3b, 0x03, 0x8f, 0x4e, 0x7d, 0x39, 0x63, 0x23, - 0x87, 0x83, 0x77, 0xb5, 0xab, 0x44, 0x5b, 0x97, 0x3e, 0xf3, 0x00, 0x26, 0x5b, 0x25, 0xd3, 0xe2, - 0xf6, 0x6d, 0x92, 0x35, 0x42, 0x57, 0x35, 0xc6, 0x3e, 0x45, 0x8a, 0x36, 0x85, 0x32, 0x61, 0x4f, - 0x17, 0x7d, 0xc0, 0xa4, 0x28, 0x2e, 0xcb, 0xcc, 0x04, 0x07, 0x01, 0x8b, 0x46, 0xd8, 0x11, 0x7c, - 0x0e, 0x70, 0x9e, 0x95, 0x49, 0x9b, 0x3b, 0x0c, 0x58, 0xc4, 0xb0, 0xc7, 0x84, 0xa7, 0x70, 0x44, - 0x95, 0xbe, 0x4a, 0xaa, 0x4e, 0x1b, 0xfb, 0x97, 0xb6, 0xcf, 0x0c, 0x8e, 0x2f, 0x1b, 0x21, 0x77, - 0x28, 0x3e, 0x34, 0xa2, 0xd6, 0x1e, 0x68, 0xdc, 0xaa, 0x34, 0x80, 0x3f, 0x84, 0xe1, 0x36, 0x4b, - 0x6f, 0x84, 0xe9, 0x94, 0x87, 0x2d, 0x22, 0xad, 0x5d, 0x87, 0x6b, 0xad, 0x75, 0x84, 0x7d, 0x8a, - 0xfb, 0x70, 0x74, 0xd9, 0x24, 0x85, 0x6a, 0x72, 0x2d, 0x75, 0x8c, 0x16, 0xd2, 0x9b, 0x28, 0xf2, - 0x52, 0x59, 0x99, 0x2d, 0x0a, 0xbf, 0x30, 0x38, 0x69, 0x4b, 0xaa, 0xab, 0xb2, 0xa8, 0x05, 0xf5, - 0x7d, 0x2d, 0xa5, 0xed, 0xfb, 0x5a, 0x4a, 0x7e, 0x0a, 0x47, 0x28, 0xea, 0x26, 0x53, 0xd6, 0xba, - 0x07, 0x9d, 0x3c, 0x9b, 0xdb, 0x64, 0x0a, 0xed, 0x2d, 0xfe, 0x1c, 0xa6, 0x77, 0x46, 0x81, 0x6a, - 0xa5, 0xbc, 0x47, 0x5d, 0xde, 0x9d, 0x38, 0xfe, 0x75, 0x3d, 0xfc, 0xc4, 0x60, 0xd2, 0x7b, 0x99, - 0x47, 0x76, 0x4d, 0x74, 0x59, 0x93, 0xb3, 0x59, 0xf7, 0x90, 0xe1, 0xd1, 0xae, 0xd1, 0x31, 0xb0, - 0x8b, 0x76, 0x40, 0xd8, 0x05, 0xd9, 0x42, 0xab, 0x61, 0xbf, 0xdf, 0xb3, 0x85, 0x68, 0x34, 0x41, - 0xea, 0xda, 0xf2, 0x7d, 0x52, 0xdc, 0x8a, 0x77, 0xba, 0x6b, 0x23, 0xb4, 0x30, 0xfc, 0xc6, 0xe0, - 0x64, 0x93, 0x57, 0xa5, 0x54, 0xd6, 0xb1, 0x29, 0x38, 0xab, 0xb8, 0x6d, 0x8e, 0xb3, 0x8a, 0xc9, - 0xc1, 0x73, 0x99, 0xe4, 0x66, 0x28, 0xc7, 0x68, 0x00, 0xb1, 0xda, 0x33, 0xed, 0x91, 0x87, 0x06, - 0x68, 0x0f, 0x68, 0xc9, 0x6a, 0xdf, 0x33, 0xbe, 0x1a, 0x44, 0x53, 0x68, 0x77, 0xac, 0xf6, 0x07, - 0x3a, 0xd4, 0x11, 0x34, 0x85, 0x87, 0x25, 0xab, 0xfd, 0x61, 0xe0, 0x46, 0x2e, 0xf6, 0x98, 0x78, - 0xf6, 0x63, 0x3f, 0x67, 0x3f, 0xf7, 0x73, 0xf6, 0x6b, 0x3f, 0x67, 0x5f, 0x7f, 0xcf, 0xef, 0x5d, - 0x0f, 0xf5, 0x5f, 0xe6, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x79, 0x70, 0xf4, 0x75, - 0x04, 0x00, 0x00, + // 576 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, + 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, + 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4, + 0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b, + 0x8e, 0x80, 0xc2, 0x45, 0x50, 0x75, 0xbb, 0x63, 0x0f, 0x0b, 0xc4, 0xae, 0xdf, 0xab, 0xae, 0x76, + 0xbd, 0x7a, 0x55, 0x86, 0x69, 0x55, 0x5f, 0x65, 0xe9, 0xf5, 0xaa, 0x12, 0xa5, 0x2a, 0xe9, 0x28, + 0x2d, 0x14, 0x17, 0x45, 0x92, 0x05, 0x11, 0x0c, 0xa2, 0x54, 0xe5, 0x49, 0x45, 0x29, 0xb8, 0x51, + 0xaa, 0xa4, 0x47, 0x7c, 0x27, 0x74, 0x99, 0x3e, 0xd3, 0xa7, 0xd0, 0x7f, 0xa1, 0x94, 0x90, 0x5e, + 0xcf, 0x77, 0xc2, 0xc9, 0xe9, 0x7c, 0x65, 0xf3, 0x56, 0x48, 0x33, 0x13, 0x0c, 0x56, 0xe0, 0xbe, + 0x4e, 0x52, 0x41, 0x17, 0xe0, 0xbc, 0xe4, 0x7b, 0x8f, 0xf8, 0x24, 0x74, 0x19, 0x1e, 0xe9, 0x7d, + 0xe8, 0xaf, 0xcb, 0xba, 0x50, 0x5e, 0x4f, 0x73, 0x06, 0x04, 0x6f, 0xc0, 0x89, 0x52, 0x85, 0x41, + 0x56, 0x7e, 0x88, 0x37, 0x4d, 0x82, 0x01, 0xf4, 0x31, 0x8c, 0xd6, 0x65, 0x56, 0xe7, 0x45, 0xbc, + 0x69, 0xb2, 0x8e, 0x98, 0x3e, 0x81, 0xf1, 0x65, 0x9a, 0x73, 0xa9, 0x92, 0xbc, 0xf2, 0x1c, 0x9f, + 0x84, 0x0e, 0x6b, 0x89, 0x60, 0x0b, 0x33, 0x73, 0x13, 0xab, 0xda, 0x71, 0x45, 0xe7, 0xd0, 0x3b, + 0xbe, 0xde, 0x8b, 0x37, 0xff, 0xa9, 0xe6, 0x3b, 0x01, 0x17, 0x4f, 0x5d, 0x39, 0x63, 0x23, 0x87, + 0x82, 0x7b, 0xb9, 0xaf, 0x78, 0x53, 0x97, 0x3e, 0x53, 0x1f, 0x26, 0x3b, 0x25, 0xd2, 0xe2, 0xf6, + 0x6d, 0x92, 0xd5, 0x5c, 0x57, 0x35, 0x66, 0x5d, 0x0a, 0x15, 0xc5, 0x85, 0x32, 0x61, 0x57, 0x17, + 0x7d, 0xc4, 0xa8, 0x28, 0x2a, 0xcb, 0xcc, 0x04, 0xfb, 0x3e, 0x09, 0x47, 0xac, 0x25, 0xe8, 0x12, + 0xe0, 0x2c, 0x2b, 0x93, 0x26, 0x77, 0xe0, 0x93, 0x90, 0xb0, 0x0e, 0x13, 0x9c, 0xc0, 0x10, 0x2b, + 0x7d, 0x95, 0x54, 0xad, 0x36, 0xf2, 0x2f, 0x6d, 0x9f, 0x09, 0x4c, 0x2f, 0x6a, 0x2e, 0xf6, 0x8c, + 0xbf, 0xaf, 0xb9, 0xd4, 0x1e, 0x68, 0xdc, 0xa8, 0x34, 0x80, 0x3e, 0x84, 0xc1, 0x2e, 0x4b, 0xaf, + 0xb9, 0xe9, 0x94, 0xcb, 0x1a, 0x84, 0x5a, 0xdb, 0x0e, 0x4b, 0xad, 0x75, 0xc4, 0xba, 0x14, 0xf5, + 0x60, 0x78, 0x51, 0x27, 0x85, 0xaa, 0x73, 0x2d, 0x75, 0xcc, 0x2c, 0xc4, 0x37, 0x19, 0xcf, 0x4b, + 0x65, 0x65, 0x36, 0x28, 0xf8, 0x42, 0x60, 0xd6, 0x94, 0x24, 0xab, 0xb2, 0x90, 0x1c, 0xfb, 0xbe, + 0x15, 0xc2, 0xf6, 0x7d, 0x2b, 0x04, 0x3d, 0x81, 0x21, 0xe3, 0xb2, 0xce, 0x94, 0xb5, 0xee, 0x41, + 0x2b, 0xcf, 0xe6, 0xd6, 0x99, 0x62, 0xf6, 0x16, 0x7d, 0x0e, 0xf3, 0x3b, 0xa3, 0x80, 0xb5, 0x62, + 0xde, 0xa3, 0x36, 0xef, 0x4e, 0x9c, 0xfd, 0x75, 0x3d, 0xf8, 0x44, 0x60, 0xd2, 0x79, 0x99, 0x86, + 0x76, 0x4d, 0x74, 0x59, 0x93, 0xd3, 0x45, 0xfb, 0x90, 0xe1, 0x99, 0x5d, 0xa3, 0x29, 0x90, 0xf3, + 0x66, 0x40, 0xc8, 0x39, 0xda, 0x82, 0xab, 0x61, 0xbf, 0xdf, 0xb1, 0x05, 0x69, 0x66, 0x82, 0xd8, + 0xb5, 0xf5, 0xbb, 0xa4, 0xb8, 0xe5, 0x37, 0xba, 0x6b, 0x23, 0x66, 0x61, 0xf0, 0x8d, 0xc0, 0x2c, + 0xce, 0xab, 0x52, 0xa8, 0x8e, 0x63, 0x71, 0x71, 0xc3, 0x3f, 0x5a, 0xc7, 0x34, 0x40, 0xf6, 0x4c, + 0x24, 0xb9, 0x19, 0xcd, 0x31, 0x33, 0x00, 0x59, 0xed, 0x9c, 0x76, 0xca, 0x65, 0x06, 0x68, 0x27, + 0x70, 0xd5, 0xa4, 0xe7, 0x1a, 0x77, 0x0d, 0xc2, 0x59, 0xb4, 0x9b, 0x26, 0xbd, 0xbe, 0x0e, 0xb5, + 0x04, 0xce, 0xe2, 0x71, 0xd5, 0xa4, 0x37, 0xf0, 0x9d, 0xd0, 0x61, 0x1d, 0x26, 0x5a, 0xfc, 0x38, + 0x2c, 0xc9, 0xcf, 0xc3, 0x92, 0xfc, 0x3a, 0x2c, 0xc9, 0xd7, 0xdf, 0xcb, 0x7b, 0x57, 0x03, 0xfd, + 0xaf, 0x79, 0xf6, 0x27, 0x00, 0x00, 0xff, 0xff, 0x48, 0x20, 0x4b, 0xe7, 0x7b, 0x04, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 91c1f31e8..571fe0364 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -58,7 +58,7 @@ message QueryResult { } message ImportRequest { - string DB = 1; + string Index = 1; string Frame = 2; uint64 Slice = 3; repeated uint64 RowIDs = 4; diff --git a/pilosa.go b/pilosa.go index 8ffb54155..16739fd7a 100644 --- a/pilosa.go +++ b/pilosa.go @@ -11,9 +11,9 @@ import ( var ( ErrHostRequired = errors.New("host required") - ErrDatabaseRequired = errors.New("database required") - ErrDatabaseExists = errors.New("database already exists") - ErrDatabaseNotFound = errors.New("database not found") + ErrIndexRequired = errors.New("index required") + ErrIndexExists = errors.New("index already exists") + ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. ErrFrameRequired = errors.New("frame required") @@ -24,18 +24,18 @@ var ( ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") ) -// Regular expression to valuate db and frame's name +// Regular expression to valuate index and frame's name // Todo: remove . when frame doesn't require . for topN var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`) -// ColumnAttrSet represents a set of attributes for a vertical column in a database. +// ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` diff --git a/server.go b/server.go index ba11f1fae..0f178898c 100644 --- a/server.go +++ b/server.go @@ -219,16 +219,16 @@ func (s *Server) monitorMaxSlices() { for _, node := range s.Cluster.Nodes { if s.Host != node.Host { maxSlices, _ := checkMaxSlices(node.Host) - for db, newmax := range maxSlices { - // if we don't know about a db locally, log an error because - // db's should be created and synced prior to slice creation - if localdb := s.Holder.DB(db); localdb != nil { - if newmax > oldmaxslices[db] { - oldmaxslices[db] = newmax - localdb.SetRemoteMaxSlice(newmax) + for index, newmax := range maxSlices { + // if we don't know about an index locally, log an error because + // indexes should be created and synced prior to slice creation + if localIndex := s.Holder.Index(index); localIndex != nil { + if newmax > oldmaxslices[index] { + oldmaxslices[index] = newmax + localIndex.SetRemoteMaxSlice(newmax) } } else { - s.logger().Printf("Local DB not found: %s", db) + s.logger().Printf("Local Index not found: %s", index) } } } @@ -240,31 +240,31 @@ func (s *Server) monitorMaxSlices() { func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { case *internal.CreateSliceMessage: - d := s.Holder.DB(obj.DB) + d := s.Holder.Index(obj.Index) if d == nil { - return fmt.Errorf("Local DB not found: %s", obj.DB) + return fmt.Errorf("Local Index not found: %s", obj.Index) } d.SetRemoteMaxSlice(obj.Slice) - case *internal.CreateDBMessage: - opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel} - _, err := s.Holder.CreateDB(obj.DB, opt) + case *internal.CreateIndexMessage: + opt := IndexOptions{ColumnLabel: obj.Meta.ColumnLabel} + _, err := s.Holder.CreateIndex(obj.Index, opt) if err != nil { return err } - case *internal.DeleteDBMessage: - if err := s.Holder.DeleteDB(obj.DB); err != nil { + case *internal.DeleteIndexMessage: + if err := s.Holder.DeleteIndex(obj.Index); err != nil { return err } case *internal.CreateFrameMessage: - db := s.Holder.DB(obj.DB) + index := s.Holder.Index(obj.Index) opt := FrameOptions{RowLabel: obj.Meta.RowLabel} - _, err := db.CreateFrame(obj.Frame, opt) + _, err := index.CreateFrame(obj.Frame, opt) if err != nil { return err } case *internal.DeleteFrameMessage: - db := s.Holder.DB(obj.DB) - if err := db.DeleteFrame(obj.Frame); err != nil { + index := s.Holder.Index(obj.Index) + if err := index.DeleteFrame(obj.Frame); err != nil { return err } } @@ -273,16 +273,16 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { // Server implements gossip.StateHandler. // LocalState returns the state of the local node as well as the -// holder (dbs/frames) according to the local node. +// holder (indexes/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalState() (proto.Message, error) { if s.Holder == nil { return nil, errors.New("Server.Holder is nil.") } return &internal.NodeState{ - Host: s.Host, - State: "OK", // TODO: make this work, pull from s.Cluster.Node - DBs: encodeDBs(s.Holder.DBs()), + Host: s.Host, + State: "OK", // TODO: make this work, pull from s.Cluster.Node + Indexes: encodeIndexes(s.Holder.Indexes()), }, nil } @@ -294,18 +294,18 @@ func (s *Server) HandleRemoteState(pb proto.Message) error { func (s *Server) mergeRemoteState(ns *internal.NodeState) error { // TODO: update some node state value in the cluster (it should be in cluster.node i guess) - // Create databases that don't exist. - for _, db := range ns.DBs { - opt := DBOptions{ - ColumnLabel: db.Meta.ColumnLabel, - TimeQuantum: TimeQuantum(db.Meta.TimeQuantum), + // Create indexes that don't exist. + for _, index := range ns.Indexes { + opt := IndexOptions{ + ColumnLabel: index.Meta.ColumnLabel, + TimeQuantum: TimeQuantum(index.Meta.TimeQuantum), } - d, err := s.Holder.CreateDBIfNotExists(db.Name, opt) + d, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) if err != nil { return err } // Create frames that don't exist. - for _, f := range db.Frames { + for _, f := range index.Frames { opt := FrameOptions{ RowLabel: f.Meta.RowLabel, TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), diff --git a/server/server_test.go b/server/server_test.go index 0f8a2f5d9..4e3fa5e36 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -39,13 +39,13 @@ func TestMain_Set_Quick(t *testing.T) { // Execute SetBit() commands. for _, cmd := range cmds { - if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { + if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } @@ -61,7 +61,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -84,7 +84,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -109,47 +109,47 @@ func TestMain_SetRowAttrs(t *testing.T) { // Create frames. client := m.Client() - if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "neg", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set bits on different rows in different frames. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("d", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x.n/1. - if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x.n/2. - if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -160,19 +160,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x.n/2. - if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -186,26 +186,26 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Create frames. client := m.Client() - if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set bits on row. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("d", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { t.Fatal(err) } // Query row. - if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -216,7 +216,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -230,26 +230,26 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { // Create frames. client := m.Client() - if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrDatabaseExists { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Set bits on row. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil { t.Fatal(err) } // Set column attributes. - if _, err := m.Query("d", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { + if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } // Query row. - if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -274,14 +274,14 @@ func TestMain_FrameRestore(t *testing.T) { // Create frames. client := m0.Client() - if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } // Write data on first cluster. - if _, err := m0.Query("d", "", ` + if _, err := m0.Query("x", "", ` SetBit(id=1, frame="f", columnID=100) SetBit(id=1, frame="f", columnID=1000) SetBit(id=1, frame="f", columnID=100000) @@ -294,7 +294,7 @@ func TestMain_FrameRestore(t *testing.T) { } // Query row on first cluster. - if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { + if res, err := m0.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -308,16 +308,16 @@ func TestMain_FrameRestore(t *testing.T) { client, err := pilosa.NewClient(m2.Server.Host) if err != nil { t.Fatal(err) - } else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + } else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil { + } else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil { + } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil { t.Fatal(err) } // Query row on second cluster. - if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { + if res, err := m2.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -431,8 +431,9 @@ func (m *Main) Client() *pilosa.Client { } // Query executes a query against the program through the HTTP API. -func (m *Main) Query(db, rawQuery, query string) (string, error) { - resp := MustDo("POST", m.URL()+fmt.Sprintf("/db/%s/query?", db)+rawQuery, query) +func (m *Main) Query(index, rawQuery, query string) (string, error) { + fmt.Println("Query:", index, query) + resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) } diff --git a/view.go b/view.go index 4fa096943..9f67b516d 100644 --- a/view.go +++ b/view.go @@ -26,7 +26,7 @@ func IsValidView(name string) bool { type View struct { mu sync.Mutex path string - db string + index string frame string name string @@ -43,10 +43,10 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, db, frame, name string, cacheSize uint32) *View { +func NewView(path, index, frame, name string, cacheSize uint32) *View { return &View{ path: path, - db: db, + index: index, frame: frame, name: name, cacheSize: cacheSize, @@ -62,8 +62,8 @@ func NewView(path, db, frame, name string, cacheSize uint32) *View { // Name returns the name the view was initialized with. func (v *View) Name() string { return v.name } -// DB returns the database name the view was initialized with. -func (v *View) DB() string { return v.db } +// Index returns the index name the view was initialized with. +func (v *View) Index() string { return v.index } // Frame returns the frame name the view was initialized with. func (v *View) Frame() string { return v.frame } @@ -216,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.db, v.frame, v.name, slice) + frag := NewFragment(path, v.index, v.frame, v.name, slice) frag.cacheType = v.cacheType frag.cacheSize = v.cacheSize frag.LogOutput = v.LogOutput diff --git a/view_test.go b/view_test.go index c531e1fce..adaa0ced2 100644 --- a/view_test.go +++ b/view_test.go @@ -14,7 +14,7 @@ type View struct { } // NewView returns a new instance of View with a temporary path. -func NewView(db, frame, name string) *View { +func NewView(index, frame, name string) *View { file, err := ioutil.TempFile("", "pilosa-view-") if err != nil { panic(err) @@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), + View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize), RowAttrStore: MustOpenAttrStore(), } v.View.RowAttrStore = v.RowAttrStore.AttrStore @@ -30,8 +30,8 @@ func NewView(db, frame, name string) *View { } // MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(db, frame, name string) *View { - v := NewView(db, frame, name) +func MustOpenView(index, frame, name string) *View { + v := NewView(index, frame, name) if err := v.Open(); err != nil { panic(err) } @@ -52,7 +52,7 @@ func (v *View) Reopen() error { return err } - v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) + v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) v.View.RowAttrStore = v.RowAttrStore.AttrStore if err := v.Open(); err != nil { return err From 13439793081b9240fede49501713e0a36e34d137 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 24 Apr 2017 11:17:30 -0500 Subject: [PATCH 58/63] Fixes up a few overlooked `d := Index()` --- client_test.go | 4 ++-- executor.go | 40 ++++++++++++++++++++-------------------- handler_test.go | 6 +++--- holder.go | 16 ++++++++-------- holder_test.go | 10 +++++----- server.go | 10 +++++----- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/client_test.go b/client_test.go index d2111144d..70676e399 100644 --- a/client_test.go +++ b/client_test.go @@ -195,11 +195,11 @@ func TestClient_ImportInverseEnabled(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() - d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) frameOpts := pilosa.FrameOptions{ InverseEnabled: true, } - frame, err := d.CreateFrameIfNotExists("f", frameOpts) + frame, err := idx.CreateFrameIfNotExists("f", frameOpts) if err != nil { panic(err) } diff --git a/executor.go b/executor.go index 8c7fef54a..d1fd8b829 100644 --- a/executor.go +++ b/executor.go @@ -160,11 +160,11 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { - d := e.Holder.Index(index) - if d != nil { - columnLabel := d.ColumnLabel() + idx := e.Holder.Index(index) + if idx != nil { + columnLabel := idx.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { - attrs, err := d.ColumnAttrStore().Attrs(columnID) + attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err } @@ -173,7 +173,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return nil, err } else { frame, _ := c.Args["frame"].(string) - if fr := d.Frame(frame); fr != nil { + if fr := idx.Frame(frame); fr != nil { rowLabel := fr.RowLabel() rowID, _, err := c.UintArg(rowLabel) if err != nil { @@ -362,11 +362,11 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c * func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { // Fetch column label from index. - d := e.Holder.Index(index) - if d == nil { + idx := e.Holder.Index(index) + if idx == nil { return nil, ErrIndexNotFound } - columnLabel := d.ColumnLabel() + columnLabel := idx.ColumnLabel() // Fetch frame & row label based on argument. frame, _ := c.Args["frame"].(string) @@ -548,17 +548,17 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // Retrieve frame. - d := e.Holder.Index(index) - if d == nil { + idx := e.Holder.Index(index) + if idx == nil { return false, ErrIndexNotFound } - f := d.Frame(frame) + f := idx.Frame(frame) if f == nil { return false, ErrFrameNotFound } // Retrieve labels. - columnLabel := d.ColumnLabel() + columnLabel := idx.ColumnLabel() rowLabel := f.RowLabel() // Read fields using labels. @@ -642,17 +642,17 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // Retrieve frame. - d := e.Holder.Index(index) - if d == nil { + idx := e.Holder.Index(index) + if idx == nil { return false, ErrIndexNotFound } - f := d.Frame(frame) + f := idx.Frame(frame) if f == nil { return false, ErrFrameNotFound } // Retrieve labels. - columnLabel := d.ColumnLabel() + columnLabel := idx.ColumnLabel() rowLabel := f.RowLabel() // Read fields using labels. @@ -886,8 +886,8 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // executeSetColumnAttrs executes a SetColumnAttrs() call. func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { // Retrieve index. - d := e.Holder.Index(index) - if d == nil { + idx := e.Holder.Index(index) + if idx == nil { return ErrIndexNotFound } @@ -895,7 +895,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p id, okID, errID := c.UintArg("id") if errID != nil || !okID { // Retrieve columnLabel - columnLabel := d.columnLabel + columnLabel := idx.columnLabel col, okCol, errCol := c.UintArg(columnLabel) if errCol != nil || !okCol { return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) @@ -911,7 +911,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p delete(attrs, colName) // Set attributes. - if err := d.ColumnAttrStore().SetAttrs(id, attrs); err != nil { + if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil { return err } diff --git a/handler_test.go b/handler_test.go index fbf7041e8..d0b467f90 100644 --- a/handler_test.go +++ b/handler_test.go @@ -648,8 +648,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer s.Close() // Set attributes on the index. - d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := d.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -701,7 +701,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) - // Begin backing up from slice d/f/0. + // Begin backing up from slice i/f/0. resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") if err != nil { t.Fatal(err) diff --git a/holder.go b/holder.go index 054cffe57..426817410 100644 --- a/holder.go +++ b/holder.go @@ -281,11 +281,11 @@ func (h *Holder) DeleteIndex(name string) error { // Frame returns the frame for an index and name. func (h *Holder) Frame(index, name string) *Frame { - d := h.Index(index) - if d == nil { + idx := h.Index(index) + if idx == nil { return nil } - return d.Frame(name) + return idx.Frame(name) } // View returns the view for an index, frame, and name. @@ -423,13 +423,13 @@ func (s *HolderSyncer) SyncHolder() error { // syncIndex synchronizes index attributes with the rest of the cluster. func (s *HolderSyncer) syncIndex(index string) error { // Retrieve index reference. - d := s.Holder.Index(index) - if d == nil { + idx := s.Holder.Index(index) + if idx == nil { return nil } // Read block checksums. - blks, err := d.ColumnAttrStore().Blocks() + blks, err := idx.ColumnAttrStore().Blocks() if err != nil { return err } @@ -451,12 +451,12 @@ func (s *HolderSyncer) syncIndex(index string) error { } // Update local copy. - if err := d.ColumnAttrStore().SetBulkAttrs(m); err != nil { + if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil { return err } // Recompute blocks. - blks, err = d.ColumnAttrStore().Blocks() + blks, err = idx.ColumnAttrStore().Blocks() if err != nil { return err } diff --git a/holder_test.go b/holder_test.go index 7452123b4..78c4700f8 100644 --- a/holder_test.go +++ b/holder_test.go @@ -150,7 +150,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) a := f.Row(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) } if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) @@ -199,11 +199,11 @@ func (h *Holder) Close() error { // MustCreateIndexIfNotExists returns a given index. Panic on error. func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index { - d, err := h.Holder.CreateIndexIfNotExists(index, opt) + idx, err := h.Holder.CreateIndexIfNotExists(index, opt) if err != nil { panic(err) } - return &Index{Index: d} + return &Index{Index: idx} } // MustCreateFrameIfNotExists returns a given frame. Panic on error. @@ -217,8 +217,8 @@ func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { - d := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } diff --git a/server.go b/server.go index 0f178898c..6aef3dd62 100644 --- a/server.go +++ b/server.go @@ -240,11 +240,11 @@ func (s *Server) monitorMaxSlices() { func (s *Server) ReceiveMessage(pb proto.Message) error { switch obj := pb.(type) { case *internal.CreateSliceMessage: - d := s.Holder.Index(obj.Index) - if d == nil { + idx := s.Holder.Index(obj.Index) + if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - d.SetRemoteMaxSlice(obj.Slice) + idx.SetRemoteMaxSlice(obj.Slice) case *internal.CreateIndexMessage: opt := IndexOptions{ColumnLabel: obj.Meta.ColumnLabel} _, err := s.Holder.CreateIndex(obj.Index, opt) @@ -300,7 +300,7 @@ func (s *Server) mergeRemoteState(ns *internal.NodeState) error { ColumnLabel: index.Meta.ColumnLabel, TimeQuantum: TimeQuantum(index.Meta.TimeQuantum), } - d, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) + idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt) if err != nil { return err } @@ -311,7 +311,7 @@ func (s *Server) mergeRemoteState(ns *internal.NodeState) error { TimeQuantum: TimeQuantum(f.Meta.TimeQuantum), CacheSize: f.Meta.CacheSize, } - _, err := d.CreateFrameIfNotExists(f.Name, opt) + _, err := idx.CreateFrameIfNotExists(f.Name, opt) if err != nil { return err } From 74bf4731f5ca4060076a83b44fdb22b57f3ce7f1 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 21 Apr 2017 18:27:46 -0500 Subject: [PATCH 59/63] WIP: Implement NodeState as an attribute of *Node NodeState is shared among nodes in the cluster (via gossip in a gossip implementation) and cached locally in Cluster.Nodes in order to be available to the /status endpoint. --- cluster.go | 47 ++++++-- cluster_test.go | 12 +- gossip/gossip.go | 22 ++-- handler.go | 15 ++- internal/private.pb.go | 248 ++++++++++++++++++++++++++++++++--------- internal/private.proto | 6 +- server.go | 51 +++++++-- 7 files changed, 305 insertions(+), 96 deletions(-) diff --git a/cluster.go b/cluster.go index b9d799aec..f9f22464f 100644 --- a/cluster.go +++ b/cluster.go @@ -3,6 +3,8 @@ package pilosa import ( "encoding/binary" "hash/fnv" + + "github.com/pilosa/pilosa/internal" ) const ( @@ -12,15 +14,30 @@ const ( // DefaultReplicaN is the default number of replicas per partition. DefaultReplicaN = 1 - // HealthStatus is the return value of the /health endpoint for a node in the cluster. - HealthStatusUp = "UP" - HealthStatusDown = "DOWN" + // NodeState represents node state returned in /status endpoint for a node in the cluster. + NodeStateUp = "UP" + NodeStateDown = "DOWN" ) // Node represents a node in the cluster. type Node struct { Host string `json:"host"` InternalHost string `json:"internalHost"` + + status *internal.NodeStatus `json:"state"` +} + +// SetStatus sets the NodeStatus. +func (n *Node) SetStatus(s *internal.NodeStatus) { + n.status = s +} + +// SetState sets the Node.status.state. +func (n *Node) SetState(s string) { + if n.status == nil { + n.status = &internal.NodeStatus{} + } + n.status.State = s } // Nodes represents a list of nodes. @@ -120,21 +137,37 @@ func (c *Cluster) NodeSetHosts() []string { return a } -// Health returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. -func (c *Cluster) Health() map[string]string { +// NodeStates returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value. +func (c *Cluster) NodeStates() map[string]string { h := make(map[string]string) for _, n := range c.Nodes { - h[n.Host] = HealthStatusDown + h[n.Host] = NodeStateDown } // we are assuming that NodeSetHosts is a subset of c.Nodes for _, m := range c.NodeSetHosts() { if _, ok := h[m]; ok { - h[m] = HealthStatusUp + h[m] = NodeStateUp } } return h } +// State returns the internal ClusterState representation. +func (c *Cluster) Status() *internal.ClusterStatus { + return &internal.ClusterStatus{ + Nodes: encodeClusterStatus(c.Nodes), + } +} + +// encodeClusterStatus converts a into its internal representation. +func encodeClusterStatus(a []*Node) []*internal.NodeStatus { + other := make([]*internal.NodeStatus, len(a)) + for i := range a { + other[i] = a[i].status + } + return other +} + // NodeByHost returns a node reference by host. func (c *Cluster) NodeByHost(host string) *Node { for _, n := range c.Nodes { diff --git a/cluster_test.go b/cluster_test.go index e561223fb..4c96cd5d9 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -89,7 +89,7 @@ func TestCluster_NodeSetHosts(t *testing.T) { } // Ensure cluster can compare its Nodes and Members -func TestCluster_Health(t *testing.T) { +func TestCluster_NodeStates(t *testing.T) { c := pilosa.Cluster{ Nodes: []*pilosa.Node{ {Host: "serverA:1000"}, @@ -109,12 +109,12 @@ func TestCluster_Health(t *testing.T) { } // Verify a DOWN node is reported, and extraneous nodes are ignored - if a := c.Health(); !reflect.DeepEqual(a, map[string]string{ - "serverA:1000": pilosa.HealthStatusUp, - "serverB:1000": pilosa.HealthStatusDown, - "serverC:1000": pilosa.HealthStatusUp, + if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{ + "serverA:1000": pilosa.NodeStateUp, + "serverB:1000": pilosa.NodeStateDown, + "serverC:1000": pilosa.NodeStateUp, }) { - t.Fatalf("unexpected health: %s", spew.Sdump(a)) + t.Fatalf("unexpected node state: %s", spew.Sdump(a)) } } diff --git a/gossip/gossip.go b/gossip/gossip.go index 772892926..c3ded776a 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -14,14 +14,6 @@ import ( "github.com/pilosa/pilosa/internal" ) -// StateHandler specifies two methods which an object must implement to share -// state in the cluster. These are used by the GossipNodeSet to implement the -// LocalState and MergeRemoteState methods of memberlist.Delegate -type StateHandler interface { - LocalState() (proto.Message, error) - HandleRemoteState(proto.Message) error -} - // GossipNodeSet represents a gossip implementation of NodeSet using memberlist // GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster // GossipNodeSet also represents an implementation of memberlist.Delegate @@ -31,8 +23,8 @@ type GossipNodeSet struct { broadcasts *memberlist.TransmitLimitedQueue - stateHandler StateHandler - config *GossipConfig + statusHandler pilosa.StatusHandler + config *GossipConfig // The writer for any logging. LogOutput io.Writer @@ -89,7 +81,7 @@ type GossipConfig struct { } // NewGossipNodeSet returns a new instance of GossipNodeSet. -func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh StateHandler) *GossipNodeSet { +func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler) *GossipNodeSet { g := &GossipNodeSet{ LogOutput: os.Stderr, } @@ -106,7 +98,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed g.config.memberlistConfig.AdvertisePort = gossipPort g.config.memberlistConfig.Delegate = g - g.stateHandler = sh + g.statusHandler = sh return g } @@ -176,7 +168,7 @@ func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte { } func (g *GossipNodeSet) LocalState(join bool) []byte { - pb, err := g.stateHandler.LocalState() + pb, err := g.statusHandler.LocalStatus() if err != nil { g.logger().Printf("error getting local state, err=%s", err) return []byte{} @@ -193,12 +185,12 @@ func (g *GossipNodeSet) LocalState(join bool) []byte { func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) { // Unmarshal nodestate data. - var pb internal.NodeState + var pb internal.NodeStatus if err := proto.Unmarshal(buf, &pb); err != nil { g.logger().Printf("error unmarshalling nodestate data, err=%s", err) return } - err := g.stateHandler.HandleRemoteState(&pb) + err := g.statusHandler.HandleRemoteStatus(&pb) if err != nil { g.logger().Printf("merge state error: %s", err) } diff --git a/handler.go b/handler.go index 904db8ce5..77dcb5700 100644 --- a/handler.go +++ b/handler.go @@ -27,8 +27,9 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Holder *Holder - Broadcaster Broadcaster + Holder *Holder + Broadcaster Broadcaster + StatusHandler StatusHandler // Local hostname & cluster configuration. Host string @@ -85,6 +86,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/nodes", handler.handleGetNodes).Methods("GET") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") // TODO: Apply MethodNotAllowed statuses to all endpoints. @@ -116,8 +118,13 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + status, err := h.StatusHandler.ClusterStatus() + if err != nil { + h.logger().Printf("cluster status error: %s", err) + return + } if err := json.NewEncoder(w).Encode(getStatusResponse{ - Health: h.Cluster.Health(), + Status: status, }); err != nil { h.logger().Printf("write status response error: %s", err) } @@ -128,7 +135,7 @@ type getSchemaResponse struct { } type getStatusResponse struct { - Health map[string]string `json:"health"` + Status proto.Message `json:"status"` } // handlePostQuery handles /query requests. diff --git a/internal/private.pb.go b/internal/private.pb.go index 0e86dd7b6..a41df4788 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -23,7 +23,8 @@ DeleteFrameMessage Frame Index - NodeState + NodeStatus + ClusterStatus */ package internal @@ -231,24 +232,40 @@ func (m *Index) GetFrames() []*Frame { return nil } -type NodeState struct { +type NodeStatus struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"` } -func (m *NodeState) Reset() { *m = NodeState{} } -func (m *NodeState) String() string { return proto.CompactTextString(m) } -func (*NodeState) ProtoMessage() {} -func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (m *NodeStatus) Reset() { *m = NodeStatus{} } +func (m *NodeStatus) String() string { return proto.CompactTextString(m) } +func (*NodeStatus) ProtoMessage() {} +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } -func (m *NodeState) GetIndexes() []*Index { +func (m *NodeStatus) GetIndexes() []*Index { if m != nil { return m.Indexes } return nil } +type ClusterStatus struct { + Nodes []*NodeStatus `protobuf:"bytes,1,rep,name=Nodes" json:"Nodes,omitempty"` +} + +func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } +func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } +func (*ClusterStatus) ProtoMessage() {} +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } + +func (m *ClusterStatus) GetNodes() []*NodeStatus { + if m != nil { + return m.Nodes + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -264,7 +281,8 @@ func init() { proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Index)(nil), "internal.Index") - proto.RegisterType((*NodeState)(nil), "internal.NodeState") + proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") + proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -780,7 +798,7 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *NodeState) Marshal() (dAtA []byte, err error) { +func (m *NodeStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -790,7 +808,7 @@ func (m *NodeState) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *NodeState) MarshalTo(dAtA []byte) (int, error) { +func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -822,6 +840,36 @@ func (m *NodeState) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ClusterStatus) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Nodes) > 0 { + for _, msg := range m.Nodes { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -1074,7 +1122,7 @@ func (m *Index) Size() (n int) { return n } -func (m *NodeState) Size() (n int) { +func (m *NodeStatus) Size() (n int) { var l int _ = l l = len(m.Host) @@ -1094,6 +1142,18 @@ func (m *NodeState) Size() (n int) { return n } +func (m *ClusterStatus) Size() (n int) { + var l int + _ = l + if len(m.Nodes) > 0 { + for _, e := range m.Nodes { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -2899,7 +2959,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { } return nil } -func (m *NodeState) Unmarshal(dAtA []byte) error { +func (m *NodeStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2922,10 +2982,10 @@ func (m *NodeState) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: NodeState: wiretype end group for non-group") + return fmt.Errorf("proto: NodeStatus: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: NodeState: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: NodeStatus: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3038,6 +3098,87 @@ func (m *NodeState) Unmarshal(dAtA []byte) error { } return nil } +func (m *ClusterStatus) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ClusterStatus: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ClusterStatus: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nodes = append(m.Nodes, &NodeStatus{}) + if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -3146,43 +3287,44 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 594 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x4c, - 0x10, 0xfe, 0x9d, 0xb8, 0xfd, 0xe3, 0x89, 0x1a, 0xd2, 0x05, 0x21, 0x53, 0xa1, 0x28, 0xda, 0x03, - 0x0d, 0x3d, 0xe4, 0x50, 0x2e, 0x08, 0x71, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, - 0x66, 0x93, 0x8c, 0xc0, 0x8a, 0x63, 0x07, 0x7b, 0x93, 0x34, 0x1c, 0xb8, 0xf3, 0x06, 0x48, 0x3c, - 0x03, 0xef, 0xc1, 0x91, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x8e, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc, - 0x76, 0xbe, 0x99, 0x9d, 0xef, 0x9b, 0xcf, 0xb3, 0x86, 0xbd, 0x79, 0xe4, 0x2d, 0xa5, 0xc2, 0xf6, - 0x3c, 0x0a, 0x55, 0xc8, 0x2a, 0x5e, 0xa0, 0x30, 0x0a, 0xa4, 0xcf, 0x5f, 0x80, 0xd3, 0x0f, 0x26, - 0x78, 0x39, 0x40, 0x25, 0x59, 0x13, 0xaa, 0x9d, 0xd0, 0x5f, 0xcc, 0x82, 0xe7, 0x72, 0x84, 0xbe, - 0x6b, 0x35, 0xad, 0x96, 0x23, 0xb6, 0x21, 0x5d, 0x71, 0xe1, 0xcd, 0xf0, 0xe5, 0x42, 0x06, 0x6a, - 0x31, 0x73, 0x4b, 0x49, 0xc5, 0x16, 0xc4, 0xbf, 0x58, 0xe0, 0x3c, 0x8b, 0xe4, 0x0c, 0xa9, 0xe3, - 0x01, 0x54, 0x44, 0xb8, 0xda, 0x6e, 0x97, 0xc5, 0xec, 0x01, 0xd4, 0xfa, 0xc1, 0x12, 0xa3, 0x18, - 0x7b, 0x81, 0x1c, 0xf9, 0x38, 0xa1, 0x76, 0x15, 0x51, 0x40, 0xd9, 0x7d, 0x70, 0x3a, 0x72, 0xfc, - 0x16, 0x2f, 0xd6, 0x73, 0x74, 0xcb, 0xd4, 0x24, 0x07, 0xb2, 0xec, 0xd0, 0x7b, 0x8f, 0xae, 0xdd, - 0xb4, 0x5a, 0x7b, 0x22, 0x07, 0x8a, 0x7a, 0x77, 0xae, 0xea, 0xe5, 0x50, 0xeb, 0xcf, 0xe6, 0x61, - 0xa4, 0x04, 0xc6, 0xf3, 0x30, 0x88, 0x91, 0xd5, 0xa1, 0xdc, 0x8b, 0x22, 0x23, 0x57, 0x1f, 0xf9, - 0x07, 0xa8, 0x9f, 0xfa, 0xe1, 0x78, 0xda, 0x95, 0x4a, 0x0a, 0x7c, 0xb7, 0xc0, 0x58, 0xb1, 0x3b, - 0xb0, 0x43, 0xc6, 0x99, 0xba, 0x24, 0xd0, 0x28, 0x0d, 0x6f, 0x9c, 0x49, 0x02, 0x8d, 0xd2, 0x7d, - 0x52, 0x6f, 0x8b, 0x24, 0xd0, 0xe8, 0xd0, 0xf7, 0xc6, 0x89, 0x6a, 0x5b, 0x24, 0x01, 0x63, 0x60, - 0xbf, 0xf2, 0x70, 0x65, 0xa4, 0xd2, 0x99, 0xf7, 0x61, 0x7f, 0x8b, 0xdf, 0xc8, 0xbc, 0x0b, 0xbb, - 0x22, 0x5c, 0xf5, 0xbb, 0xb1, 0x6b, 0x35, 0xcb, 0x2d, 0x5b, 0x98, 0x88, 0x0c, 0xa1, 0x2f, 0xa6, - 0x53, 0x25, 0x4a, 0xe5, 0x00, 0xbf, 0x07, 0x3b, 0xe4, 0x8e, 0x9e, 0x32, 0xbf, 0xab, 0x8f, 0xfc, - 0xb3, 0x05, 0xfb, 0x03, 0x79, 0x49, 0x32, 0xe2, 0x8c, 0xe6, 0x0c, 0x9c, 0x0c, 0xa4, 0xea, 0xea, - 0xf1, 0x51, 0x3b, 0x5d, 0x9f, 0xf6, 0x95, 0xfa, 0x1c, 0xe9, 0x05, 0x2a, 0x5a, 0x8b, 0xfc, 0xf2, - 0xc1, 0x53, 0xa8, 0xfd, 0x9a, 0xd4, 0x1a, 0xa6, 0xb8, 0x4e, 0x9d, 0x9e, 0xe2, 0x5a, 0x7b, 0xb2, - 0x94, 0xfe, 0x22, 0xf1, 0xcf, 0x16, 0x49, 0xf0, 0xa4, 0xf4, 0xd8, 0xe2, 0x27, 0xc0, 0x3a, 0x11, - 0x4a, 0x85, 0xd4, 0x60, 0x80, 0x71, 0x2c, 0xdf, 0xe0, 0xf5, 0x5f, 0x21, 0x71, 0xb6, 0xb4, 0xe5, - 0x2c, 0x3f, 0x02, 0xd6, 0x45, 0x1f, 0x15, 0x9a, 0x85, 0xff, 0x43, 0x07, 0x3e, 0x4c, 0xd9, 0x6e, - 0xae, 0x65, 0x87, 0x60, 0xeb, 0x5d, 0x27, 0xb2, 0xea, 0xf1, 0xed, 0xdc, 0x9c, 0xec, 0x61, 0x09, - 0x2a, 0xe0, 0x5e, 0xda, 0xd4, 0xbc, 0x8f, 0x1b, 0x46, 0xf8, 0xcd, 0x22, 0xa5, 0x54, 0xe5, 0x22, - 0x55, 0xf6, 0xe2, 0x0c, 0xd5, 0x49, 0x3a, 0xeb, 0xbf, 0x52, 0xf1, 0xae, 0x41, 0xf5, 0x42, 0x9e, - 0xeb, 0x6c, 0x72, 0x87, 0xce, 0xd7, 0x8f, 0x5c, 0xd4, 0xf1, 0xd1, 0x32, 0x94, 0x7f, 0xd7, 0xa6, - 0xe0, 0x9c, 0xfe, 0x8d, 0xa4, 0xab, 0x63, 0xde, 0x50, 0x16, 0xb3, 0x43, 0xd8, 0x25, 0xd6, 0xd8, - 0xb5, 0x69, 0x3b, 0x6f, 0x15, 0xd4, 0x08, 0x93, 0xe6, 0xaf, 0xc1, 0x39, 0x0f, 0x27, 0x38, 0x54, - 0x52, 0xd1, 0x54, 0x67, 0x61, 0xac, 0x52, 0x39, 0xfa, 0x4c, 0x6b, 0xa3, 0x93, 0xa9, 0x11, 0x49, - 0xe5, 0x43, 0xf8, 0x9f, 0xe4, 0x60, 0xec, 0x96, 0x8b, 0x04, 0x94, 0x10, 0x69, 0xfe, 0xb4, 0xfe, - 0x75, 0xd3, 0xb0, 0xbe, 0x6d, 0x1a, 0xd6, 0xf7, 0x4d, 0xc3, 0xfa, 0xf4, 0xa3, 0xf1, 0xdf, 0x68, - 0x97, 0xfe, 0xb7, 0x8f, 0x7e, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x68, 0xdc, 0x63, 0x80, 0x05, - 0x00, 0x00, + // 617 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0xc5, 0x89, 0x53, 0x9a, 0xa9, 0x5a, 0xda, 0xa5, 0x42, 0xa6, 0x42, 0x51, 0xb4, 0x07, 0x5a, + 0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, 0xdf, + 0x24, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0xdc, 0xf9, 0x03, 0x24, 0xbe, 0x81, + 0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0xc7, 0x6b, 0x3b, 0xb8, 0x94, 0x0a, 0x6e, 0x3b, + 0x6f, 0x66, 0xe7, 0xbd, 0x7d, 0x9e, 0x31, 0x6c, 0x4e, 0x63, 0x7f, 0x26, 0x35, 0x1e, 0x4d, 0xe3, + 0x48, 0x47, 0x6c, 0xdd, 0x0f, 0x35, 0xc6, 0xa1, 0x0c, 0xf8, 0x2b, 0xa8, 0x77, 0xc3, 0x11, 0x5e, + 0xf6, 0x50, 0x4b, 0xd6, 0x84, 0x8d, 0x56, 0x14, 0x24, 0x93, 0xf0, 0xa5, 0x1c, 0x60, 0xe0, 0x39, + 0x4d, 0xe7, 0xa0, 0x2e, 0x56, 0x21, 0x53, 0x71, 0xe1, 0x4f, 0xf0, 0x75, 0x22, 0x43, 0x9d, 0x4c, + 0xbc, 0x4a, 0x5a, 0xb1, 0x02, 0xf1, 0xaf, 0x0e, 0xd4, 0x5f, 0xc4, 0x72, 0x82, 0xd4, 0x71, 0x0f, + 0xd6, 0x45, 0x34, 0x5f, 0x6d, 0x97, 0xc7, 0xec, 0x21, 0x6c, 0x75, 0xc3, 0x19, 0xc6, 0x0a, 0x3b, + 0xa1, 0x1c, 0x04, 0x38, 0xa2, 0x76, 0xeb, 0xa2, 0x84, 0xb2, 0x07, 0x50, 0x6f, 0xc9, 0xe1, 0x3b, + 0xbc, 0x58, 0x4c, 0xd1, 0xab, 0x52, 0x93, 0x02, 0xc8, 0xb3, 0x7d, 0xff, 0x03, 0x7a, 0x6e, 0xd3, + 0x39, 0xd8, 0x14, 0x05, 0x50, 0xd6, 0x5b, 0xbb, 0xaa, 0x97, 0xc3, 0x56, 0x77, 0x32, 0x8d, 0x62, + 0x2d, 0x50, 0x4d, 0xa3, 0x50, 0x21, 0xdb, 0x86, 0x6a, 0x27, 0x8e, 0xad, 0x5c, 0x73, 0xe4, 0x1f, + 0x61, 0xfb, 0x34, 0x88, 0x86, 0xe3, 0xb6, 0xd4, 0x52, 0xe0, 0xfb, 0x04, 0x95, 0x66, 0xbb, 0x50, + 0x23, 0xe3, 0x6c, 0x5d, 0x1a, 0x18, 0x94, 0x1e, 0x6f, 0x9d, 0x49, 0x03, 0x83, 0xd2, 0x7d, 0x52, + 0xef, 0x8a, 0x34, 0x30, 0x68, 0x3f, 0xf0, 0x87, 0xa9, 0x6a, 0x57, 0xa4, 0x01, 0x63, 0xe0, 0xbe, + 0xf1, 0x71, 0x6e, 0xa5, 0xd2, 0x99, 0x77, 0x61, 0x67, 0x85, 0xdf, 0xca, 0xbc, 0x07, 0x6b, 0x22, + 0x9a, 0x77, 0xdb, 0xca, 0x73, 0x9a, 0xd5, 0x03, 0x57, 0xd8, 0x88, 0x0c, 0xa1, 0x2f, 0x66, 0x52, + 0x15, 0x4a, 0x15, 0x00, 0xbf, 0x0f, 0x35, 0x72, 0xc7, 0xbc, 0xb2, 0xb8, 0x6b, 0x8e, 0xfc, 0x8b, + 0x03, 0x3b, 0x3d, 0x79, 0x49, 0x32, 0x54, 0x4e, 0x73, 0x06, 0xf5, 0x1c, 0xa4, 0xea, 0x8d, 0xe3, + 0xc3, 0xa3, 0x6c, 0x7c, 0x8e, 0xae, 0xd4, 0x17, 0x48, 0x27, 0xd4, 0xf1, 0x42, 0x14, 0x97, 0xf7, + 0x9e, 0xc3, 0xd6, 0xef, 0x49, 0xa3, 0x61, 0x8c, 0x8b, 0xcc, 0xe9, 0x31, 0x2e, 0x8c, 0x27, 0x33, + 0x19, 0x24, 0xa9, 0x7f, 0xae, 0x48, 0x83, 0xa7, 0x95, 0x27, 0x0e, 0x3f, 0x01, 0xd6, 0x8a, 0x51, + 0x6a, 0xa4, 0x06, 0x3d, 0x54, 0x4a, 0xbe, 0xc5, 0xeb, 0xbf, 0x42, 0xea, 0x6c, 0x65, 0xc5, 0x59, + 0x7e, 0x08, 0xac, 0x8d, 0x01, 0x6a, 0xb4, 0x03, 0xff, 0x97, 0x0e, 0xbc, 0x9f, 0xb1, 0xdd, 0x5c, + 0xcb, 0xf6, 0xc1, 0x35, 0xb3, 0x4e, 0x64, 0x1b, 0xc7, 0x77, 0x0b, 0x73, 0xf2, 0xc5, 0x12, 0x54, + 0xc0, 0xfd, 0xac, 0xa9, 0xdd, 0x8f, 0x1b, 0x9e, 0xf0, 0x87, 0x41, 0xca, 0xa8, 0xaa, 0x65, 0xaa, + 0x7c, 0xe3, 0x2c, 0xd5, 0x49, 0xf6, 0xd6, 0xff, 0xa5, 0xe2, 0x6d, 0x8b, 0x9a, 0x81, 0x3c, 0x37, + 0xd9, 0xf4, 0x0e, 0x9d, 0xaf, 0x7f, 0x72, 0x59, 0xc7, 0x27, 0xc7, 0x52, 0xfe, 0x5b, 0x9b, 0x92, + 0x73, 0xe6, 0x37, 0x92, 0x8d, 0x8e, 0xdd, 0xa1, 0x3c, 0x66, 0xfb, 0xb0, 0x46, 0xac, 0xca, 0x73, + 0x69, 0x3a, 0xef, 0x94, 0xd4, 0x08, 0x9b, 0xe6, 0x12, 0xe0, 0x3c, 0x1a, 0x61, 0x5f, 0x4b, 0x9d, + 0x28, 0xa3, 0xe7, 0x2c, 0x52, 0x3a, 0xd3, 0x63, 0xce, 0x34, 0x37, 0x5a, 0xea, 0xdc, 0x09, 0x0a, + 0xd8, 0x23, 0xb8, 0x4d, 0x7a, 0x50, 0x79, 0xd5, 0x32, 0x03, 0x25, 0x44, 0x96, 0xe7, 0xcf, 0x60, + 0xb3, 0x15, 0x24, 0x4a, 0x63, 0x6c, 0x59, 0x0e, 0xa1, 0x66, 0x38, 0xb3, 0xcd, 0xd9, 0x2d, 0x6e, + 0x16, 0x52, 0x44, 0x5a, 0x72, 0xba, 0xfd, 0x6d, 0xd9, 0x70, 0xbe, 0x2f, 0x1b, 0xce, 0x8f, 0x65, + 0xc3, 0xf9, 0xfc, 0xb3, 0x71, 0x6b, 0xb0, 0x46, 0x7f, 0xeb, 0xc7, 0xbf, 0x02, 0x00, 0x00, 0xff, + 0xff, 0x77, 0x02, 0x29, 0xbb, 0xbe, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index b791efb6c..b83683d51 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -77,8 +77,12 @@ message Index { repeated Frame Frames = 4; } -message NodeState { +message NodeStatus { string Host = 1; string State = 2; repeated Index Indexes = 3; } + +message ClusterStatus { + repeated NodeStatus Nodes = 1; +} diff --git a/server.go b/server.go index 6aef3dd62..336d76b54 100644 --- a/server.go +++ b/server.go @@ -118,6 +118,7 @@ func (s *Server) Open() error { // Initialize HTTP handler. s.Handler.Broadcaster = s.Broadcaster + s.Handler.StatusHandler = s s.Handler.Host = s.Host s.Handler.Cluster = s.Cluster s.Handler.Executor = e @@ -271,28 +272,49 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { return nil } -// Server implements gossip.StateHandler. -// LocalState returns the state of the local node as well as the +// Server implements StatusHandler. +// LocalStatus returns the state of the local node as well as the // holder (indexes/frames) according to the local node. // In a gossip implementation, memberlist.Delegate.LocalState() uses this. -func (s *Server) LocalState() (proto.Message, error) { +func (s *Server) LocalStatus() (proto.Message, error) { if s.Holder == nil { return nil, errors.New("Server.Holder is nil.") } - return &internal.NodeState{ + return &internal.NodeStatus{ Host: s.Host, - State: "OK", // TODO: make this work, pull from s.Cluster.Node + State: NodeStateUp, Indexes: encodeIndexes(s.Holder.Indexes()), }, nil } -// HandleRemoteState receives incoming NodeState from remote nodes. -func (s *Server) HandleRemoteState(pb proto.Message) error { - return s.mergeRemoteState(pb.(*internal.NodeState)) +// ClusterStatus returns the NodeState for all nodes in the cluster. +func (s *Server) ClusterStatus() (proto.Message, error) { + // Update local Node.state. + ns, err := s.LocalStatus() + if err != nil { + return nil, err + } + node := s.Cluster.NodeByHost(s.Host) + node.SetStatus(ns.(*internal.NodeStatus)) + + // Update NodeState for all nodes. + for host, nodeState := range s.Cluster.NodeStates() { + node := s.Cluster.NodeByHost(host) + node.SetState(nodeState) + } + + return s.Cluster.Status(), nil } -func (s *Server) mergeRemoteState(ns *internal.NodeState) error { - // TODO: update some node state value in the cluster (it should be in cluster.node i guess) +// HandleRemoteStatus receives incoming NodeState from remote nodes. +func (s *Server) HandleRemoteStatus(pb proto.Message) error { + return s.mergeRemoteStatus(pb.(*internal.NodeStatus)) +} + +func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { + // Update Node.state. + node := s.Cluster.NodeByHost(ns.Host) + node.SetStatus(ns) // Create indexes that don't exist. for _, index := range ns.Indexes { @@ -364,3 +386,12 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { return pb.MaxSlices, nil } + +// StatusHandler specifies two methods which an object must implement to share +// state in the cluster. These are used by the GossipNodeSet to implement the +// LocalState and MergeRemoteState methods of memberlist.Delegate +type StatusHandler interface { + LocalStatus() (proto.Message, error) + ClusterStatus() (proto.Message, error) + HandleRemoteStatus(proto.Message) error +} From 5857438d8bbd44c6604c065b05f14595a9d20e7c Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 24 Apr 2017 13:56:26 -0500 Subject: [PATCH 60/63] fixed state to status --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index f9f22464f..0ed3dfcb2 100644 --- a/cluster.go +++ b/cluster.go @@ -24,7 +24,7 @@ type Node struct { Host string `json:"host"` InternalHost string `json:"internalHost"` - status *internal.NodeStatus `json:"state"` + status *internal.NodeStatus `json:"status"` } // SetStatus sets the NodeStatus. From acc4b022d78cb77881996ba0081355d5aaef1db8 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 24 Apr 2017 13:57:35 -0500 Subject: [PATCH 61/63] added Join method to Static Nodeset so that Status is calculated with the local node --- broadcast.go | 5 +++++ server/server.go | 1 + 2 files changed, 6 insertions(+) diff --git a/broadcast.go b/broadcast.go index 5a1600fbb..9e04d04f6 100644 --- a/broadcast.go +++ b/broadcast.go @@ -34,6 +34,11 @@ func (s *StaticNodeSet) Open() error { return nil } +func (s *StaticNodeSet) Join(nodes []*Node) error { + s.nodes = nodes + return nil +} + // Broadcaster is an interface for broadcasting messages. type Broadcaster interface { SendSync(pb proto.Message) error diff --git a/server/server.go b/server/server.go index 153f2dc23..d5983d7c3 100644 --- a/server/server.go +++ b/server/server.go @@ -163,6 +163,7 @@ func (m *Command) SetupServer() error { m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver + m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes) default: return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type) } From 1dc5d6f3485537f75223e609659e6efb42e43eda Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 24 Apr 2017 14:14:16 -0500 Subject: [PATCH 62/63] handle Join error --- server/server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index d5983d7c3..da5fe9a8e 100644 --- a/server/server.go +++ b/server/server.go @@ -163,7 +163,10 @@ func (m *Command) SetupServer() error { m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes) + err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes) + if err != nil { + return err + } default: return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type) } From 445023a04f177c7706a5e958e4556cd22b003435 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Mon, 24 Apr 2017 14:21:50 -0500 Subject: [PATCH 63/63] fixed some comments --- server/server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/server.go b/server/server.go index da5fe9a8e..b6087ba21 100644 --- a/server/server.go +++ b/server/server.go @@ -1,4 +1,4 @@ -// package server contains the `pilosa server` subcommand which runs Pilosa +// Package server contains the `pilosa server` subcommand which runs Pilosa // itself. The purpose of this package is to define an easily tested Command // object which handles interpreting configuration and setting up all the // objects that Pilosa needs. @@ -50,7 +50,7 @@ type Command struct { Done chan struct{} } -// NewMain returns a new instance of Main. +// NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { return &Command{ Server: pilosa.NewServer(), @@ -89,6 +89,7 @@ func (m *Command) Run(args ...string) (err error) { return nil } +// SetupServer use the cluster configuration to setup this server func (m *Command) SetupServer() error { cluster := pilosa.NewCluster() cluster.ReplicaN = m.Config.Cluster.ReplicaN @@ -168,7 +169,7 @@ func (m *Command) SetupServer() error { return err } default: - return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type) + return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type) } // Set configuration options.