diff --git a/.travis.yml b/.travis.yml index fe4a2b8f1..beb41e392 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - 1.7 - 1.8 + - 1.9 - master addons: before_install: diff --git a/Gopkg.lock b/Gopkg.lock index 17ba49821..3cfe7b2cf 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -194,6 +194,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "84ff0992f3a6023a9d4832d6aea43d6aec19878a4b9e991ba8ff8269b589e816" + inputs-digest = "e8e78a7c61547d8f4d967c8deac9334b3151e7c10c4f7909e80758956e9c8204" solver-name = "gps-cdcl" solver-version = 1 diff --git a/bitmap.go b/bitmap.go index ecdb4c076..72296efab 100644 --- a/bitmap.go +++ b/bitmap.go @@ -97,6 +97,26 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return &Bitmap{segments: segments} } +// Xor returns the xor of b and other. +func (b *Bitmap) Xor(other *Bitmap) *Bitmap { + var segments []BitmapSegment + + itr := newMergeSegmentIterator(b.segments, other.segments) + for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { + if s1 == nil { + segments = append(segments, *s0) + continue + } else if s0 == nil { + segments = append(segments, *s1) + continue + } + + segments = append(segments, *s0.Xor(s1)) + } + + return &Bitmap{segments: segments} +} + // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { var segments []BitmapSegment @@ -342,6 +362,17 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment { } } +// Xor returns the xor of s and other. +func (s *BitmapSegment) Xor(other *BitmapSegment) *BitmapSegment { + data := s.data.Xor(&other.data) + + return &BitmapSegment{ + data: *data, + slice: s.slice, + n: data.Count(), + } +} + // SetBit sets the i-th bit of the bitmap. func (s *BitmapSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() diff --git a/bitmap_test.go b/bitmap_test.go new file mode 100644 index 000000000..4fcd1b115 --- /dev/null +++ b/bitmap_test.go @@ -0,0 +1,92 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "reflect" + "testing" + + "github.com/pilosa/pilosa" +) + +// Ensure a bitmap can be merged +func TestBitmap_Merge(t *testing.T) { + bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth) + bm2 := pilosa.NewBitmap(3, 4, 5) + bm1.Merge(bm2) + + if bm1.Count() != 7 { + t.Fatalf("Count after merge %d != 7\n", bm1.Count()) + } + +} + +// Ensure a bitmap can Xor'ed +func TestBitmap_Xor(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{1, SliceWidth, 2 * SliceWidth} + + res := bm1.Xor(bm2) + if res.Count() != 3 { + t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count()) + } + + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp) + } + res = bm2.Xor(bm1) + if res.Count() != 3 { + t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp) + } + +} +func TestBitmap_Union_Segment(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth} + res := bm1.Union(bm2) + + if res.Count() != 4 { + t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + } + res = bm2.Union(bm1) + if res.Count() != 4 { + t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + } +} + +func TestBitmap_Difference_Segment(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{1, SliceWidth} + res := bm1.Difference(bm2) + + if res.Count() != 2 { + t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp) + } +} diff --git a/broadcast.go b/broadcast.go index 866b03d02..0785a21cc 100644 --- a/broadcast.go +++ b/broadcast.go @@ -115,6 +115,7 @@ const ( MessageTypeDeleteFrame = 5 MessageTypeCreateInputDefinition = 6 MessageTypeDeleteInputDefinition = 7 + MessageTypeDeleteView = 8 ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -135,6 +136,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) { typ = MessageTypeCreateInputDefinition case *internal.DeleteInputDefinitionMessage: typ = MessageTypeDeleteInputDefinition + case *internal.DeleteViewMessage: + typ = MessageTypeDeleteView default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -165,6 +168,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { m = &internal.CreateInputDefinitionMessage{} case MessageTypeDeleteInputDefinition: m = &internal.DeleteInputDefinitionMessage{} + case MessageTypeDeleteView: + m = &internal.DeleteViewMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) } diff --git a/cache.go b/cache.go index 242e5a6ab..2e66d35b9 100644 --- a/cache.go +++ b/cache.go @@ -256,8 +256,10 @@ func (c *RankCache) recalculate() { length := len(c.rankings) c.stats.Gauge("RankCache", float64(length), 1.0) + var removeItems []BitmapPair // cached, ordered list if length > int(c.maxEntries) { c.thresholdValue = rankings[c.maxEntries].Count + removeItems = c.rankings[c.maxEntries:] c.rankings = c.rankings[0:c.maxEntries] } else { c.thresholdValue = 1 @@ -269,10 +271,8 @@ func (c *RankCache) recalculate() { // If size is larger than the threshold then trim it. if len(c.entries) > c.thresholdBuffer { c.stats.Count("cache.threshold", 1, 1.0) - for id, cnt := range c.entries { - if cnt <= c.thresholdValue { - delete(c.entries, id) - } + for _, pair := range removeItems { + delete(c.entries, pair.ID) } } } diff --git a/cache_test.go b/cache_test.go new file mode 100644 index 000000000..21d8acf07 --- /dev/null +++ b/cache_test.go @@ -0,0 +1,35 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "testing" + + "github.com/pilosa/pilosa" +) + +// Ensure a bitmap query can be executed. +func TestCache_Rank(t *testing.T) { + cacheSize := uint32(3) + cache := pilosa.NewRankCache(cacheSize) + for i := 1; i < int(2*cacheSize); i++ { + cache.Add(uint64(i), 3) + } + cache.Recalculate() + if cache.Len() != int(cacheSize) { + t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize) + } + +} diff --git a/ctl/export_test.go b/ctl/export_test.go index 433078374..4f9d9b184 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -16,13 +16,13 @@ package ctl import ( "bytes" + "context" "net/http" "strings" "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" - "golang.org/x/net/context" ) func TestExportCommand_Validation(t *testing.T) { diff --git a/ctl/import_test.go b/ctl/import_test.go index 8c4d3793e..dd9eba675 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -16,6 +16,7 @@ package ctl import ( "bytes" + "context" "io" "io/ioutil" "net/http" @@ -24,7 +25,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" - "golang.org/x/net/context" ) func TestImportCommand_Validation(t *testing.T) { diff --git a/ctl/restore_test.go b/ctl/restore_test.go index 5fd5af1d8..39ac92388 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -17,13 +17,13 @@ package ctl import ( "bufio" "bytes" + "context" "io" "io/ioutil" "testing" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/test" - "golang.org/x/net/context" ) func TestRestoreCommand_FileRequired(t *testing.T) { diff --git a/docs/administration.md b/docs/administration.md index 04b1b2bf0..786f10992 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -1,5 +1,12 @@ +++ -title = "Administration Guide" +title = "Administration" +weight = 13 +nav = [ + "Installing in production", + "Imports and Exports", + "Versioning", + "Backup/restore", +] +++ ## Administration Guide @@ -164,4 +171,4 @@ We currently track the following events Goroutines: Number of running Goroutines. -OpenFiles: Number of open file handles associated with running Pilosa process ID. \ No newline at end of file +OpenFiles: Number of open file handles associated with running Pilosa process ID. diff --git a/docs/api-reference.md b/docs/api-reference.md index 0ee460562..bf599fafe 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1,5 +1,7 @@ +++ title = "API Reference" +weight = 10 +nav = [] +++ @@ -113,6 +115,8 @@ Response: } ``` +By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`. + ### Change index time quantum `PATCH /index//time-quantum` diff --git a/docs/architecture.md b/docs/architecture.md index 0ff35565c..5cc5ff6ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,7 @@ +++ title = "Architecture" +weight = 6 +nav = [] +++ ## Architecture diff --git a/docs/client-libraries.md b/docs/client-libraries.md index e7a439371..e896fbab4 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -1,5 +1,11 @@ +++ title = "Client Libraries" +weight = 12 +nav = [ + "Go", + "Python", + "Java", +] +++ ## Client Libraries diff --git a/docs/configuration.md b/docs/configuration.md index 08ff7c4a3..0c63e635a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,5 +1,12 @@ +++ title = "Configuration" +weight = 7 +nav = [ + "Command line flags", + "Environment variables", + "Config file", + "All Options", +] +++ ## Configuration @@ -200,3 +207,43 @@ Any flag that has a value that is a comma separated list on the command line bec [metric] poll-interval = "0m15s" ``` + +### Example Cluster Configuration + +A three node cluster could be minimally configured as follows: + +#### Node 0 + + data-dir = "/home/pilosa/data" + bind = "node0.pilosa.com:10101" + gossip-port = 12000 + gossip-seed = "node0.pilosa.com:12000" + + [cluster] + replicas = 1 + type = "gossip" + hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + +#### Node 1 + + data-dir = "/home/pilosa/data" + bind = "node1.pilosa.com:10101" + gossip-port = 12000 + gossip-seed = "node0.pilosa.com:12000" + + [cluster] + replicas = 1 + type = "gossip" + hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] + +#### Node 2 + + data-dir = "/home/pilosa/data" + bind = "node2.pilosa.com:10101" + gossip-port = 12000 + gossip-seed = "node0.pilosa.com:12000" + + [cluster] + replicas = 1 + type = "gossip" + hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"] diff --git a/docs/data-model.md b/docs/data-model.md index f8987f87b..2e7ac03e8 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -1,5 +1,17 @@ +++ title = "Data Model" +weight = 5 +nav = [ + "Overview", + "Index", + "Column", + "Row", + "Frame", + "Time Quantum", + "Attribute", + "Slice", + "View", +] +++ ## Data Model diff --git a/docs/faq.md b/docs/faq.md index cea161aa3..ef9f8bfc8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,5 +1,7 @@ +++ title = "FAQ" +weight = 15 +nav = [] +++ ## FAQ diff --git a/docs/getting-started.md b/docs/getting-started.md index 0c42683e9..534e733b4 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,5 +1,12 @@ +++ title = "Getting Started" +weight = 3 +nav = [ + "Starting Pilosa", + "Sample Project", + "Input Definition", + "What's Next?", +] +++ ## Getting Started diff --git a/docs/glossary.md b/docs/glossary.md index e4bef02e3..50906a0cf 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,5 +1,7 @@ +++ title = "Glossary" +weight = 14 +nav = [] +++ ## Glossary diff --git a/docs/input-definition.md b/docs/input-definition.md index 4b73c4991..c0c34ff7f 100644 --- a/docs/input-definition.md +++ b/docs/input-definition.md @@ -1,12 +1,17 @@ +++ title = "Input Definition" +weight = 8 +nav = [ + "Create the Schema", + "Import Data", +] +++ ## Input Definition This document builds on the data import concepts introduced in [Getting Started](../getting-started/). Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data. -#### Create the Schema Using an Input Definition +### Create the Schema Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations. @@ -26,7 +31,8 @@ curl localhost:10101/index/repository/input-definition/stargazer \ "frames": [ { "name": "language", - "options": { + "options": { + "rowLabel": "language_id", "inverseEnabled": true, "timeQuantum": "YMD" } @@ -34,6 +40,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \ { "name": "stargazer", "options": { + "rowLabel": "stargazer_id", "inverseEnabled": true, "timeQuantum": "YMD" } @@ -55,7 +62,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \ "Go": 5, "Java": 21, "JavaScript": 13, - "Python": 17, + "Python": 17 } } ], @@ -77,7 +84,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \ "valueDestination": "set-timestamp" } ], - "name": "time_value + "name": "time_value" } ] }' @@ -91,7 +98,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel - mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type. - set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame -#### Import Data Using an Input Definition +### Import Data The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started). @@ -104,14 +111,14 @@ curl localhost:10101/index/repository/input/stargazer \ { "language_id": "Go", "repo_id": 91720568, - "stargazer_id": 513114 + "stargazer_id": 513114, "time_value": "2017-05-18T20:40" }, { "language_id": "Python", "repo_id": 95122322 - }' - ] + } + ]' ``` As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID. @@ -121,8 +128,8 @@ The data input above is equivalent to the following `SetBit()` operations: curl localhost:10101/index/repository/query \ -X POST \ -d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114) - 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") + SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40") SetBit(frame="language", repo_id=91720568, language_id=5) SetBit(frame="language", repo_id=95122322, language_id=17) ' -``` \ No newline at end of file +``` diff --git a/docs/installation.md b/docs/installation.md index 9936fb95c..b6c81a2b4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,5 +1,10 @@ +++ title = "Installation" +weight = 2 +nav = [ + "Installing on MacOS", + "Installing on Linux", +] +++ diff --git a/docs/introduction.md b/docs/introduction.md index 87cd94a3c..d499c8c0a 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,5 +1,7 @@ +++ title = "Introduction" +weight = 1 +nav = [] +++ diff --git a/docs/pdk.md b/docs/pdk.md index 5d50a5b98..8c02dc279 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -1,5 +1,10 @@ +++ title = "PDK" +weight = 11 +nav = [ + "Library", + "Examples", +] +++ ## PDK diff --git a/docs/query-language.md b/docs/query-language.md index 93ddb846a..c9d5f1e05 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -1,5 +1,12 @@ +++ title = "Query Language" +weight = 6 +nav = [ + "Conventions", + "Arguments and Types", + "Write Operations", + "Read Operations", +] +++ ## Query Language diff --git a/docs/tutorials.md b/docs/tutorials.md index 350aa3849..d304dc80d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -1,5 +1,10 @@ +++ title = "Tutorials" +weight = 4 +nav = [ + "Transportation", + "Chemical similarity search", +] +++ ## Tutorials @@ -192,7 +197,7 @@ for pcount, topn in zip(pcounts, resp.json()['results']): average_amounts.append(float(wsum)/count) ``` -For more examples and details, see this [ipython notebook](https://github.com/alanbernstein/pilosa-notebooks/blob/master/taxi-use-case.ipynb). +For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb). ### Chemical similarity search diff --git a/docs/webui.md b/docs/webui.md index 8084cbb80..1e123393f 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -1,5 +1,10 @@ +++ title = "WebUI" +weight = 9 +nav = [ + "Console", + "Cluster Admin", +] +++ ## WebUI diff --git a/executor.go b/executor.go index d58be42f1..dfd63be90 100644 --- a/executor.go +++ b/executor.go @@ -161,6 +161,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s indexTag := fmt.Sprintf("index:%s", index) // Special handling for mutation and top-n calls. switch c.Name { + case "Average": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeAverage(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -174,6 +177,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": return nil, e.executeSetColumnAttrs(ctx, index, c, opt) + case "Sum": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeSum(ctx, index, c, slices, opt) case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, slices, opt) @@ -202,6 +208,41 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { return nil } +// executeAverage executes an average() call. +func (e *Executor) executeAverage(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return 0, errors.New("Average(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return 0, errors.New("Average(): field required") + } + + if len(c.Children) > 1 { + return 0, errors.New("Average() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeSumCountSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(SumCount) + return other.Add(v.(SumCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return 0, err + } + other, _ := result.(SumCount) + + if other.Count == 0 { + return 0, nil + } + return other.Sum / other.Count, nil +} + // executeBitmapCall executes a call that returns a bitmap. func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { // Execute calls in bulk on each remote node and merge. @@ -229,36 +270,43 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // If the row label is used then return bitmap attributes. bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { - - idx := e.Holder.Index(index) - if idx != nil { - columnLabel := idx.ColumnLabel() - if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { - attrs, err := idx.ColumnAttrStore().Attrs(columnID) - if err != nil { - return nil, err - } - bm.Attrs = attrs - } else if err != nil { - return nil, err - } else { - frame, _ := c.Args["frame"].(string) - if fr := idx.Frame(frame); fr != nil { - rowLabel := fr.RowLabel() - rowID, _, err := c.UintArg(rowLabel) - if err != nil { - return nil, err - } - attrs, err := fr.RowAttrStore().Attrs(rowID) + if opt.ExcludeAttrs { + bm.Attrs = map[string]interface{}{} + } else { + idx := e.Holder.Index(index) + if idx != nil { + columnLabel := idx.ColumnLabel() + if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { + attrs, err := idx.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err } bm.Attrs = attrs + } else if err != nil { + return nil, err + } else { + frame, _ := c.Args["frame"].(string) + if fr := idx.Frame(frame); fr != nil { + rowLabel := fr.RowLabel() + rowID, _, err := c.UintArg(rowLabel) + if err != nil { + return nil, err + } + attrs, err := fr.RowAttrStore().Attrs(rowID) + if err != nil { + return nil, err + } + bm.Attrs = attrs + } } } } } + if opt.ExcludeBits { + bm.segments = []BitmapSegment{} + } + return bm, nil } @@ -275,11 +323,84 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * return e.executeRangeSlice(ctx, index, c, slice) case "Union": return e.executeUnionSlice(ctx, index, c, slice) + case "Xor": + return e.executeXorSlice(ctx, index, c, slice) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } } +// executeSum executes a sum() call. +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return 0, errors.New("Sum(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return 0, errors.New("Sum(): field required") + } + + if len(c.Children) > 1 { + return 0, errors.New("Sum() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeSumCountSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(SumCount) + return other.Add(v.(SumCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return 0, err + } + other, _ := result.(SumCount) + + return other.Sum, nil +} + +// executeSumCountSlice executes calculates the sum & count for fields on a slice. +func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return SumCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return SumCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return SumCount{}, nil + } + + view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if view == nil { + return SumCount{}, nil + } + + vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + if err != nil { + return SumCount{}, err + } + return SumCount{ + Sum: int64(vsum) + (int64(vcount) * field.Min), + Count: int64(vcount), + }, nil +} + // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. @@ -508,6 +629,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + // Handle field ranges differently. + if c.HasConditionArg() { + return e.executeFieldRangeSlice(ctx, index, c, slice) + } + // Parse frame, use default if unset. frame, _ := c.Args["frame"].(string) if frame == "" { @@ -562,7 +688,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Parse end time. - endTimeStr, _ := c.Args["end"].(string) + endTimeStr, ok := c.Args["end"].(string) if !ok { return nil, errors.New("Range() end time required") } @@ -590,6 +716,64 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return bm, nil } +// executeFieldRangeSlice executes a range(field) call for a local slice. +func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + // Parse frame, use default if unset. + frame, _ := c.Args["frame"].(string) + if frame == "" { + frame = DefaultFrame + } + f := e.Holder.Frame(index, frame) + if f == nil { + return nil, ErrFrameNotFound + } + + // Remove frame field. + args := pql.CopyArgs(c.Args) + delete(args, "frame") + + // Only one conditional field should remain. + if len(args) == 0 { + return nil, errors.New("Range(): condition required") + } else if len(args) > 1 { + return nil, errors.New("Range(): too many arguments") + } + + // Extract condition field. + var fieldName string + var cond *pql.Condition + for k, v := range args { + vv, ok := v.(*pql.Condition) + if !ok { + return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) + } + fieldName, cond = k, vv + } + + // Only support integers for now. + value, ok := cond.Value.(int64) + if !ok { + return nil, errors.New("Range(): conditions only support integer values") + } + + // Find field. + field := f.Field(fieldName) + if field == nil { + return nil, ErrFieldNotFound + } else if value < field.Min || value > field.Max { + return NewBitmap(), nil + } + + // Retrieve fragment. + frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + if frag == nil { + return NewBitmap(), nil + } + + f.Stats.Count("range:field", 1, 1.0) + return frag.FieldRange(cond.Op, field.BitDepth(), uint64(value-field.Min)) +} + // executeUnionSlice executes a union() call for a local slice. func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { other := NewBitmap() @@ -609,6 +793,25 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C return other, nil } +// executeXorSlice executes a xor() call for a local slice. +func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + other := NewBitmap() + for i, input := range c.Children { + bm, err := e.executeBitmapCallSlice(ctx, index, input, slice) + if err != nil { + return nil, err + } + + if i == 0 { + other = bm + } else { + other = other.Xor(bm) + } + } + other.InvalidateCount() + return other, nil +} + // executeCount executes a count() call. func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { if len(c.Children) == 0 { @@ -1179,6 +1382,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu var err error switch call.Name { + case "Average", "Sum": + v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": @@ -1297,7 +1502,6 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod if n.Host == e.Host { resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn) } else if !opt.Remote { - results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt) if len(results) > 0 { resp.result = results[0] @@ -1371,7 +1575,9 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { - Remote bool + Remote bool + ExcludeAttrs bool + ExcludeBits bool } // decodeError returns an error representation of s if s is non-blank. @@ -1414,3 +1620,30 @@ func needsSlices(calls []*pql.Call) bool { } return false } + +// SumCount represents a grouping of sum & count for Sum() and Average() calls. +type SumCount struct { + Sum int64 `json:"sum"` + Count int64 `json:"count"` +} + +func (sc *SumCount) Add(other SumCount) SumCount { + return SumCount{ + Sum: sc.Sum + other.Sum, + Count: sc.Count + other.Count, + } +} + +func encodeSumCount(sc SumCount) *internal.SumCount { + return &internal.SumCount{ + Sum: sc.Sum, + Count: sc.Count, + } +} + +func decodeSumCount(pb *internal.SumCount) SumCount { + return SumCount{ + Sum: pb.Sum, + Count: pb.Count, + } +} diff --git a/executor_test.go b/executor_test.go index 3c0f7e8ad..2b99d60c7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -59,6 +59,25 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } + + // Inhibit bits. + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + t.Fatal(err) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected bits: %+v", bits) + } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + + // Inhibit attributes. + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + t.Fatal(err) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) + } else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { + fmt.Println("ATTRS", attrs) + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } }) t.Run("Column", func(t *testing.T) { @@ -187,6 +206,25 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { } } +// Ensure a xor query can be executed. +func TestExecutor_Execute_Xor(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + t.Fatal(err) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) + } +} + // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() @@ -422,6 +460,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { } }) } + func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -559,7 +598,128 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } +} +// Ensure a Sum() query can be executed. +func TestExecutor_Execute_Sum(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, + {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + + SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) + SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { + t.Fatal(err) + } else if result[0] != int64(200) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil { + t.Fatal(err) + } else if result[0] != int64(80) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) +} + +// Ensure a Average() query can be executed. +func TestExecutor_Execute_Average(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, + {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) + + SetFieldValue(frame=f, foo=20, bar=2000, columnID=0) + SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(frame=f, field=foo)`), nil, nil); err != nil { + t.Fatal(err) + } else if result[0] != int64(40) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil { + t.Fatal(err) + } else if result[0] != int64(30) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) } // Ensure a range query can be executed. @@ -613,6 +773,119 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } +// Ensure a Range(field) query can be executed. +func TestExecutor_Execute_FieldRange(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, + {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`) + + SetFieldValue(frame=f, foo=20, bar=2000, columnID=50) + SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`) + SetFieldValue(frame=other, foo=1000, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("EQ", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LT", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTE", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GT", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GTE", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BelowMin", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("AboveMax", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("ErrFrameNotFound", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { + t.Fatal(err) + } + }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + t.Fatal(err) + } + }) +} + // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c := test.NewCluster(2) diff --git a/fragment.go b/fragment.go index db0d3fb13..7e56c5597 100644 --- a/fragment.go +++ b/fragment.go @@ -39,6 +39,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" ) @@ -293,11 +294,13 @@ func (f *Fragment) close() error { // Flush cache if closing gracefully. if err := f.flushCache(); err != nil { f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path) + return err } // Close underlying storage. if err := f.closeStorage(); err != nil { f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path) + return err } // Remove checksums. @@ -536,14 +539,45 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) ( return changed, nil } -func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { +// FieldSum returns the sum of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + // Compute count based on the existance bit. + row := f.row(uint64(bitDepth), true, true) + if filter != nil { + row = row.Intersect(filter) + } + count = row.Count() + + // Compute the sum based on the bit count of each row multiplied by the + // place value of each row. For example, 10 bits in the 1's place plus + // 4 bits in the 2's place plus 3 bits in the 4's place equals a total + // sum of 30: + // + // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 + // + for i := uint(0); i < bitDepth; i++ { + row := f.row(uint64(i), true, true) + if filter != nil { + row = row.Intersect(filter) + } + sum += (1 << i) * row.Count() + } + + return sum, count, nil +} + +func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { - case RangeOpEQ: + case pql.EQ: return f.fieldRangeEQ(bitDepth, predicate) - case RangeOpLT, RangeOpLTE: - return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE) - case RangeOpGT, RangeOpGTE: - return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE) + case pql.LT, pql.LTE: + return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + case pql.GT, pql.GTE: + return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } diff --git a/fragment_test.go b/fragment_test.go index e7f112837..941554db4 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -24,6 +24,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" ) @@ -216,6 +217,45 @@ func TestFragment_SetFieldValue(t *testing.T) { }) } +// Ensure a fragment can sum field values. +func TestFragment_FieldSum(t *testing.T) { + const bitDepth = 16 + + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } + + t.Run("NoFilter", func(t *testing.T) { + if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { + t.Fatal(err) + } else if n != 4 { + t.Fatalf("unexpected count: %d", n) + } else if sum != 3800 { + t.Fatalf("unexpected sum: %d", sum) + } + }) + + t.Run("WithFilter", func(t *testing.T) { + if sum, n, err := f.FieldSum(pilosa.NewBitmap(2000, 4000, 5000), bitDepth); err != nil { + t.Fatal(err) + } else if n != 2 { + t.Fatalf("unexpected count: %d", n) + } else if sum != 600 { + t.Fatalf("unexpected sum: %d", sum) + } + }) +} + // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { const bitDepth = 16 @@ -236,7 +276,7 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for equality. - if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) @@ -263,28 +303,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields less than (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than or equal to (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields less than or equal to (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) @@ -311,28 +351,28 @@ func TestFragment_FieldRange(t *testing.T) { } // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) } // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected bits: %+v", b.Bits()) diff --git a/frame.go b/frame.go index de3dd6fbb..73412a1d3 100644 --- a/frame.go +++ b/frame.go @@ -27,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // Default frame settings. @@ -40,15 +41,6 @@ const ( DefaultCacheSize = 50000 ) -// List of operators for field range queries. -const ( - RangeOpEQ = "eq" - RangeOpLT = "lt" - RangeOpLTE = "lte" - RangeOpGT = "gt" - RangeOpGTE = "gte" -) - // Frame represents a container for views. type Frame struct { mu sync.Mutex @@ -124,11 +116,15 @@ func (f *Frame) MaxSlice() uint64 { f.mu.Lock() defer f.mu.Unlock() - view := f.views[ViewStandard] - if view == nil { - return 0 + var max uint64 + for _, view := range f.views { + if view.name == ViewInverse { + continue + } else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { + max = viewMaxSlice + } } - return view.MaxSlice() + return max } // MaxInverseSlice returns the max inverse slice in the frame. @@ -397,7 +393,9 @@ func (f *Frame) Close() error { // Close all views. for _, view := range f.views { - _ = view.Close() + if err := view.Close(); err != nil { + return err + } } f.views = make(map[string]*View) @@ -509,6 +507,28 @@ func (f *Frame) newView(path, name string) *View { return view } +// DeleteView removes the view from the frame. +func (f *Frame) DeleteView(name string) error { + view := f.views[name] + if view == nil { + return ErrInvalidView + } + + // Close data files before deletion. + if err := view.Close(); err != nil { + return err + } + + // Delete view directory. + if err := os.RemoveAll(view.Path()); err != nil { + return err + } + + delete(f.views, name) + + return nil +} + // SetBit sets a bit on a view within the frame. func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. @@ -639,7 +659,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change return view.SetFieldValue(columnID, field.BitDepth(), baseValue) } -func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) { +// FieldSum returns the sum and count for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil +} + +func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) if field == nil { diff --git a/frame_test.go b/frame_test.go index 8cb48cca8..50f5fdef5 100644 --- a/frame_test.go +++ b/frame_test.go @@ -307,3 +307,36 @@ func TestFrame_RowLabelValidation(t *testing.T) { } } + +// Ensure frame can open and retrieve a view. +func TestFrame_DeleteView(t *testing.T) { + f := test.MustOpenFrame() + defer f.Close() + + viewName := pilosa.ViewStandard + "_v" + + // Create view. + view, err := f.CreateViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + err = f.DeleteView(viewName) + if err != nil { + t.Fatal(err) + } + + if f.View(viewName) != nil { + t.Fatal("view still exists in frame") + } + + // Recreate view with same name, verify that the old view was not reused. + view2, err := f.CreateViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == view2 { + t.Fatal("failed to create new view") + } +} diff --git a/gossip/gossip.go b/gossip/gossip.go index a967079cb..e2e43a00b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -69,6 +69,12 @@ func (g *GossipNodeSet) Open() error { return err } g.memberlist = ml + g.broadcasts = &memberlist.TransmitLimitedQueue{ + NumNodes: func() int { + return ml.NumMembers() + }, + RetransmitMult: 3, + } // attach to gossip seed node nodes := []*pilosa.Node{&pilosa.Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds @@ -76,12 +82,6 @@ func (g *GossipNodeSet) Open() error { if err != nil { return err } - g.broadcasts = &memberlist.TransmitLimitedQueue{ - NumNodes: func() int { - return ml.NumMembers() - }, - RetransmitMult: 3, - } return nil } diff --git a/handler.go b/handler.go index f3bf5c786..0aae927e0 100644 --- a/handler.go +++ b/handler.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "log" "net/http" + // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" "os" "runtime/debug" @@ -43,6 +44,7 @@ import ( "unicode" + // Allow building Pilosa without the web UI. _ "github.com/pilosa/pilosa/statik" "github.com/rakyll/statik/fs" ) @@ -116,6 +118,7 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET") + router.HandleFunc("/index/{index}/frame/{frame}/view/{view}", handler.handleDeleteView).Methods("DELETE") router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") @@ -241,7 +244,9 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Build execution options. opt := &ExecOptions{ - Remote: req.Remote, + Remote: req.Remote, + ExcludeAttrs: req.ExcludeAttrs, + ExcludeBits: req.ExcludeBits, } // Parse query string. @@ -257,7 +262,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp := &QueryResponse{Results: results, Err: err} // Fill column attributes if requested. - if req.ColumnAttrs { + if req.ColumnAttrs && !req.ExcludeBits { // Consolidate all column ids across all calls. var columnIDs []uint64 for _, result := range results { @@ -789,6 +794,47 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { } } +// handleDeleteView handles Delete /frame/view request. +func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + frameName := mux.Vars(r)["frame"] + viewName := mux.Vars(r)["view"] + + // Retrieve frame. + f := h.Holder.Frame(indexName, frameName) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Delete the view. + if err := f.DeleteView(viewName); err != nil { + // Ingore this error becuase views do not exist on all nodes due to slice distribution. + if err != ErrInvalidView { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + + // Send the delete view message to all nodes. + err := h.Broadcaster.SendSync( + &internal.DeleteViewMessage{ + Index: indexName, + Frame: frameName, + View: viewName, + }) + if err != nil { + h.logger().Printf("problem sending DeleteView message: %s", err) + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type deleteViewResponse struct{} + type getFrameViewsResponse struct { Views []string `json:"views,omitempty"` } @@ -925,9 +971,11 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeAttrs: q.Get("excludeAttrs") == "true", + ExcludeBits: q.Get("excludeBits") == "true", }, nil } @@ -1396,6 +1444,12 @@ type QueryRequest struct { // Return column attributes, if true. ColumnAttrs bool + // Do not return row attributes, if true. + ExcludeAttrs bool + + // Do not return bits, if true. + ExcludeBits bool + // If true, indicates that query is part of a larger distributed query. // If false, this request is on the originating node. Remote bool @@ -1403,10 +1457,12 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeAttrs: pb.ExcludeAttrs, + ExcludeBits: pb.ExcludeBits, } return req @@ -1455,6 +1511,8 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { pb.Results[i].Bitmap = encodeBitmap(result) case []Pair: pb.Results[i].Pairs = encodePairs(result) + case SumCount: + pb.Results[i].SumCount = encodeSumCount(result) case uint64: pb.Results[i].N = result case bool: diff --git a/handler_test.go b/handler_test.go index 22a1fd4f3..dacb04b38 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1511,3 +1511,25 @@ func TestHandler_GetTimeStamp(t *testing.T) { t.Fatalf("Expected Ignore nonexistent fields") } } + +// Ensure handler can delete a view. +func TestHandler_DeleteView(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + viewName := pilosa.ViewStandard + "_2017" + hldr.MustCreateFragmentIfNotExists("i0", "f0", viewName, 1).MustSetBits(30, (1*SliceWidth)+1) + hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD") + + h := test.NewHandler() + h.Holder = hldr.Holder + h.Cluster = test.NewCluster(1) + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader(""))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := hldr.Index("i0").Frame("f0").View(viewName); f != nil { + t.Fatal("expected nil view") + } +} diff --git a/holder.go b/holder.go index f7c363524..4270c2594 100644 --- a/holder.go +++ b/holder.go @@ -133,7 +133,9 @@ func (h *Holder) Close() error { h.wg.Wait() for _, index := range h.indexes { - index.Close() + if err := index.Close(); err != nil { + return err + } } return nil } diff --git a/holder_test.go b/holder_test.go index 108349bb2..b3947abb8 100644 --- a/holder_test.go +++ b/holder_test.go @@ -34,8 +34,9 @@ func TestHolder_Open(t *testing.T) { if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } - if err := h.Reopen(); err != nil { t.Fatal(err) } else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) { @@ -49,6 +50,8 @@ func TestHolder_Open(t *testing.T) { if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil { t.Fatal(err) } @@ -64,6 +67,8 @@ func TestHolder_Open(t *testing.T) { if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { t.Fatal(err) } @@ -78,6 +83,8 @@ func TestHolder_Open(t *testing.T) { if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".data"), 2); err != nil { t.Fatal(err) } @@ -95,6 +102,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil { t.Fatal(err) } @@ -112,6 +121,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil { t.Fatal(err) } @@ -128,6 +139,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil { t.Fatal(err) } @@ -147,6 +160,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil { t.Fatal(err) } @@ -166,6 +181,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil { t.Fatal(err) } @@ -188,6 +205,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := view.SetBit(0, 0); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil { t.Fatal(err) } @@ -209,6 +228,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if _, err := view.SetBit(0, 0); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil { t.Fatal(err) } @@ -232,6 +253,8 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } else if err := view.Fragment(0).FlushCache(); err != nil { t.Fatal(err) + } else if err := h.Holder.Close(); err != nil { + t.Fatal(err) } else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil { t.Fatal(err) } diff --git a/index.go b/index.go index 8f1846c5f..345215690 100644 --- a/index.go +++ b/index.go @@ -248,7 +248,9 @@ func (i *Index) Close() error { // Close all frames. for _, f := range i.frames { - f.Close() + if err := f.Close(); err != nil { + return err + } } i.frames = make(map[string]*Frame) diff --git a/internal/private.pb.go b/internal/private.pb.go index abde718e9..0a9956279 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,6 +32,7 @@ ClusterStatus FrameSchema Field + DeleteViewMessage */ package internal @@ -405,6 +406,17 @@ func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +type DeleteViewMessage struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` +} + +func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } +func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } +func (*DeleteViewMessage) ProtoMessage() {} +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } + func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") @@ -429,6 +441,7 @@ func init() { proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema") proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -1344,6 +1357,42 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if len(m.View) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.View))) + i += copy(dAtA[i:], m.View) + } + return i, nil +} + func encodeFixed64Private(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) dAtA[offset+1] = uint8(v >> 8) @@ -1773,6 +1822,24 @@ func (m *Field) Size() (n int) { return n } +func (m *DeleteViewMessage) Size() (n int) { + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + l = len(m.View) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + return n +} + func sovPrivate(x uint64) (n int) { for { n++ @@ -4882,6 +4949,143 @@ func (m *Field) Unmarshal(dAtA []byte) error { } return nil } +func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DeleteViewMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DeleteViewMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field View", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.View = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPrivate(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -4990,62 +5194,64 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 912 bytes of a gzipped FileDescriptorProto + // 929 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45, - 0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81, - 0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd, - 0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0xa6, 0xdb, 0x49, 0xcc, 0x81, 0x23, - 0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0x8a, 0xba, 0xba, - 0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x85, 0x5b, 0xd7, 0xeb, 0xd7, 0x55, 0xaf, 0x6a, 0xaa, 0xca, 0x86, - 0xf5, 0x69, 0x16, 0x9d, 0x4b, 0x8d, 0x07, 0xd3, 0x2c, 0xd5, 0x29, 0x6b, 0x45, 0x89, 0xc6, 0x2c, - 0x91, 0x31, 0xff, 0x0a, 0xda, 0x83, 0xe4, 0x14, 0x2f, 0x87, 0xa8, 0x25, 0xdb, 0x85, 0xb5, 0xc3, - 0x34, 0x9e, 0x4d, 0x92, 0x2f, 0xe5, 0x09, 0xc6, 0xdd, 0x60, 0x37, 0xd8, 0x6b, 0x8b, 0x32, 0x64, - 0x18, 0x2f, 0xa2, 0x09, 0x7e, 0x33, 0x93, 0x89, 0x9e, 0x4d, 0xba, 0x35, 0xcb, 0x28, 0x41, 0xfc, - 0xcf, 0x00, 0xda, 0xcf, 0x32, 0x39, 0x41, 0xf2, 0xb8, 0x0d, 0x2d, 0x91, 0x5e, 0x94, 0xdd, 0xe5, - 0x36, 0x7b, 0x0f, 0x36, 0x06, 0xc9, 0x39, 0x66, 0x0a, 0x8f, 0x12, 0x79, 0x12, 0xe3, 0x29, 0xb9, - 0x6b, 0x89, 0x0a, 0xca, 0x76, 0xa0, 0x7d, 0x28, 0xc7, 0xaf, 0xf1, 0xc5, 0x7c, 0x8a, 0xdd, 0x3a, - 0x39, 0x29, 0x80, 0xfc, 0x76, 0x14, 0xfd, 0x88, 0xdd, 0x70, 0x37, 0xd8, 0x5b, 0x17, 0x05, 0x50, - 0xd5, 0xdb, 0x58, 0xd2, 0xcb, 0x38, 0xbc, 0x25, 0x64, 0xf2, 0x2a, 0xd7, 0xd0, 0x24, 0x0d, 0x0b, - 0x18, 0xe7, 0xb0, 0x31, 0x98, 0x4c, 0xd3, 0x4c, 0x0b, 0x54, 0xd3, 0x34, 0x51, 0xc8, 0x3a, 0x50, - 0x3f, 0xca, 0x32, 0x97, 0x92, 0x39, 0xf2, 0x9f, 0xa0, 0xf3, 0x34, 0x4e, 0xc7, 0x67, 0x7d, 0xa9, - 0xa5, 0xc0, 0x1f, 0x66, 0xa8, 0x34, 0xdb, 0x82, 0x06, 0x15, 0xd7, 0xf1, 0xac, 0x61, 0x50, 0x2a, - 0x90, 0xab, 0x9e, 0x35, 0x0c, 0x4a, 0xef, 0x29, 0xc3, 0x50, 0x58, 0xc3, 0xa0, 0xa3, 0x38, 0x1a, - 0xdb, 0xcc, 0x42, 0x61, 0x0d, 0xc6, 0x20, 0x7c, 0x19, 0xe1, 0x85, 0x4b, 0x87, 0xce, 0x7c, 0x00, - 0xf7, 0x4a, 0xf1, 0x9d, 0xcc, 0xfb, 0xd0, 0x14, 0xe9, 0xc5, 0xa0, 0xaf, 0xba, 0xc1, 0x6e, 0x7d, - 0x2f, 0x14, 0xce, 0xa2, 0xa2, 0xd1, 0x57, 0x35, 0x57, 0x35, 0xba, 0x2a, 0x00, 0xfe, 0x00, 0x1a, - 0x54, 0x41, 0x93, 0x65, 0xf1, 0xd6, 0x1c, 0xf9, 0xaf, 0x01, 0xdc, 0x1b, 0xca, 0x4b, 0x92, 0xa1, - 0xf2, 0x30, 0xc7, 0xd0, 0xce, 0x41, 0x62, 0xaf, 0x3d, 0xda, 0x3f, 0xf0, 0x2d, 0x76, 0xb0, 0xc4, - 0x2f, 0x90, 0xa3, 0x44, 0x67, 0x73, 0x51, 0x3c, 0xde, 0xfe, 0x14, 0x36, 0x16, 0x2f, 0x8d, 0x86, - 0x33, 0x9c, 0xfb, 0x4a, 0x9f, 0xe1, 0xdc, 0xd4, 0xe4, 0x5c, 0xc6, 0x33, 0x5b, 0xbf, 0x50, 0x58, - 0xe3, 0x93, 0xda, 0x47, 0x01, 0xff, 0x0e, 0xd8, 0x61, 0x86, 0x52, 0x23, 0x39, 0x18, 0xa2, 0x52, - 0xf2, 0x15, 0xae, 0xfe, 0x0a, 0xb6, 0xb2, 0xb5, 0x72, 0x65, 0x77, 0xa0, 0x3d, 0x50, 0xae, 0xff, - 0xe8, 0x4b, 0xb4, 0x44, 0x01, 0xf0, 0x7d, 0x60, 0x7d, 0x8c, 0x51, 0xa3, 0x1b, 0x99, 0x1b, 0xfc, - 0xf3, 0x91, 0xd7, 0x72, 0x3b, 0x97, 0x3d, 0x84, 0xd0, 0x4c, 0x0b, 0x49, 0x59, 0x7b, 0xf4, 0x76, - 0x51, 0xba, 0x7c, 0x34, 0x05, 0x11, 0x78, 0xe4, 0x9d, 0xba, 0x09, 0xbb, 0x25, 0xc1, 0x6b, 0xda, - 0xcc, 0x87, 0xaa, 0x57, 0x43, 0xe5, 0x33, 0xeb, 0x42, 0x7d, 0xe6, 0x73, 0xfd, 0xbf, 0xa1, 0x78, - 0xdf, 0xa1, 0xa6, 0x5d, 0x9f, 0x9b, 0x5b, 0xfb, 0x86, 0xce, 0xab, 0x53, 0xae, 0xea, 0xf8, 0x27, - 0x70, 0x21, 0xef, 0xe6, 0xa6, 0x52, 0x39, 0xb3, 0x88, 0x7c, 0x63, 0xb9, 0x09, 0xcb, 0x6d, 0xf6, - 0x10, 0x9a, 0x14, 0x55, 0x75, 0x43, 0xea, 0xdd, 0xcd, 0x8a, 0x1a, 0xe1, 0xae, 0xcd, 0x38, 0xb9, - 0x26, 0x6f, 0xd8, 0x71, 0xb2, 0x16, 0x3b, 0x82, 0xce, 0x20, 0x99, 0xce, 0x74, 0x1f, 0xbf, 0x8f, - 0x92, 0x48, 0x47, 0x69, 0xa2, 0xba, 0x4d, 0x72, 0xf5, 0xa0, 0xac, 0x68, 0x81, 0x21, 0x96, 0x9e, - 0xf0, 0x9f, 0x03, 0xd8, 0xac, 0x80, 0x2b, 0x92, 0xf6, 0x7a, 0x6b, 0x37, 0xeb, 0x7d, 0x0c, 0xcd, - 0x67, 0x11, 0xc6, 0xa7, 0xaa, 0x5b, 0x27, 0x62, 0x6f, 0xa5, 0x1a, 0xa2, 0x09, 0xc7, 0xe6, 0xbf, - 0x05, 0xb0, 0x75, 0x1d, 0xe1, 0x5a, 0x35, 0x3d, 0x80, 0xaf, 0xb3, 0x68, 0x22, 0xb3, 0xf9, 0x17, - 0x38, 0x77, 0x2b, 0xbc, 0x84, 0xb0, 0x6f, 0xe1, 0x7e, 0xc5, 0xd7, 0xe7, 0x63, 0x5b, 0x22, 0x2b, - 0xea, 0xdd, 0x95, 0xa2, 0x2c, 0x4f, 0xac, 0x78, 0xce, 0xff, 0x0d, 0xe0, 0x9d, 0x6b, 0xaf, 0x8a, - 0x7e, 0x0c, 0xca, 0xad, 0xbf, 0x0f, 0x9d, 0x97, 0x66, 0x55, 0xf4, 0x51, 0xe9, 0x28, 0x91, 0x86, - 0xe9, 0x1a, 0x76, 0x09, 0x67, 0x03, 0x68, 0x11, 0x36, 0x94, 0x53, 0x27, 0xf3, 0x83, 0x5b, 0x64, - 0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xf9, 0x73, 0x23, 0x86, 0xb6, 0xae, 0x5f, 0xe1, 0x64, 0x6c, 0x3f, - 0x81, 0xf5, 0x85, 0x07, 0x77, 0xda, 0x73, 0x29, 0xec, 0xf8, 0xdd, 0xb2, 0xa0, 0xe4, 0xe6, 0x29, - 0xfd, 0x18, 0xa0, 0xa0, 0xba, 0x05, 0x70, 0x43, 0x7f, 0x96, 0xc8, 0xfc, 0x18, 0x76, 0xfc, 0xe2, - 0xbb, 0x43, 0x40, 0xdf, 0x2d, 0xb5, 0xa2, 0x5b, 0xb8, 0x04, 0x78, 0x9e, 0x9e, 0xe2, 0x48, 0x4b, - 0x3d, 0x53, 0x86, 0x71, 0x9c, 0x2a, 0xed, 0xfb, 0xc9, 0x9c, 0x69, 0x31, 0x6b, 0xa9, 0xf3, 0x65, - 0x42, 0x06, 0x7b, 0x1f, 0xde, 0x24, 0xa7, 0xe8, 0xdb, 0x66, 0xb3, 0x32, 0xeb, 0xc2, 0xdf, 0xf3, - 0x27, 0xb0, 0x7e, 0x18, 0xcf, 0x94, 0xc6, 0xcc, 0x45, 0xd9, 0x87, 0x86, 0x89, 0xe9, 0x7f, 0x9a, - 0xb6, 0x8a, 0x97, 0x85, 0x14, 0x61, 0x29, 0xfc, 0x31, 0xac, 0x51, 0xb7, 0x8c, 0xc6, 0xaf, 0x71, - 0x22, 0x69, 0xd4, 0xec, 0x04, 0x05, 0x4b, 0xa3, 0xb6, 0x30, 0x32, 0x23, 0x68, 0xac, 0x1e, 0x11, - 0x06, 0x21, 0xfd, 0x79, 0x71, 0x85, 0xa0, 0xff, 0x2d, 0x1d, 0xa8, 0x0f, 0x23, 0xfb, 0x19, 0xea, - 0xc2, 0x1c, 0x09, 0x91, 0x97, 0xd4, 0x26, 0x06, 0x91, 0x97, 0x4f, 0x3b, 0xbf, 0x5f, 0xf5, 0x82, - 0x3f, 0xae, 0x7a, 0xc1, 0x5f, 0x57, 0xbd, 0xe0, 0x97, 0xbf, 0x7b, 0x6f, 0x9c, 0x34, 0xe9, 0xff, - 0xdb, 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x29, 0x07, 0x36, 0x04, 0xd0, 0x09, 0x00, 0x00, + 0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0x71, 0x9a, 0xb0, 0xf2, 0x46, 0x91, 0x89, 0xfa, 0xc0, + 0x86, 0x48, 0xe4, 0xb0, 0x48, 0x2b, 0x60, 0x39, 0xc0, 0xc6, 0x59, 0xc5, 0x02, 0x2f, 0xd0, 0x5e, + 0x2d, 0x37, 0xa4, 0x8e, 0x53, 0xec, 0x8e, 0x32, 0x9e, 0x31, 0x33, 0x3d, 0x49, 0xcc, 0x81, 0x23, + 0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0xba, 0xea, 0xea, + 0xee, 0x99, 0xf1, 0x38, 0x76, 0x94, 0xbd, 0x75, 0xbd, 0x7e, 0x5d, 0xf5, 0xba, 0xa6, 0xaa, 0xa6, + 0x61, 0x7d, 0x9a, 0x04, 0x17, 0x52, 0xe1, 0xe1, 0x34, 0x89, 0x55, 0xcc, 0x5a, 0x41, 0xa4, 0x30, + 0x89, 0x64, 0xc8, 0xbf, 0x81, 0xf6, 0x20, 0x3a, 0xc3, 0xab, 0x21, 0x2a, 0xc9, 0xf6, 0x60, 0xed, + 0x28, 0x0e, 0xb3, 0x49, 0xf4, 0xb5, 0x3c, 0xc5, 0xb0, 0xeb, 0xed, 0x79, 0xfb, 0x6d, 0x51, 0x86, + 0x34, 0xe3, 0x79, 0x30, 0xc1, 0xef, 0x32, 0x19, 0xa9, 0x6c, 0xd2, 0xad, 0x19, 0x46, 0x09, 0xe2, + 0x7f, 0x7b, 0xd0, 0x7e, 0x9a, 0xc8, 0x09, 0x92, 0xc7, 0x1d, 0x68, 0x89, 0xf8, 0xb2, 0xec, 0x2e, + 0xb7, 0xd9, 0x07, 0xb0, 0x31, 0x88, 0x2e, 0x30, 0x49, 0xf1, 0x38, 0x92, 0xa7, 0x21, 0x9e, 0x91, + 0xbb, 0x96, 0xa8, 0xa0, 0x6c, 0x17, 0xda, 0x47, 0x72, 0xfc, 0x0a, 0x9f, 0xcf, 0xa6, 0xd8, 0xad, + 0x93, 0x93, 0x02, 0xc8, 0x77, 0x47, 0xc1, 0xcf, 0xd8, 0xf5, 0xf7, 0xbc, 0xfd, 0x75, 0x51, 0x00, + 0x55, 0xbd, 0x8d, 0x05, 0xbd, 0x8c, 0xc3, 0x3b, 0x42, 0x46, 0x2f, 0x73, 0x0d, 0x4d, 0xd2, 0x30, + 0x87, 0x71, 0x0e, 0x1b, 0x83, 0xc9, 0x34, 0x4e, 0x94, 0xc0, 0x74, 0x1a, 0x47, 0x29, 0xb2, 0x0e, + 0xd4, 0x8f, 0x93, 0xc4, 0x5e, 0x49, 0x2f, 0xf9, 0x2f, 0xd0, 0x79, 0x12, 0xc6, 0xe3, 0xf3, 0xbe, + 0x54, 0x52, 0xe0, 0x4f, 0x19, 0xa6, 0x8a, 0x6d, 0x43, 0x83, 0x92, 0x6b, 0x79, 0xc6, 0xd0, 0x28, + 0x25, 0xc8, 0x66, 0xcf, 0x18, 0x1a, 0xa5, 0xf3, 0x74, 0x43, 0x5f, 0x18, 0x43, 0xa3, 0xa3, 0x30, + 0x18, 0x9b, 0x9b, 0xf9, 0xc2, 0x18, 0x8c, 0x81, 0xff, 0x22, 0xc0, 0x4b, 0x7b, 0x1d, 0x5a, 0xf3, + 0x01, 0x6c, 0x95, 0xe2, 0x5b, 0x99, 0xf7, 0xa0, 0x29, 0xe2, 0xcb, 0x41, 0x3f, 0xed, 0x7a, 0x7b, + 0xf5, 0x7d, 0x5f, 0x58, 0x8b, 0x92, 0x46, 0x5f, 0x55, 0x6f, 0xd5, 0x68, 0xab, 0x00, 0xf8, 0x7d, + 0x68, 0x50, 0x06, 0xf5, 0x2d, 0x8b, 0xb3, 0x7a, 0xc9, 0x7f, 0xf7, 0x60, 0x6b, 0x28, 0xaf, 0x48, + 0x46, 0x9a, 0x87, 0x39, 0x81, 0x76, 0x0e, 0x12, 0x7b, 0xed, 0xe1, 0xc1, 0xa1, 0x2b, 0xb1, 0xc3, + 0x05, 0x7e, 0x81, 0x1c, 0x47, 0x2a, 0x99, 0x89, 0xe2, 0xf0, 0xce, 0xe7, 0xb0, 0x31, 0xbf, 0xa9, + 0x35, 0x9c, 0xe3, 0xcc, 0x65, 0xfa, 0x1c, 0x67, 0x3a, 0x27, 0x17, 0x32, 0xcc, 0x4c, 0xfe, 0x7c, + 0x61, 0x8c, 0xcf, 0x6a, 0x9f, 0x78, 0xfc, 0x07, 0x60, 0x47, 0x09, 0x4a, 0x85, 0xe4, 0x60, 0x88, + 0x69, 0x2a, 0x5f, 0xe2, 0xf2, 0xaf, 0x60, 0x32, 0x5b, 0x2b, 0x67, 0x76, 0x17, 0xda, 0x83, 0xd4, + 0xd6, 0x1f, 0x7d, 0x89, 0x96, 0x28, 0x00, 0x7e, 0x00, 0xac, 0x8f, 0x21, 0x2a, 0xb4, 0x2d, 0xb3, + 0xc2, 0x3f, 0x1f, 0x39, 0x2d, 0xb7, 0x73, 0xd9, 0x03, 0xf0, 0x75, 0xb7, 0x90, 0x94, 0xb5, 0x87, + 0xef, 0x16, 0xa9, 0xcb, 0x5b, 0x53, 0x10, 0x81, 0x07, 0xce, 0xa9, 0xed, 0xb0, 0x5b, 0x2e, 0x78, + 0x43, 0x99, 0xb9, 0x50, 0xf5, 0x6a, 0xa8, 0xbc, 0x67, 0x6d, 0xa8, 0x2f, 0xdc, 0x5d, 0xdf, 0x34, + 0x14, 0xef, 0x5b, 0x54, 0x97, 0xeb, 0x33, 0xbd, 0x6b, 0xce, 0xd0, 0x7a, 0xf9, 0x95, 0xab, 0x3a, + 0xfe, 0xf3, 0x6c, 0xc8, 0xbb, 0xb9, 0xa9, 0x64, 0x4e, 0x0f, 0x22, 0x57, 0x58, 0xb6, 0xc3, 0x72, + 0x9b, 0x3d, 0x80, 0x26, 0x45, 0x4d, 0xbb, 0x3e, 0xd5, 0xee, 0x66, 0x45, 0x8d, 0xb0, 0xdb, 0xba, + 0x9d, 0x6c, 0x91, 0x37, 0x4c, 0x3b, 0x19, 0x8b, 0x1d, 0x43, 0x67, 0x10, 0x4d, 0x33, 0xd5, 0xc7, + 0x1f, 0x83, 0x28, 0x50, 0x41, 0x1c, 0xa5, 0xdd, 0x26, 0xb9, 0xba, 0x5f, 0x56, 0x34, 0xc7, 0x10, + 0x0b, 0x47, 0xf8, 0xaf, 0x1e, 0x6c, 0x56, 0xc0, 0x25, 0x97, 0x76, 0x7a, 0x6b, 0xab, 0xf5, 0x3e, + 0x82, 0xe6, 0xd3, 0x00, 0xc3, 0xb3, 0xb4, 0x5b, 0x27, 0x62, 0x6f, 0xa9, 0x1a, 0xa2, 0x09, 0xcb, + 0xe6, 0x7f, 0x78, 0xb0, 0x7d, 0x13, 0xe1, 0x46, 0x35, 0x3d, 0x80, 0x6f, 0x93, 0x60, 0x22, 0x93, + 0xd9, 0x57, 0x38, 0xb3, 0x23, 0xbc, 0x84, 0xb0, 0xef, 0xe1, 0x5e, 0xc5, 0xd7, 0x97, 0x63, 0x93, + 0x22, 0x23, 0xea, 0xfd, 0xa5, 0xa2, 0x0c, 0x4f, 0x2c, 0x39, 0xce, 0xff, 0xf7, 0xe0, 0xbd, 0x1b, + 0xb7, 0x8a, 0x7a, 0xf4, 0xca, 0xa5, 0x7f, 0x00, 0x9d, 0x17, 0x7a, 0x54, 0xf4, 0x31, 0x55, 0x41, + 0x24, 0x35, 0xd3, 0x16, 0xec, 0x02, 0xce, 0x06, 0xd0, 0x22, 0x6c, 0x28, 0xa7, 0x56, 0xe6, 0x47, + 0xb7, 0xc8, 0x3c, 0x74, 0x7c, 0x33, 0xd3, 0xf2, 0xe3, 0x5a, 0x0c, 0x4d, 0x5d, 0x37, 0xc2, 0xc9, + 0xd8, 0x79, 0x0c, 0xeb, 0x73, 0x07, 0xee, 0x34, 0xe7, 0x62, 0xd8, 0x75, 0xb3, 0x65, 0x4e, 0xc9, + 0xea, 0x2e, 0xfd, 0x14, 0xa0, 0xa0, 0xda, 0x01, 0xb0, 0xa2, 0x3e, 0x4b, 0x64, 0x7e, 0x02, 0xbb, + 0x6e, 0xf0, 0xdd, 0x21, 0xa0, 0xab, 0x96, 0x5a, 0x51, 0x2d, 0x5c, 0x02, 0x3c, 0x8b, 0xcf, 0x70, + 0xa4, 0xa4, 0xca, 0x52, 0xcd, 0x38, 0x89, 0x53, 0xe5, 0xea, 0x49, 0xaf, 0x69, 0x30, 0x2b, 0xa9, + 0xf2, 0x61, 0x42, 0x06, 0xfb, 0x10, 0xde, 0x26, 0xa7, 0xe8, 0xca, 0x66, 0xb3, 0xd2, 0xeb, 0xc2, + 0xed, 0xf3, 0xc7, 0xb0, 0x7e, 0x14, 0x66, 0xa9, 0xc2, 0xc4, 0x46, 0x39, 0x80, 0x86, 0x8e, 0xe9, + 0x7e, 0x4d, 0xdb, 0xc5, 0xc9, 0x42, 0x8a, 0x30, 0x14, 0xfe, 0x08, 0xd6, 0xa8, 0x5a, 0x46, 0xe3, + 0x57, 0x38, 0x91, 0xd4, 0x6a, 0xa6, 0x83, 0xbc, 0x85, 0x56, 0x9b, 0x6b, 0x99, 0x11, 0x34, 0x96, + 0xb7, 0x08, 0x03, 0x9f, 0x1e, 0x2f, 0x36, 0x11, 0xf4, 0x6e, 0xe9, 0x40, 0x7d, 0x18, 0x98, 0xcf, + 0x50, 0x17, 0x7a, 0x49, 0x88, 0xbc, 0xa2, 0x32, 0xd1, 0x88, 0xd4, 0xff, 0x90, 0x2d, 0x93, 0x76, + 0xfd, 0x87, 0x7f, 0x93, 0x69, 0xef, 0x1e, 0x0a, 0xf5, 0xe2, 0xa1, 0xf0, 0xa4, 0xf3, 0xe7, 0x75, + 0xcf, 0xfb, 0xeb, 0xba, 0xe7, 0xfd, 0x73, 0xdd, 0xf3, 0x7e, 0xfb, 0xb7, 0xf7, 0xd6, 0x69, 0x93, + 0x1e, 0x85, 0x1f, 0xbf, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x9f, 0xcc, 0xe2, 0x4c, 0x25, 0x0a, 0x00, + 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 3173b12a2..68e9a5e5f 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -131,3 +131,9 @@ message Field { int64 Min = 3; int64 Max = 4; } + +message DeleteViewMessage { + string Index = 1; + string Frame = 2; + string View = 3; +} \ No newline at end of file diff --git a/internal/public.pb.go b/internal/public.pb.go index 43a9d7d35..19fd9e55f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,6 +11,7 @@ It has these top-level messages: Bitmap Pair + SumCount Bit ColumnAttrSet Attr @@ -66,6 +67,16 @@ func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } +type SumCount struct { + Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` +} + +func (m *SumCount) Reset() { *m = SumCount{} } +func (m *SumCount) String() string { return proto.CompactTextString(m) } +func (*SumCount) ProtoMessage() {} +func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } + type Bit struct { RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` @@ -75,7 +86,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` @@ -85,7 +96,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { @@ -106,7 +117,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } type AttrMap struct { Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` @@ -115,7 +126,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -125,16 +136,18 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` - ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"` + ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` @@ -145,7 +158,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { @@ -162,16 +175,17 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryResult) GetBitmap() *Bitmap { if m != nil { @@ -187,6 +201,13 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } +func (m *QueryResult) GetSumCount() *SumCount { + if m != nil { + return m.SumCount + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` @@ -199,11 +220,12 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*SumCount)(nil), "internal.SumCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") @@ -288,6 +310,34 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *SumCount) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SumCount) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Sum != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) + } + return i, nil +} + func (m *Bit) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -499,6 +549,26 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.ExcludeAttrs { + dAtA[i] = 0x30 + i++ + if m.ExcludeAttrs { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.ExcludeBits { + dAtA[i] = 0x38 + i++ + if m.ExcludeBits { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -602,6 +672,16 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.SumCount != nil { + dAtA[i] = 0x2a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size())) + n6, err := m.SumCount.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n6 + } return i, nil } @@ -638,56 +718,56 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) } if len(m.RowIDs) > 0 { - dAtA7 := make([]byte, len(m.RowIDs)*10) - var j6 int + dAtA8 := make([]byte, len(m.RowIDs)*10) + var j7 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) + dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j6++ + j7++ } - dAtA7[j6] = uint8(num) - j6++ + dAtA8[j7] = uint8(num) + j7++ } dAtA[i] = 0x22 i++ - i = encodeVarintPublic(dAtA, i, uint64(j6)) - i += copy(dAtA[i:], dAtA7[:j6]) + i = encodeVarintPublic(dAtA, i, uint64(j7)) + i += copy(dAtA[i:], dAtA8[:j7]) } if len(m.ColumnIDs) > 0 { - dAtA9 := make([]byte, len(m.ColumnIDs)*10) - var j8 int + dAtA10 := make([]byte, len(m.ColumnIDs)*10) + var j9 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) + dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j8++ + j9++ } - dAtA9[j8] = uint8(num) - j8++ + dAtA10[j9] = uint8(num) + j9++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j8)) - i += copy(dAtA[i:], dAtA9[:j8]) + i = encodeVarintPublic(dAtA, i, uint64(j9)) + i += copy(dAtA[i:], dAtA10[:j9]) } if len(m.Timestamps) > 0 { - dAtA11 := make([]byte, len(m.Timestamps)*10) - var j10 int + dAtA12 := make([]byte, len(m.Timestamps)*10) + var j11 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j10++ + j11++ } - dAtA11[j10] = uint8(num) - j10++ + dAtA12[j11] = uint8(num) + j11++ } dAtA[i] = 0x32 i++ - i = encodeVarintPublic(dAtA, i, uint64(j10)) - i += copy(dAtA[i:], dAtA11[:j10]) + i = encodeVarintPublic(dAtA, i, uint64(j11)) + i += copy(dAtA[i:], dAtA12[:j11]) } return i, nil } @@ -750,6 +830,18 @@ func (m *Pair) Size() (n int) { return n } +func (m *SumCount) Size() (n int) { + var l int + _ = l + if m.Sum != 0 { + n += 1 + sovPublic(uint64(m.Sum)) + } + if m.Count != 0 { + n += 1 + sovPublic(uint64(m.Count)) + } + return n +} + func (m *Bit) Size() (n int) { var l int _ = l @@ -838,6 +930,12 @@ func (m *QueryRequest) Size() (n int) { if m.Remote { n += 2 } + if m.ExcludeAttrs { + n += 2 + } + if m.ExcludeBits { + n += 2 + } return n } @@ -882,6 +980,10 @@ func (m *QueryResult) Size() (n int) { if m.Changed { n += 2 } + if m.SumCount != nil { + l = m.SumCount.Size() + n += 1 + l + sovPublic(uint64(l)) + } return n } @@ -1167,6 +1269,94 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *SumCount) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SumCount: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + } + m.Sum = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Sum |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Bit) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1799,6 +1989,46 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } } m.Remote = bool(v != 0) + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ExcludeAttrs = bool(v != 0) + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ExcludeBits = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -2093,6 +2323,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } } m.Changed = bool(v != 0) + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SumCount == nil { + m.SumCount = &SumCount{} + } + if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -2535,41 +2798,44 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 563 bytes of a gzipped FileDescriptorProto + // 621 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40, - 0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57, - 0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xe8, 0x0c, 0xec, 0x3d, 0x33, 0xad, - 0xc1, 0x92, 0x7f, 0x74, 0xb7, 0x81, 0x1c, 0x80, 0x13, 0xb0, 0xe1, 0x06, 0x70, 0x14, 0x96, 0x1c, - 0x01, 0x85, 0x8b, 0xa0, 0xea, 0x76, 0xc7, 0x1e, 0x16, 0x68, 0x76, 0xfd, 0x5e, 0x75, 0xb5, 0xeb, - 0xd5, 0xab, 0x32, 0x4c, 0xab, 0xfa, 0x32, 0x4b, 0xaf, 0x56, 0x95, 0x28, 0x55, 0x49, 0x47, 0x69, - 0xa1, 0xb8, 0x28, 0x92, 0x2c, 0x88, 0x60, 0x10, 0xa5, 0x2a, 0x4f, 0x2a, 0x4a, 0xc1, 0x8d, 0x52, - 0x25, 0x3d, 0xe2, 0x3b, 0xa1, 0xcb, 0xf4, 0x99, 0x3e, 0x85, 0xfe, 0x0b, 0xa5, 0x84, 0xf4, 0x7a, - 0xbe, 0x13, 0x4e, 0x4e, 0xe7, 0x2b, 0x9b, 0xb7, 0x42, 0x9a, 0x99, 0x60, 0xb0, 0x02, 0xf7, 0x75, - 0x92, 0x0a, 0xba, 0x00, 0xe7, 0x25, 0xdf, 0x7b, 0xc4, 0x27, 0xa1, 0xcb, 0xf0, 0x48, 0xef, 0x43, - 0x7f, 0x5d, 0xd6, 0x85, 0xf2, 0x7a, 0x9a, 0x33, 0x20, 0x78, 0x0b, 0x4e, 0x94, 0x2a, 0x0c, 0xb2, - 0xf2, 0x53, 0xbc, 0x69, 0x12, 0x0c, 0xa0, 0x8f, 0x61, 0xb4, 0x2e, 0xb3, 0x3a, 0x2f, 0xe2, 0x4d, - 0x93, 0x75, 0xc4, 0xf4, 0x09, 0x8c, 0x2f, 0xd2, 0x9c, 0x4b, 0x95, 0xe4, 0x95, 0xe7, 0xf8, 0x24, - 0x74, 0x58, 0x4b, 0x04, 0x5b, 0x98, 0x99, 0x9b, 0x58, 0xd5, 0x8e, 0x2b, 0x3a, 0x87, 0xde, 0xf1, - 0xf5, 0x5e, 0xbc, 0xb9, 0xa3, 0x9a, 0x1f, 0x04, 0x5c, 0x3c, 0x75, 0xe5, 0x8c, 0x8d, 0x1c, 0x0a, - 0xee, 0xc5, 0xbe, 0xe2, 0x4d, 0x5d, 0xfa, 0x4c, 0x7d, 0x98, 0xec, 0x94, 0x48, 0x8b, 0x9b, 0x77, - 0x49, 0x56, 0x73, 0x5d, 0xd5, 0x98, 0x75, 0x29, 0x54, 0x14, 0x17, 0xca, 0x84, 0x5d, 0x5d, 0xf4, - 0x11, 0xa3, 0xa2, 0xa8, 0x2c, 0x33, 0x13, 0xec, 0xfb, 0x24, 0x1c, 0xb1, 0x96, 0xa0, 0x4b, 0x80, - 0xb3, 0xac, 0x4c, 0x9a, 0xdc, 0x81, 0x4f, 0x42, 0xc2, 0x3a, 0x4c, 0x70, 0x02, 0x43, 0xac, 0xf4, - 0x55, 0x52, 0xb5, 0xda, 0xc8, 0xff, 0xb4, 0x7d, 0x84, 0xe9, 0x9b, 0x9a, 0x8b, 0x3d, 0xe3, 0x1f, - 0x6a, 0x2e, 0xb5, 0x05, 0x1a, 0x37, 0x22, 0x0d, 0xa0, 0x0f, 0x61, 0xb0, 0xcb, 0xd2, 0x2b, 0x6e, - 0x1a, 0xe5, 0xb2, 0x06, 0xa1, 0xd4, 0xb6, 0xc1, 0x52, 0x4b, 0x1d, 0xb1, 0x2e, 0x85, 0x99, 0x8c, - 0xe7, 0xa5, 0xb2, 0x5a, 0x1a, 0x14, 0x7c, 0x25, 0x30, 0x6b, 0x3e, 0x2c, 0xab, 0xb2, 0x90, 0x1c, - 0x9b, 0xbb, 0x15, 0xc2, 0x36, 0x77, 0x2b, 0x04, 0x3d, 0x81, 0x21, 0xe3, 0xb2, 0xce, 0x94, 0xf5, - 0xe7, 0x41, 0xab, 0xc1, 0xe6, 0xd6, 0x99, 0x62, 0xf6, 0x16, 0x7d, 0x0e, 0xf3, 0x5b, 0x7e, 0x63, - 0x45, 0x98, 0xf7, 0xa8, 0xcd, 0xbb, 0x15, 0x67, 0xff, 0x5c, 0x0f, 0xbe, 0x10, 0x98, 0x74, 0x5e, - 0xa6, 0xa1, 0xdd, 0x05, 0x5d, 0xd6, 0xe4, 0x74, 0xd1, 0x3e, 0x64, 0x78, 0x66, 0x77, 0x65, 0x0a, - 0xe4, 0xbc, 0x99, 0x02, 0x72, 0x8e, 0xbd, 0xc7, 0xf9, 0xb7, 0xdf, 0xef, 0xf4, 0x1e, 0x69, 0x66, - 0x82, 0xd4, 0x83, 0xe1, 0xfa, 0x7d, 0x52, 0xdc, 0xf0, 0x6b, 0x3d, 0x05, 0x23, 0x66, 0x61, 0xf0, - 0x9d, 0xc0, 0x2c, 0xce, 0xab, 0x52, 0xa8, 0x8e, 0x2f, 0x71, 0x71, 0xcd, 0x3f, 0x5b, 0x5f, 0x34, - 0x40, 0xf6, 0x4c, 0x24, 0xb9, 0x99, 0xbf, 0x31, 0x33, 0x00, 0x59, 0xed, 0x8f, 0xf6, 0xc3, 0x65, - 0x06, 0x68, 0x27, 0x70, 0x9f, 0xa4, 0xe7, 0x1a, 0x0f, 0x0d, 0xc2, 0x81, 0xb3, 0xeb, 0x24, 0xbd, - 0xbe, 0x0e, 0xb5, 0x04, 0x0e, 0xdc, 0x71, 0x9f, 0xa4, 0x37, 0xf0, 0x9d, 0xd0, 0x61, 0x1d, 0x26, - 0x5a, 0xfc, 0x3c, 0x2c, 0xc9, 0xaf, 0xc3, 0x92, 0xfc, 0x3e, 0x2c, 0xc9, 0xb7, 0x3f, 0xcb, 0x7b, - 0x97, 0x03, 0xfd, 0x43, 0x79, 0xf6, 0x37, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x6d, 0x5d, 0xc9, 0x60, - 0x04, 0x00, 0x00, + 0x10, 0xa5, 0x63, 0x27, 0x71, 0x2a, 0x99, 0x51, 0xd4, 0xe2, 0x63, 0x21, 0x14, 0x59, 0x16, 0x0b, + 0xaf, 0x32, 0x52, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xe8, 0x0c, 0xec, 0x3d, 0x49, + 0x6b, 0xb0, 0xe4, 0x1f, 0xed, 0xb6, 0x98, 0x9c, 0x83, 0x0d, 0x37, 0x80, 0x0b, 0xb0, 0xe3, 0x00, + 0x2c, 0x39, 0x02, 0x0a, 0x17, 0x41, 0xd5, 0xed, 0x8e, 0x1d, 0x16, 0x88, 0x5d, 0xbd, 0x57, 0x5d, + 0xe5, 0xfa, 0xbc, 0x32, 0x4c, 0xca, 0xfa, 0x26, 0x4d, 0xb6, 0xf3, 0x52, 0x14, 0xb2, 0xa0, 0x4e, + 0x92, 0x4b, 0x2e, 0xf2, 0x38, 0xf5, 0x43, 0x18, 0x84, 0x89, 0xcc, 0xe2, 0x92, 0x52, 0xb0, 0xc3, + 0x44, 0x56, 0x2e, 0xf1, 0xac, 0xc0, 0x66, 0xca, 0xa6, 0x4f, 0xa1, 0xff, 0x42, 0x4a, 0x51, 0xb9, + 0x3d, 0xcf, 0x0a, 0xc6, 0x8b, 0xf3, 0xb9, 0x89, 0x9b, 0x23, 0xcd, 0xb4, 0xd3, 0x9f, 0x83, 0xfd, + 0x3a, 0x4e, 0x04, 0x9d, 0x82, 0xf5, 0x92, 0xef, 0x5d, 0xe2, 0x91, 0xc0, 0x66, 0x68, 0xd2, 0xfb, + 0xd0, 0x5f, 0x16, 0x75, 0x2e, 0xdd, 0x9e, 0xe2, 0x34, 0xf0, 0x17, 0xe0, 0x6c, 0xea, 0x4c, 0xd9, + 0x18, 0xb3, 0xa9, 0x33, 0x15, 0x63, 0x31, 0x34, 0x4f, 0x63, 0x2c, 0x13, 0xf3, 0x16, 0xac, 0x30, + 0x91, 0xe8, 0x64, 0xc5, 0xc7, 0x68, 0xd5, 0x7c, 0x44, 0x03, 0xfa, 0x18, 0x9c, 0x65, 0x91, 0xd6, + 0x59, 0x1e, 0xad, 0x9a, 0x2f, 0x1d, 0x31, 0x7d, 0x02, 0xa3, 0xeb, 0x24, 0xe3, 0x95, 0x8c, 0xb3, + 0xd2, 0xb5, 0x54, 0xca, 0x96, 0xf0, 0xd7, 0x70, 0xa6, 0x5f, 0x62, 0x27, 0x1b, 0x2e, 0xe9, 0x39, + 0xf4, 0x8e, 0xd9, 0x7b, 0xd1, 0xea, 0x3f, 0x27, 0xf0, 0x95, 0x80, 0x8d, 0x56, 0x77, 0x04, 0x23, + 0x3d, 0x02, 0x0a, 0xf6, 0xf5, 0xbe, 0xe4, 0x4d, 0x5d, 0xca, 0xa6, 0x1e, 0x8c, 0x37, 0x52, 0x24, + 0xf9, 0xed, 0xbb, 0x38, 0xad, 0xb9, 0xaa, 0x6a, 0xc4, 0xba, 0x14, 0x76, 0x14, 0xe5, 0x52, 0xbb, + 0x6d, 0x55, 0xf4, 0x11, 0x63, 0x47, 0x61, 0x51, 0xa4, 0xda, 0xd9, 0xf7, 0x48, 0xe0, 0xb0, 0x96, + 0xa0, 0x33, 0x80, 0xcb, 0xb4, 0x88, 0x9b, 0xd8, 0x81, 0x47, 0x02, 0xc2, 0x3a, 0x8c, 0x7f, 0x01, + 0x43, 0xac, 0xf4, 0x55, 0x5c, 0xb6, 0xbd, 0x91, 0x7f, 0xf5, 0xf6, 0x9d, 0xc0, 0xe4, 0x4d, 0xcd, + 0xc5, 0x9e, 0xf1, 0x0f, 0x35, 0xaf, 0xd4, 0x0e, 0x14, 0x6e, 0xba, 0xd4, 0x80, 0x3e, 0x84, 0xc1, + 0x26, 0x4d, 0xb6, 0x5c, 0x4f, 0xca, 0x66, 0x0d, 0xc2, 0x5e, 0xdb, 0x09, 0x57, 0xaa, 0x57, 0x87, + 0x75, 0x29, 0x8c, 0x64, 0x3c, 0x2b, 0xa4, 0x69, 0xa6, 0x41, 0xd4, 0x87, 0xc9, 0xfa, 0x6e, 0x9b, + 0xd6, 0x3b, 0xae, 0x43, 0x07, 0xca, 0x7b, 0xc2, 0x61, 0xf6, 0x06, 0x2b, 0xed, 0x0e, 0x75, 0xf6, + 0x0e, 0xe5, 0x7f, 0x22, 0x70, 0xd6, 0x94, 0x5f, 0x95, 0x45, 0x5e, 0x71, 0xdc, 0xd1, 0x5a, 0x08, + 0xb3, 0xa3, 0xb5, 0x10, 0xf4, 0x02, 0x86, 0x8c, 0x57, 0x75, 0x2a, 0xcd, 0x9a, 0x1f, 0xb4, 0xa3, + 0x30, 0xb1, 0x75, 0x2a, 0x99, 0x79, 0x45, 0x9f, 0xc3, 0xf9, 0x89, 0x6c, 0xb0, 0x2f, 0x8c, 0x7b, + 0xd4, 0xc6, 0x9d, 0xf8, 0xd9, 0x5f, 0xcf, 0xfd, 0x6f, 0x04, 0xc6, 0x9d, 0xcc, 0x34, 0x30, 0x67, + 0xa8, 0xca, 0x1a, 0x2f, 0xa6, 0x6d, 0x22, 0xcd, 0x33, 0x73, 0xa6, 0x13, 0x20, 0x57, 0x8d, 0x98, + 0xc8, 0x15, 0xae, 0x10, 0x4f, 0xcf, 0x7c, 0xbf, 0xb3, 0x42, 0xa4, 0x99, 0x76, 0x52, 0x17, 0x86, + 0xcb, 0xf7, 0x71, 0x7e, 0xcb, 0x77, 0x4a, 0x4c, 0x0e, 0x33, 0x90, 0xce, 0xdb, 0x53, 0x54, 0xd3, + 0x1f, 0x2f, 0x68, 0x9b, 0xc2, 0x78, 0xd8, 0xf1, 0x8d, 0xff, 0x85, 0xc0, 0x59, 0x94, 0x95, 0x85, + 0x90, 0x1d, 0x35, 0x44, 0xf9, 0x8e, 0xdf, 0x19, 0x35, 0x28, 0x80, 0xec, 0xa5, 0x88, 0x33, 0x2d, + 0xfb, 0x11, 0xd3, 0x00, 0x59, 0xa5, 0x0a, 0xa5, 0x02, 0x9b, 0x69, 0xa0, 0xf6, 0x8f, 0x67, 0x5c, + 0xb9, 0xb6, 0x56, 0x8e, 0x46, 0xa8, 0x73, 0x73, 0xc5, 0x95, 0xdb, 0x57, 0xae, 0x96, 0x40, 0x9d, + 0x1f, 0xcf, 0x18, 0xb5, 0x61, 0x05, 0x16, 0xeb, 0x30, 0xe1, 0xf4, 0xc7, 0x61, 0x46, 0x7e, 0x1e, + 0x66, 0xe4, 0xd7, 0x61, 0x46, 0x3e, 0xff, 0x9e, 0xdd, 0xbb, 0x19, 0xa8, 0x7f, 0xdf, 0xb3, 0x3f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x27, 0x5d, 0xef, 0xb2, 0x0b, 0x05, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index ff161d930..88bfefb00 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -12,6 +12,11 @@ message Pair { uint64 Count = 2; } +message SumCount { + int64 Sum = 1; + int64 Count = 2; +} + message Bit { uint64 RowID = 1; uint64 ColumnID = 2; @@ -41,6 +46,8 @@ message QueryRequest { repeated uint64 Slices = 2; bool ColumnAttrs = 3; bool Remote = 5; + bool ExcludeAttrs = 6; + bool ExcludeBits = 7; } message QueryResponse { @@ -53,6 +60,7 @@ message QueryResult { Bitmap Bitmap = 1; uint64 N = 2; repeated Pair Pairs = 3; + SumCount SumCount = 5; bool Changed = 4; } diff --git a/pql/ast.go b/pql/ast.go index 9555f9d90..48e5f1771 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -198,6 +198,16 @@ func (c *Call) IsInverse(rowLabel, columnLabel string) bool { return false } +// HasConditionArg returns true if any arg is a conditional. +func (c *Call) HasConditionArg() bool { + for _, v := range c.Args { + if _, ok := v.(*Condition); ok { + return true + } + } + return false +} + // Condition represents an operation & value. // When used in an argument map it represents a binary expression. type Condition struct { diff --git a/roaring/roaring.go b/roaring/roaring.go index cfcda97d1..77dfd8803 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -197,6 +197,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { if len(b.keys) == 0 { return } + skey := highbits(start) ekey := highbits(end) @@ -208,31 +209,28 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end)))) } - // Count first partial container. if i < 0 { - // start is before container, so we should start counting - // at first container that has value - if skey < b.keys[0] { - i = -1 - } else { - i = -i - } + // start's container did not exist + // set i to the index of the first container we have with values higher than start + i = -i - 1 } else { + // Count first partial container and advance i so we don't recount it n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1)) + i += 1 } // Count last container. if j < 0 { - j = -j - if j > len(b.containers) { - j = len(b.containers) - } + // end's container did not exist + // set j to the index of the first container with values higher than end (or len(containers)) + j = -j - 1 } else { + // end's container exists, count it up to end n += uint64(b.containers[j].countRange(0, int(lowbits(end)))) } // Count containers in between. - for x := i + 1; x < j; x++ { + for x := i; x < j; x++ { n += uint64(b.containers[x].n) } @@ -1994,6 +1992,10 @@ func intersectBitmapRun(a, b *container) *container { if a.bitmapContains(i) { output.array = append(output.array, i) } + // If the run ends the container, break to avoid an infinite loop. + if i == 65535 { + break + } } } output.n = len(output.array) @@ -2506,7 +2508,7 @@ func differenceRunBitmap(a, b *container) *container { func differenceRunIterator(a *container, itr containerIterator) *container { - output := &container{runs: make([]interval16, 0, a.n)} + output := &container{runs: make([]interval16, 0, a.n), container_type: ContainerRun} vb, eof := itr.next() j := 0 @@ -2574,7 +2576,7 @@ func differenceRunRun(a, b *container) *container { alen := len(a.runs) blen := len(b.runs) - output := &container{runs: make([]interval16, 0, alen+blen)} // TODO allocate max then truncate? or something else + output := &container{runs: make([]interval16, 0, alen+blen), container_type: ContainerRun} // TODO allocate max then truncate? or something else // cardinality upper bound: sum of number of runs // each B-run could split an A-run in two, up to len(b.runs) times @@ -2619,6 +2621,7 @@ func differenceRunRun(a, b *container) *container { } } + output.n = output.count() return output } @@ -2867,7 +2870,8 @@ func (*op) size() int { return 1 + 8 + 4 } func highbits(v uint64) uint64 { return uint64(v >> 16) } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } -// search32 returns the index of v in a. +// search32 returns the index of value in a. If value is not found, it works the +// same way as search64. func search32(a []uint16, value uint16) int { // Optimize for elements and the last element. n := len(a) @@ -2904,7 +2908,13 @@ func search32(a []uint16, value uint16) int { return -(lo + 1) } -// search64 returns the index of v in a. +// search64 returns the index of value in a. If value is not found, -1 * (1 + +// the index where v would be if it were inserted) is returned. This is done in +// order to both signal that value was not found (negative number), and also +// return information about where v would go if it were inserted. The +1 offset +// is necessary due to the case where v is not found, but would go at index 0. +// since negative 0 is no different from positive 0, we offset the returned +// negative indices by 1. See the test for this function for examples. func search64(a []uint64, value uint64) int { // Optimize for elements and the last element. n := len(a) @@ -3147,8 +3157,9 @@ func xorArrayRun(a, b *container) *container { } else if va > vb.start { if va < vb.last { output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1}) - vb.start = va + 1 i++ + vb.start = va + 1 + if vb.start > vb.last { j++ } @@ -3157,15 +3168,22 @@ func xorArrayRun(a, b *container) *container { j++ } else { // va == vb.last vb.last-- - if vb.start < vb.last { + if vb.start <= vb.last { output.n += output.runAppendInterval(vb) } j++ i++ } - } else { - vb.start++ + } else { // we know va == vb.start + if vb.start == maxContainerVal { // protect overflow + j++ + } else { + vb.start++ + if vb.start > vb.last { + j++ + } + } i++ } } @@ -3213,9 +3231,15 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) { r1 = interval16{start: x.va.start, last: x.vb.start - 1} has_data = true } - x.va.start = x.vb.last + 1 - if x.va.start > x.va.last { + + if x.vb.last == maxContainerVal { // Check for overflow x.va_valid = false + + } else { + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } } } else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside @@ -3225,26 +3249,39 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) { has_data = true } - x.vb.start = x.va.last + 1 - if x.vb.start > x.vb.last { + if x.va.last == maxContainerVal { //check for overflow x.vb_valid = false + } else { + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } } } else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap x.va_valid = false r1 = interval16{start: x.va.start, last: x.vb.start - 1} has_data = true - x.vb.start = x.va.last + 1 - if x.vb.start > x.vb.last { + if x.va.last == maxContainerVal { // check for overflow x.vb_valid = false + } else { + x.vb.start = x.va.last + 1 + if x.vb.start > x.vb.last { + x.vb_valid = false + } } } else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap x.vb_valid = false r1 = interval16{start: x.vb.start, last: x.va.start - 1} has_data = true - x.va.start = x.vb.last + 1 - if x.va.start > x.va.last { + + if x.vb.last == maxContainerVal { // check for overflow x.va_valid = false + } else { + x.va.start = x.vb.last + 1 + if x.va.start > x.va.last { + x.va_valid = false + } } } return diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index ea4e5f0ef..5d1a985d1 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -623,7 +623,7 @@ func TestIntersectBitmapRunArray(t *testing.T) { }, { bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1}, - runs: []interval16{{start: 63, last: 10000}}, + runs: []interval16{{start: 63, last: 10000}, {start: 65000, last: 65535}}, exp: []uint16{63, 64, 128, 192, 257, 259, 320, 384, 512}, expN: 9, }, @@ -1601,6 +1601,7 @@ func TestDifferenceRunRun(t *testing.T) { aruns []interval16 bruns []interval16 exp []interval16 + expn int }{ { // this tests all six overlap combinations @@ -1609,6 +1610,7 @@ func TestDifferenceRunRun(t *testing.T) { aruns: []interval16{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}}, bruns: []interval16{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}}, exp: []interval16{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}}, + expn: 13, }, } for i, test := range tests { @@ -1620,6 +1622,9 @@ func TestDifferenceRunRun(t *testing.T) { if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) } + if ret.n != test.expn { + t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.n) + } } } @@ -1711,31 +1716,43 @@ func TestWriteReadRun(t *testing.T) { } func TestXorArrayRun(t *testing.T) { - a := &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray} - b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun} - exp := []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16} - - //ret := xorArrayRun(a, b) - ret := xor(a, b) - if !reflect.DeepEqual(ret.array, exp) { - t.Fatalf("test #1 expected %v, but got %v", exp, ret.array) + tests := []struct { + a *container + b *container + exp *container + }{ + { + a: &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray}, + b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}, + exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, container_type: ContainerArray, n: 12}, + }, { + a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}, + b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}, + exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, container_type: ContainerArray, n: 12}, + }, { + a: &container{array: []uint16{65535}, container_type: ContainerArray}, + b: &container{runs: []interval16{{start: 65534, last: 65535}}, container_type: ContainerRun}, + exp: &container{array: []uint16{65534}, container_type: ContainerArray, n: 1}, + }, { + a: &container{array: []uint16{65535}, container_type: ContainerArray}, + b: &container{runs: []interval16{{start: 65535, last: 65535}}, container_type: ContainerRun}, + exp: &container{array: []uint16{}, container_type: ContainerArray, n: 0}, + }, } - ret = xor(b, a) - if !reflect.DeepEqual(ret.array, exp) { - t.Fatalf("test #2 expected %v, but got %v", exp, ret.array) - } - c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray} - // exp = []int16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16} - expr := []interval16{{start: 1, last: 4}, {start: 6, last: 9}, {start: 11, last: 11}, {start: 14, last: 16}} - ret = xor(b, c) - if !reflect.DeepEqual(ret.runs, expr) { - t.Fatalf("test #3 expected %v, but got %v", exp, ret.runs) - } - ret = xor(c, b) - if !reflect.DeepEqual(ret.runs, expr) { - t.Fatalf("test #4 expected %v, but got %v", exp, ret.array) + for i, test := range tests { + test.a.n = test.a.count() + test.b.n = test.b.count() + ret := xor(test.a, test.b) + if !reflect.DeepEqual(ret, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret) + } + ret = xor(test.b, test.a) + if !reflect.DeepEqual(ret, test.exp) { + t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret) + } } + } //special case that didn't fit the xorrunrun table testing below. @@ -1832,6 +1849,11 @@ func TestXorRunRun(t *testing.T) { bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}}, exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}}, }, + { + aruns: []interval16{{start: 65530, last: 65535}}, + bruns: []interval16{{start: 65532, last: 65535}}, + exp: []interval16{{start: 65530, last: 65531}}, + }, } for i, test := range tests { a.runs = test.aruns @@ -2310,3 +2332,81 @@ func Test_BufBitmapIterator_UnreadPanic(t *testing.T) { itr.unread() itr.unread() } + +func TestSearc64(t *testing.T) { + tests := []struct { + a []uint64 + value uint64 + exp int + }{ + { + a: []uint64{1, 5, 10, 12}, + value: 5, + exp: 1, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 1, + exp: 0, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 0, + exp: -1, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 2, + exp: -2, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 7, + exp: -3, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 11, + exp: -4, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 13, + exp: -5, + }, + { + a: []uint64{1, 5, 10, 12}, + value: 3843534, + exp: -5, + }, + { + a: []uint64{}, + value: 3843534, + exp: -1, + }, + { + a: []uint64{}, + value: 0, + exp: -1, + }, + { + a: []uint64{0}, + value: 0, + exp: 0, + }, + { + a: []uint64{0}, + value: 1, + exp: -2, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%d in %v", test.value, test.a), func(t *testing.T) { + actual := search64(test.a, test.value) + if actual != test.exp { + t.Errorf("got: %d, exp: %d", actual, test.exp) + } + }) + } +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index c721f3804..7e524f51f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -54,6 +54,104 @@ func TestContainerCount(t *testing.T) { } } +func TestCountRange(t *testing.T) { + tests := []struct { + name string + bitmap []uint64 + start uint64 + end uint64 + exp uint64 + }{ + { + name: "j < 0 : 1", + bitmap: []uint64{0, 1, 2, 3 * 65536}, + start: 0, + end: 65536, + exp: 3, + }, + { + name: "i < 0 : 1", + bitmap: []uint64{0, 1, 2, 2 * 65536, 3 * 65536}, + start: 65536, + end: 3 * 65536, + exp: 1, + }, + { + name: "single-container-run", + bitmap: []uint64{0, 2, 3, 4, 5, 2 * 65536, 3 * 65536}, + start: 2, + end: 5, + exp: 3, + }, + { + name: "single-container-beg", + bitmap: []uint64{1, 2, 3, 4, 5, 2 * 65536, 3 * 65536}, + start: 1, + end: 4, + exp: 3, + }, + { + name: "partial-start", + bitmap: []uint64{1, 2, 3, 4, 5, 2 * 65536, 3 * 65536}, + start: 5, + end: 3 * 65536, + exp: 2, + }, + { + name: "partial-end", + bitmap: []uint64{1, 2 * 65536, 3 * 65536, 3*65536 + 1, 3*65536 + 2}, + start: 0, + end: (3 * 65536) + 1, + exp: 3, + }, + { + name: "partial-both", + bitmap: []uint64{65536, 65537, 65538, 2 * 65536, 2*65536 + 1, 2*65536 + 2}, + start: 65537, + end: (2 * 65536) + 1, + exp: 3, + }, + { + name: "partial-both-bookends", + bitmap: []uint64{0, 65535, 65536, 65537, 65538, 2 * 65536, 2*65536 + 1, 2*65536 + 2, 3 * 65536}, + start: 65537, + end: (2 * 65536) + 1, + exp: 3, + }, + { + name: "empty-bookends", + bitmap: []uint64{1, 65535, 5 * 65536, 5*65536 + 1}, + start: 65536, + end: 5 * 65536, + exp: 0, + }, + { + name: "i not found, j found", + bitmap: []uint64{1, 65535, 5 * 65536}, + start: 2 * 65535, + end: 5*65536 + 1, + exp: 1, + }, + { + name: "i not found, j not found", + bitmap: []uint64{1, 65535, 5 * 65536, 7 * 65536}, + start: 2 * 65535, + end: 6 * 65536, + exp: 1, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) { + b := roaring.NewBitmap(test.bitmap...) + actual := b.CountRange(test.start, test.end) + if actual != test.exp { + t.Errorf("got: %d, exp: %d", actual, test.exp) + } + }) + } +} + func TestCheckBitmap(t *testing.T) { b := roaring.NewBitmap() x := 0 diff --git a/server.go b/server.go index 97883fae4..8194278ba 100644 --- a/server.go +++ b/server.go @@ -339,6 +339,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err != nil { return err } + case *internal.DeleteViewMessage: + f := s.Holder.Frame(obj.Index, obj.Frame) + if f == nil { + return fmt.Errorf("Local Frame not found: %s", obj.Frame) + } + err := f.DeleteView(obj.View) + if err != nil { + return err + } } return nil } diff --git a/test/holder.go b/test/holder.go index 6bb346ec1..4cd0642e4 100644 --- a/test/holder.go +++ b/test/holder.go @@ -43,12 +43,9 @@ func (h *Holder) Close() error { return h.Holder.Close() } -// Reopen closes the holder and instantiates and opens a new holder. +// Reopen instantiates and opens a new holder. +// Note that the holder must be Closed first. func (h *Holder) Reopen() error { - if err := h.Holder.Close(); err != nil { - return err - } - path, logOutput := h.Path, h.Holder.LogOutput h.Holder = pilosa.NewHolder() h.Holder.Path = path diff --git a/view.go b/view.go index 1aa29bbb4..2d5b7d776 100644 --- a/view.go +++ b/view.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" ) // View layout modes. @@ -162,7 +163,9 @@ func (v *View) Close() error { // Close all fragments. for _, frag := range v.fragments { - _ = frag.Close() + if err := frag.Close(); err != nil { + return err + } } v.fragments = make(map[uint64]*Fragment) @@ -300,8 +303,21 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan return frag.SetFieldValue(columnID, bitDepth, value) } +// FieldSum returns the sum & count of a field. +func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) { + for _, f := range v.Fragments() { + fsum, fcount, err := f.FieldSum(filter, bitDepth) + if err != nil { + return sum, count, err + } + sum += fsum + count += fcount + } + return sum, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. -func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { +func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap() for _, frag := range v.Fragments() { other, err := frag.FieldRange(op, bitDepth, predicate)