Merge branch 'master' of github.com:pilosa/pilosa into rowID-zero-bug

This commit is contained in:
Linh Vo 2017-09-05 08:46:47 -05:00
commit 7562d0fb15
49 changed files with 2087 additions and 286 deletions

View file

@ -1,7 +1,7 @@
language: go
go:
- 1.7
- 1.8
- 1.9
- master
addons:
before_install:

2
Gopkg.lock generated
View file

@ -194,6 +194,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "84ff0992f3a6023a9d4832d6aea43d6aec19878a4b9e991ba8ff8269b589e816"
inputs-digest = "e8e78a7c61547d8f4d967c8deac9334b3151e7c10c4f7909e80758956e9c8204"
solver-name = "gps-cdcl"
solver-version = 1

View file

@ -97,6 +97,26 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
return &Bitmap{segments: segments}
}
// Xor returns the xor of b and other.
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
var segments []BitmapSegment
itr := newMergeSegmentIterator(b.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
segments = append(segments, *s0)
continue
} else if s0 == nil {
segments = append(segments, *s1)
continue
}
segments = append(segments, *s0.Xor(s1))
}
return &Bitmap{segments: segments}
}
// Union returns the bitwise union of b and other.
func (b *Bitmap) Union(other *Bitmap) *Bitmap {
var segments []BitmapSegment
@ -342,6 +362,17 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment {
}
}
// Xor returns the xor of s and other.
func (s *BitmapSegment) Xor(other *BitmapSegment) *BitmapSegment {
data := s.data.Xor(&other.data)
return &BitmapSegment{
data: *data,
slice: s.slice,
n: data.Count(),
}
}
// SetBit sets the i-th bit of the bitmap.
func (s *BitmapSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()

92
bitmap_test.go Normal file
View file

@ -0,0 +1,92 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"reflect"
"testing"
"github.com/pilosa/pilosa"
)
// Ensure a bitmap can be merged
func TestBitmap_Merge(t *testing.T) {
bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth)
bm2 := pilosa.NewBitmap(3, 4, 5)
bm1.Merge(bm2)
if bm1.Count() != 7 {
t.Fatalf("Count after merge %d != 7\n", bm1.Count())
}
}
// Ensure a bitmap can Xor'ed
func TestBitmap_Xor(t *testing.T) {
bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
exp := []uint64{1, SliceWidth, 2 * SliceWidth}
res := bm1.Xor(bm2)
if res.Count() != 3 {
t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
}
res = bm2.Xor(bm1)
if res.Count() != 3 {
t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
}
}
func TestBitmap_Union_Segment(t *testing.T) {
bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth}
res := bm1.Union(bm2)
if res.Count() != 4 {
t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
}
res = bm2.Union(bm1)
if res.Count() != 4 {
t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
}
}
func TestBitmap_Difference_Segment(t *testing.T) {
bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
exp := []uint64{1, SliceWidth}
res := bm1.Difference(bm2)
if res.Count() != 2 {
t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
}
}

View file

@ -115,6 +115,7 @@ const (
MessageTypeDeleteFrame = 5
MessageTypeCreateInputDefinition = 6
MessageTypeDeleteInputDefinition = 7
MessageTypeDeleteView = 8
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -135,6 +136,8 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateInputDefinition
case *internal.DeleteInputDefinitionMessage:
typ = MessageTypeDeleteInputDefinition
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -165,6 +168,8 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateInputDefinitionMessage{}
case MessageTypeDeleteInputDefinition:
m = &internal.DeleteInputDefinitionMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)
}

View file

@ -256,8 +256,10 @@ func (c *RankCache) recalculate() {
length := len(c.rankings)
c.stats.Gauge("RankCache", float64(length), 1.0)
var removeItems []BitmapPair // cached, ordered list
if length > int(c.maxEntries) {
c.thresholdValue = rankings[c.maxEntries].Count
removeItems = c.rankings[c.maxEntries:]
c.rankings = c.rankings[0:c.maxEntries]
} else {
c.thresholdValue = 1
@ -269,10 +271,8 @@ func (c *RankCache) recalculate() {
// If size is larger than the threshold then trim it.
if len(c.entries) > c.thresholdBuffer {
c.stats.Count("cache.threshold", 1, 1.0)
for id, cnt := range c.entries {
if cnt <= c.thresholdValue {
delete(c.entries, id)
}
for _, pair := range removeItems {
delete(c.entries, pair.ID)
}
}
}

35
cache_test.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"testing"
"github.com/pilosa/pilosa"
)
// Ensure a bitmap query can be executed.
func TestCache_Rank(t *testing.T) {
cacheSize := uint32(3)
cache := pilosa.NewRankCache(cacheSize)
for i := 1; i < int(2*cacheSize); i++ {
cache.Add(uint64(i), 3)
}
cache.Recalculate()
if cache.Len() != int(cacheSize) {
t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
}
}

View file

@ -16,13 +16,13 @@ package ctl
import (
"bytes"
"context"
"net/http"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestExportCommand_Validation(t *testing.T) {

View file

@ -16,6 +16,7 @@ package ctl
import (
"bytes"
"context"
"io"
"io/ioutil"
"net/http"
@ -24,7 +25,6 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestImportCommand_Validation(t *testing.T) {

View file

@ -17,13 +17,13 @@ package ctl
import (
"bufio"
"bytes"
"context"
"io"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestRestoreCommand_FileRequired(t *testing.T) {

View file

@ -1,5 +1,12 @@
+++
title = "Administration Guide"
title = "Administration"
weight = 13
nav = [
"Installing in production",
"Imports and Exports",
"Versioning",
"Backup/restore",
]
+++
## Administration Guide
@ -164,4 +171,4 @@ We currently track the following events
<strong id="goroutines">Goroutines:</strong> Number of running Goroutines.
<strong id="openfiles">OpenFiles:</strong> Number of open file handles associated with running Pilosa process ID.
<strong id="openfiles">OpenFiles:</strong> Number of open file handles associated with running Pilosa process ID.

View file

@ -1,5 +1,7 @@
+++
title = "API Reference"
weight = 10
nav = []
+++
@ -113,6 +115,8 @@ Response:
}
```
By default, all bits and attributes (*for `Bitmap` queries only*) are returned. In order to suppress returning bits, set `excludeBits` query argument to `true`; to suppress returning attributes, set `excludeAttrs` query argument to `true`.
### Change index time quantum
`PATCH /index/<index-name>/time-quantum`

View file

@ -1,5 +1,7 @@
+++
title = "Architecture"
weight = 6
nav = []
+++
## Architecture

View file

@ -1,5 +1,11 @@
+++
title = "Client Libraries"
weight = 12
nav = [
"Go",
"Python",
"Java",
]
+++
## Client Libraries

View file

@ -1,5 +1,12 @@
+++
title = "Configuration"
weight = 7
nav = [
"Command line flags",
"Environment variables",
"Config file",
"All Options",
]
+++
## Configuration
@ -200,3 +207,43 @@ Any flag that has a value that is a comma separated list on the command line bec
[metric]
poll-interval = "0m15s"
```
### Example Cluster Configuration
A three node cluster could be minimally configured as follows:
#### Node 0
data-dir = "/home/pilosa/data"
bind = "node0.pilosa.com:10101"
gossip-port = 12000
gossip-seed = "node0.pilosa.com:12000"
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
#### Node 1
data-dir = "/home/pilosa/data"
bind = "node1.pilosa.com:10101"
gossip-port = 12000
gossip-seed = "node0.pilosa.com:12000"
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]
#### Node 2
data-dir = "/home/pilosa/data"
bind = "node2.pilosa.com:10101"
gossip-port = 12000
gossip-seed = "node0.pilosa.com:12000"
[cluster]
replicas = 1
type = "gossip"
hosts = ["node0.pilosa.com:10101","node1.pilosa.com:10101","node2.pilosa.com:10101"]

View file

@ -1,5 +1,17 @@
+++
title = "Data Model"
weight = 5
nav = [
"Overview",
"Index",
"Column",
"Row",
"Frame",
"Time Quantum",
"Attribute",
"Slice",
"View",
]
+++
## Data Model

View file

@ -1,5 +1,7 @@
+++
title = "FAQ"
weight = 15
nav = []
+++
## FAQ

View file

@ -1,5 +1,12 @@
+++
title = "Getting Started"
weight = 3
nav = [
"Starting Pilosa",
"Sample Project",
"Input Definition",
"What's Next?",
]
+++
## Getting Started

View file

@ -1,5 +1,7 @@
+++
title = "Glossary"
weight = 14
nav = []
+++
## Glossary

View file

@ -1,12 +1,17 @@
+++
title = "Input Definition"
weight = 8
nav = [
"Create the Schema",
"Import Data",
]
+++
## Input Definition
This document builds on the data import concepts introduced in [Getting Started](../getting-started/).
Here we will demonstrate creating the index's schema and data definition. Then using this definition to import JSON data.
#### Create the Schema Using an Input Definition
### Create the Schema
Input definitions allow users to define a schema based on their data and to provide data to Pilosa in a more standard format like JSON. Once an input definition is created, we can send data to Pilosa as JSON, and as long as the data adheres to the definition, Pilosa will internally perform all of the appropriate mutations.
@ -26,7 +31,8 @@ curl localhost:10101/index/repository/input-definition/stargazer \
"frames": [
{
"name": "language",
"options": {
"options": {
"rowLabel": "language_id",
"inverseEnabled": true,
"timeQuantum": "YMD"
}
@ -34,6 +40,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \
{
"name": "stargazer",
"options": {
"rowLabel": "stargazer_id",
"inverseEnabled": true,
"timeQuantum": "YMD"
}
@ -55,7 +62,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \
"Go": 5,
"Java": 21,
"JavaScript": 13,
"Python": 17,
"Python": 17
}
}
],
@ -77,7 +84,7 @@ curl localhost:10101/index/repository/input-definition/stargazer \
"valueDestination": "set-timestamp"
}
],
"name": "time_value
"name": "time_value"
}
]
}'
@ -91,7 +98,7 @@ We can also set `repo_id` for multiple frames at the same time by providing fiel
- mapping: The value for this field is used to lookup a `rowID` in a map. A valueMap is required for this destination type.
- set-timestamp: The value for this field is used to lookup timestamp and set timestamp for the whole frame
#### Import Data Using an Input Definition
### Import Data
The sample data for the "Star Trace" project is at [Pilosa Getting Started repository](https://github.com/pilosa/getting-started).
@ -104,14 +111,14 @@ curl localhost:10101/index/repository/input/stargazer \
{
"language_id": "Go",
"repo_id": 91720568,
"stargazer_id": 513114
"stargazer_id": 513114,
"time_value": "2017-05-18T20:40"
},
{
"language_id": "Python",
"repo_id": 95122322
}'
]
}
]'
```
As defined in the input definition, field name `language_id` maps language to a corresponding id defined in `valueMap` and sets the appropriate bit in the `language` frame. The value corresponding to field name `stargazer_id` is added to the `stargazer` frame as rowID.
@ -121,8 +128,8 @@ The data input above is equivalent to the following `SetBit()` operations:
curl localhost:10101/index/repository/query \
-X POST \
-d 'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114)
'SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40")
SetBit(frame="stargazer", repo_id=91720568, stargazer_id=513114, timestamp="2017-05-18T20:40")
SetBit(frame="language", repo_id=91720568, language_id=5)
SetBit(frame="language", repo_id=95122322, language_id=17)
'
```
```

View file

@ -1,5 +1,10 @@
+++
title = "Installation"
weight = 2
nav = [
"Installing on MacOS",
"Installing on Linux",
]
+++

View file

@ -1,5 +1,7 @@
+++
title = "Introduction"
weight = 1
nav = []
+++

View file

@ -1,5 +1,10 @@
+++
title = "PDK"
weight = 11
nav = [
"Library",
"Examples",
]
+++
## PDK

View file

@ -1,5 +1,12 @@
+++
title = "Query Language"
weight = 6
nav = [
"Conventions",
"Arguments and Types",
"Write Operations",
"Read Operations",
]
+++
## Query Language

View file

@ -1,5 +1,10 @@
+++
title = "Tutorials"
weight = 4
nav = [
"Transportation",
"Chemical similarity search",
]
+++
## Tutorials
@ -192,7 +197,7 @@ for pcount, topn in zip(pcounts, resp.json()['results']):
average_amounts.append(float(wsum)/count)
```
For more examples and details, see this [ipython notebook](https://github.com/alanbernstein/pilosa-notebooks/blob/master/taxi-use-case.ipynb).
For more examples and details, see this [ipython notebook](https://github.com/pilosa/notebooks/blob/master/taxi-use-case.ipynb).
### Chemical similarity search

View file

@ -1,5 +1,10 @@
+++
title = "WebUI"
weight = 9
nav = [
"Console",
"Cluster Admin",
]
+++
## WebUI

View file

@ -161,6 +161,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
indexTag := fmt.Sprintf("index:%s", index)
// Special handling for mutation and top-n calls.
switch c.Name {
case "Average":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeAverage(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -174,6 +177,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
return nil, e.executeSetColumnAttrs(ctx, index, c, opt)
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
case "TopN":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeTopN(ctx, index, c, slices, opt)
@ -202,6 +208,41 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
return nil
}
// executeAverage executes an average() call.
func (e *Executor) executeAverage(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return 0, errors.New("Average(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return 0, errors.New("Average(): field required")
}
if len(c.Children) > 1 {
return 0, errors.New("Average() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeSumCountSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
other, _ := result.(SumCount)
if other.Count == 0 {
return 0, nil
}
return other.Sum / other.Count, nil
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
// Execute calls in bulk on each remote node and merge.
@ -229,36 +270,43 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// If the row label is used then return bitmap attributes.
bm, _ := other.(*Bitmap)
if c.Name == "Bitmap" {
idx := e.Holder.Index(index)
if idx != nil {
columnLabel := idx.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, err
}
bm.Attrs = attrs
} else if err != nil {
return nil, err
} else {
frame, _ := c.Args["frame"].(string)
if fr := idx.Frame(frame); fr != nil {
rowLabel := fr.RowLabel()
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
return nil, err
}
attrs, err := fr.RowAttrStore().Attrs(rowID)
if opt.ExcludeAttrs {
bm.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
if idx != nil {
columnLabel := idx.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, err
}
bm.Attrs = attrs
} else if err != nil {
return nil, err
} else {
frame, _ := c.Args["frame"].(string)
if fr := idx.Frame(frame); fr != nil {
rowLabel := fr.RowLabel()
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
return nil, err
}
attrs, err := fr.RowAttrStore().Attrs(rowID)
if err != nil {
return nil, err
}
bm.Attrs = attrs
}
}
}
}
}
if opt.ExcludeBits {
bm.segments = []BitmapSegment{}
}
return bm, nil
}
@ -275,11 +323,84 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
return e.executeRangeSlice(ctx, index, c, slice)
case "Union":
return e.executeUnionSlice(ctx, index, c, slice)
case "Xor":
return e.executeXorSlice(ctx, index, c, slice)
default:
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
}
// executeSum executes a sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return 0, errors.New("Sum(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return 0, errors.New("Sum(): field required")
}
if len(c.Children) > 1 {
return 0, errors.New("Sum() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeSumCountSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
other, _ := result.(SumCount)
return other.Sum, nil
}
// executeSumCountSlice executes calculates the sum & count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return SumCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return SumCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return SumCount{}, nil
}
view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if view == nil {
return SumCount{}, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
if err != nil {
return SumCount{}, err
}
return SumCount{
Sum: int64(vsum) + (int64(vcount) * field.Min),
Count: int64(vcount),
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
@ -508,6 +629,11 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
// Handle field ranges differently.
if c.HasConditionArg() {
return e.executeFieldRangeSlice(ctx, index, c, slice)
}
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
@ -562,7 +688,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Parse end time.
endTimeStr, _ := c.Args["end"].(string)
endTimeStr, ok := c.Args["end"].(string)
if !ok {
return nil, errors.New("Range() end time required")
}
@ -590,6 +716,64 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return bm, nil
}
// executeFieldRangeSlice executes a range(field) call for a local slice.
func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
// Remove frame field.
args := pql.CopyArgs(c.Args)
delete(args, "frame")
// Only one conditional field should remain.
if len(args) == 0 {
return nil, errors.New("Range(): condition required")
} else if len(args) > 1 {
return nil, errors.New("Range(): too many arguments")
}
// Extract condition field.
var fieldName string
var cond *pql.Condition
for k, v := range args {
vv, ok := v.(*pql.Condition)
if !ok {
return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v)
}
fieldName, cond = k, vv
}
// Only support integers for now.
value, ok := cond.Value.(int64)
if !ok {
return nil, errors.New("Range(): conditions only support integer values")
}
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
} else if value < field.Min || value > field.Max {
return NewBitmap(), nil
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
if frag == nil {
return NewBitmap(), nil
}
f.Stats.Count("range:field", 1, 1.0)
return frag.FieldRange(cond.Op, field.BitDepth(), uint64(value-field.Min))
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
other := NewBitmap()
@ -609,6 +793,25 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C
return other, nil
}
// executeXorSlice executes a xor() call for a local slice.
func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
other := NewBitmap()
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Xor(bm)
}
}
other.InvalidateCount()
return other, nil
}
// executeCount executes a count() call.
func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
if len(c.Children) == 0 {
@ -1179,6 +1382,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu
var err error
switch call.Name {
case "Average", "Sum":
v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
@ -1297,7 +1502,6 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
if n.Host == e.Host {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
if len(results) > 0 {
resp.result = results[0]
@ -1371,7 +1575,9 @@ type mapResponse struct {
// ExecOptions represents an execution context for a single Execute() call.
type ExecOptions struct {
Remote bool
Remote bool
ExcludeAttrs bool
ExcludeBits bool
}
// decodeError returns an error representation of s if s is non-blank.
@ -1414,3 +1620,30 @@ func needsSlices(calls []*pql.Call) bool {
}
return false
}
// SumCount represents a grouping of sum & count for Sum() and Average() calls.
type SumCount struct {
Sum int64 `json:"sum"`
Count int64 `json:"count"`
}
func (sc *SumCount) Add(other SumCount) SumCount {
return SumCount{
Sum: sc.Sum + other.Sum,
Count: sc.Count + other.Count,
}
}
func encodeSumCount(sc SumCount) *internal.SumCount {
return &internal.SumCount{
Sum: sc.Sum,
Count: sc.Count,
}
}
func decodeSumCount(pb *internal.SumCount) SumCount {
return SumCount{
Sum: pb.Sum,
Count: pb.Count,
}
}

View file

@ -59,6 +59,25 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := res[0].(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
fmt.Println("ATTRS", attrs)
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
t.Run("Column", func(t *testing.T) {
@ -187,6 +206,25 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
}
}
// Ensure a xor query can be executed.
func TestExecutor_Execute_Xor(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
// Ensure a count query can be executed.
func TestExecutor_Execute_Count(t *testing.T) {
hldr := test.MustOpenHolder()
@ -422,6 +460,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -559,7 +598,128 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(200) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(80) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
// Ensure a Average() query can be executed.
func TestExecutor_Execute_Average(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(40) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(30) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
// Ensure a range query can be executed.
@ -613,6 +773,119 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
}
// Ensure a Range(field) query can be executed.
func TestExecutor_Execute_FieldRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=50)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("AboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Bitmap).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("ErrFrameNotFound", func(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound {
t.Fatal(err)
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound {
t.Fatal(err)
}
})
}
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := test.NewCluster(2)

View file

@ -39,6 +39,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/roaring"
)
@ -293,11 +294,13 @@ func (f *Fragment) close() error {
// Flush cache if closing gracefully.
if err := f.flushCache(); err != nil {
f.logger().Printf("fragment: error flushing cache on close: err=%s, path=%s", err, f.path)
return err
}
// Close underlying storage.
if err := f.closeStorage(); err != nil {
f.logger().Printf("fragment: error closing storage: err=%s, path=%s", err, f.path)
return err
}
// Remove checksums.
@ -536,14 +539,45 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
return changed, nil
}
func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
f.mu.Lock()
defer f.mu.Unlock()
// Compute count based on the existance bit.
row := f.row(uint64(bitDepth), true, true)
if filter != nil {
row = row.Intersect(filter)
}
count = row.Count()
// Compute the sum based on the bit count of each row multiplied by the
// place value of each row. For example, 10 bits in the 1's place plus
// 4 bits in the 2's place plus 3 bits in the 4's place equals a total
// sum of 30:
//
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
row := f.row(uint64(i), true, true)
if filter != nil {
row = row.Intersect(filter)
}
sum += (1 << i) * row.Count()
}
return sum, count, nil
}
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
switch op {
case RangeOpEQ:
case pql.EQ:
return f.fieldRangeEQ(bitDepth, predicate)
case RangeOpLT, RangeOpLTE:
return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE)
case RangeOpGT, RangeOpGTE:
return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE)
case pql.LT, pql.LTE:
return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
case pql.GT, pql.GTE:
return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE)
default:
return nil, ErrInvalidRangeOperation
}

View file

@ -24,6 +24,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
@ -216,6 +217,45 @@ func TestFragment_SetFieldValue(t *testing.T) {
})
}
// Ensure a fragment can sum field values.
func TestFragment_FieldSum(t *testing.T) {
const bitDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(nil, bitDepth); err != nil {
t.Fatal(err)
} else if n != 4 {
t.Fatalf("unexpected count: %d", n)
} else if sum != 3800 {
t.Fatalf("unexpected sum: %d", sum)
}
})
t.Run("WithFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(pilosa.NewBitmap(2000, 4000, 5000), bitDepth); err != nil {
t.Fatal(err)
} else if n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if sum != 600 {
t.Fatalf("unexpected sum: %d", sum)
}
})
}
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16
@ -236,7 +276,7 @@ func TestFragment_FieldRange(t *testing.T) {
}
// Query for equality.
if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
@ -263,28 +303,28 @@ func TestFragment_FieldRange(t *testing.T) {
}
// Query for fields less than (ending with set bit).
if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil {
if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields less than (ending with unset bit).
if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields less than or equal to (ending with set bit).
if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil {
if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields less than or equal to (ending with unset bit).
if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
@ -311,28 +351,28 @@ func TestFragment_FieldRange(t *testing.T) {
}
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil {
if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil {
if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())

View file

@ -27,6 +27,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
)
// Default frame settings.
@ -40,15 +41,6 @@ const (
DefaultCacheSize = 50000
)
// List of operators for field range queries.
const (
RangeOpEQ = "eq"
RangeOpLT = "lt"
RangeOpLTE = "lte"
RangeOpGT = "gt"
RangeOpGTE = "gte"
)
// Frame represents a container for views.
type Frame struct {
mu sync.Mutex
@ -124,11 +116,15 @@ func (f *Frame) MaxSlice() uint64 {
f.mu.Lock()
defer f.mu.Unlock()
view := f.views[ViewStandard]
if view == nil {
return 0
var max uint64
for _, view := range f.views {
if view.name == ViewInverse {
continue
} else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
return view.MaxSlice()
return max
}
// MaxInverseSlice returns the max inverse slice in the frame.
@ -397,7 +393,9 @@ func (f *Frame) Close() error {
// Close all views.
for _, view := range f.views {
_ = view.Close()
if err := view.Close(); err != nil {
return err
}
}
f.views = make(map[string]*View)
@ -509,6 +507,28 @@ func (f *Frame) newView(path, name string) *View {
return view
}
// DeleteView removes the view from the frame.
func (f *Frame) DeleteView(name string) error {
view := f.views[name]
if view == nil {
return ErrInvalidView
}
// Close data files before deletion.
if err := view.Close(); err != nil {
return err
}
// Delete view directory.
if err := os.RemoveAll(view.Path()); err != nil {
return err
}
delete(f.views, name)
return nil
}
// SetBit sets a bit on a view within the frame.
func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
@ -639,7 +659,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change
return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
}
func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) {
// FieldSum returns the sum and count for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
}
func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)
if field == nil {

View file

@ -307,3 +307,36 @@ func TestFrame_RowLabelValidation(t *testing.T) {
}
}
// Ensure frame can open and retrieve a view.
func TestFrame_DeleteView(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in frame")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}

View file

@ -69,6 +69,12 @@ func (g *GossipNodeSet) Open() error {
return err
}
g.memberlist = ml
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
},
RetransmitMult: 3,
}
// attach to gossip seed node
nodes := []*pilosa.Node{&pilosa.Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds
@ -76,12 +82,6 @@ func (g *GossipNodeSet) Open() error {
if err != nil {
return err
}
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
},
RetransmitMult: 3,
}
return nil
}

View file

@ -27,6 +27,7 @@ import (
"io/ioutil"
"log"
"net/http"
// Imported for its side-effect of registering pprof endpoints with the server.
_ "net/http/pprof"
"os"
"runtime/debug"
@ -43,6 +44,7 @@ import (
"unicode"
// Allow building Pilosa without the web UI.
_ "github.com/pilosa/pilosa/statik"
"github.com/rakyll/statik/fs"
)
@ -116,6 +118,7 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH")
router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/view/{view}", handler.handleDeleteView).Methods("DELETE")
router.HandleFunc("/index/{index}/input/{input-definition}", handler.handlePostInput).Methods("POST")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleGetInputDefinition).Methods("GET")
router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST")
@ -241,7 +244,9 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// Build execution options.
opt := &ExecOptions{
Remote: req.Remote,
Remote: req.Remote,
ExcludeAttrs: req.ExcludeAttrs,
ExcludeBits: req.ExcludeBits,
}
// Parse query string.
@ -257,7 +262,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
resp := &QueryResponse{Results: results, Err: err}
// Fill column attributes if requested.
if req.ColumnAttrs {
if req.ColumnAttrs && !req.ExcludeBits {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
@ -789,6 +794,47 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
}
}
// handleDeleteView handles Delete /frame/view request.
func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
viewName := mux.Vars(r)["view"]
// Retrieve frame.
f := h.Holder.Frame(indexName, frameName)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Delete the view.
if err := f.DeleteView(viewName); err != nil {
// Ingore this error becuase views do not exist on all nodes due to slice distribution.
if err != ErrInvalidView {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
// Send the delete view message to all nodes.
err := h.Broadcaster.SendSync(
&internal.DeleteViewMessage{
Index: indexName,
Frame: frameName,
View: viewName,
})
if err != nil {
h.logger().Printf("problem sending DeleteView message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteViewResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type deleteViewResponse struct{}
type getFrameViewsResponse struct {
Views []string `json:"views,omitempty"`
}
@ -925,9 +971,11 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
}
return &QueryRequest{
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeAttrs: q.Get("excludeAttrs") == "true",
ExcludeBits: q.Get("excludeBits") == "true",
}, nil
}
@ -1396,6 +1444,12 @@ type QueryRequest struct {
// Return column attributes, if true.
ColumnAttrs bool
// Do not return row attributes, if true.
ExcludeAttrs bool
// Do not return bits, if true.
ExcludeBits bool
// If true, indicates that query is part of a larger distributed query.
// If false, this request is on the originating node.
Remote bool
@ -1403,10 +1457,12 @@ type QueryRequest struct {
func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
req := &QueryRequest{
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeAttrs: pb.ExcludeAttrs,
ExcludeBits: pb.ExcludeBits,
}
return req
@ -1455,6 +1511,8 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
pb.Results[i].Bitmap = encodeBitmap(result)
case []Pair:
pb.Results[i].Pairs = encodePairs(result)
case SumCount:
pb.Results[i].SumCount = encodeSumCount(result)
case uint64:
pb.Results[i].N = result
case bool:

View file

@ -1511,3 +1511,25 @@ func TestHandler_GetTimeStamp(t *testing.T) {
t.Fatalf("Expected Ignore nonexistent fields")
}
}
// Ensure handler can delete a view.
func TestHandler_DeleteView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
viewName := pilosa.ViewStandard + "_2017"
hldr.MustCreateFragmentIfNotExists("i0", "f0", viewName, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD")
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Frame("f0").View(viewName); f != nil {
t.Fatal("expected nil view")
}
}

View file

@ -133,7 +133,9 @@ func (h *Holder) Close() error {
h.wg.Wait()
for _, index := range h.indexes {
index.Close()
if err := index.Close(); err != nil {
return err
}
}
return nil
}

View file

@ -34,8 +34,9 @@ func TestHolder_Open(t *testing.T) {
if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
}
if err := h.Reopen(); err != nil {
t.Fatal(err)
} else if logOutput := h.LogOutput.String(); !strings.Contains(logOutput, `ERROR opening index: !`) {
@ -49,6 +50,8 @@ func TestHolder_Open(t *testing.T) {
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(h.IndexPath("test"), 0000); err != nil {
t.Fatal(err)
}
@ -64,6 +67,8 @@ func TestHolder_Open(t *testing.T) {
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil {
t.Fatal(err)
}
@ -78,6 +83,8 @@ func TestHolder_Open(t *testing.T) {
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".data"), 2); err != nil {
t.Fatal(err)
}
@ -95,6 +102,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar"), 0000); err != nil {
t.Fatal(err)
}
@ -112,6 +121,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".meta"), 2); err != nil {
t.Fatal(err)
}
@ -128,6 +139,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", ".data"), 2); err != nil {
t.Fatal(err)
}
@ -147,6 +160,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard"), 0000); err != nil {
t.Fatal(err)
}
@ -166,6 +181,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments"), 0000); err != nil {
t.Fatal(err)
}
@ -188,6 +205,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 0000); err != nil {
t.Fatal(err)
}
@ -209,6 +228,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Truncate(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0"), 2); err != nil {
t.Fatal(err)
}
@ -232,6 +253,8 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
} else if err := view.Fragment(0).FlushCache(); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
} else if err := os.Chmod(filepath.Join(h.Path, "foo", "bar", "views", "standard", "fragments", "0.cache"), 0000); err != nil {
t.Fatal(err)
}

View file

@ -248,7 +248,9 @@ func (i *Index) Close() error {
// Close all frames.
for _, f := range i.frames {
f.Close()
if err := f.Close(); err != nil {
return err
}
}
i.frames = make(map[string]*Frame)

View file

@ -32,6 +32,7 @@
ClusterStatus
FrameSchema
Field
DeleteViewMessage
*/
package internal
@ -405,6 +406,17 @@ func (m *Field) String() string { return proto.CompactTextString(m) }
func (*Field) ProtoMessage() {}
func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} }
type DeleteViewMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"`
}
func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} }
func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteViewMessage) ProtoMessage() {}
func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} }
func init() {
proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta")
proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta")
@ -429,6 +441,7 @@ func init() {
proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus")
proto.RegisterType((*FrameSchema)(nil), "internal.FrameSchema")
proto.RegisterType((*Field)(nil), "internal.Field")
proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage")
}
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
size := m.Size()
@ -1344,6 +1357,42 @@ func (m *Field) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *DeleteViewMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
}
if len(m.View) > 0 {
dAtA[i] = 0x1a
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.View)))
i += copy(dAtA[i:], m.View)
}
return i, nil
}
func encodeFixed64Private(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
@ -1773,6 +1822,24 @@ func (m *Field) Size() (n int) {
return n
}
func (m *DeleteViewMessage) Size() (n int) {
var l int
_ = l
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
l = len(m.Frame)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
l = len(m.View)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
return n
}
func sovPrivate(x uint64) (n int) {
for {
n++
@ -4882,6 +4949,143 @@ func (m *Field) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DeleteViewMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DeleteViewMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Frame = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field View", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.View = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipPrivate(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
@ -4990,62 +5194,64 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 912 bytes of a gzipped FileDescriptorProto
// 929 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xc1, 0x6e, 0x23, 0x45,
0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0xf1, 0x36, 0x61, 0xe5, 0x8d, 0x22, 0x13, 0xf5, 0x81,
0x0d, 0x91, 0xc8, 0x61, 0x91, 0x56, 0xc0, 0x72, 0x80, 0x8d, 0xb3, 0x8a, 0x05, 0x5e, 0xa0, 0xbd,
0x5a, 0x6e, 0x48, 0x1d, 0xa7, 0xd8, 0x1d, 0x65, 0x3c, 0x63, 0xa6, 0xdb, 0x49, 0xcc, 0x81, 0x23,
0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0x8a, 0xba, 0xba,
0x7b, 0x66, 0x3c, 0x8e, 0x13, 0x85, 0x5b, 0xd7, 0xeb, 0xd7, 0x55, 0xaf, 0x6a, 0xaa, 0xca, 0x86,
0xf5, 0x69, 0x16, 0x9d, 0x4b, 0x8d, 0x07, 0xd3, 0x2c, 0xd5, 0x29, 0x6b, 0x45, 0x89, 0xc6, 0x2c,
0x91, 0x31, 0xff, 0x0a, 0xda, 0x83, 0xe4, 0x14, 0x2f, 0x87, 0xa8, 0x25, 0xdb, 0x85, 0xb5, 0xc3,
0x34, 0x9e, 0x4d, 0x92, 0x2f, 0xe5, 0x09, 0xc6, 0xdd, 0x60, 0x37, 0xd8, 0x6b, 0x8b, 0x32, 0x64,
0x18, 0x2f, 0xa2, 0x09, 0x7e, 0x33, 0x93, 0x89, 0x9e, 0x4d, 0xba, 0x35, 0xcb, 0x28, 0x41, 0xfc,
0xcf, 0x00, 0xda, 0xcf, 0x32, 0x39, 0x41, 0xf2, 0xb8, 0x0d, 0x2d, 0x91, 0x5e, 0x94, 0xdd, 0xe5,
0x36, 0x7b, 0x0f, 0x36, 0x06, 0xc9, 0x39, 0x66, 0x0a, 0x8f, 0x12, 0x79, 0x12, 0xe3, 0x29, 0xb9,
0x6b, 0x89, 0x0a, 0xca, 0x76, 0xa0, 0x7d, 0x28, 0xc7, 0xaf, 0xf1, 0xc5, 0x7c, 0x8a, 0xdd, 0x3a,
0x39, 0x29, 0x80, 0xfc, 0x76, 0x14, 0xfd, 0x88, 0xdd, 0x70, 0x37, 0xd8, 0x5b, 0x17, 0x05, 0x50,
0xd5, 0xdb, 0x58, 0xd2, 0xcb, 0x38, 0xbc, 0x25, 0x64, 0xf2, 0x2a, 0xd7, 0xd0, 0x24, 0x0d, 0x0b,
0x18, 0xe7, 0xb0, 0x31, 0x98, 0x4c, 0xd3, 0x4c, 0x0b, 0x54, 0xd3, 0x34, 0x51, 0xc8, 0x3a, 0x50,
0x3f, 0xca, 0x32, 0x97, 0x92, 0x39, 0xf2, 0x9f, 0xa0, 0xf3, 0x34, 0x4e, 0xc7, 0x67, 0x7d, 0xa9,
0xa5, 0xc0, 0x1f, 0x66, 0xa8, 0x34, 0xdb, 0x82, 0x06, 0x15, 0xd7, 0xf1, 0xac, 0x61, 0x50, 0x2a,
0x90, 0xab, 0x9e, 0x35, 0x0c, 0x4a, 0xef, 0x29, 0xc3, 0x50, 0x58, 0xc3, 0xa0, 0xa3, 0x38, 0x1a,
0xdb, 0xcc, 0x42, 0x61, 0x0d, 0xc6, 0x20, 0x7c, 0x19, 0xe1, 0x85, 0x4b, 0x87, 0xce, 0x7c, 0x00,
0xf7, 0x4a, 0xf1, 0x9d, 0xcc, 0xfb, 0xd0, 0x14, 0xe9, 0xc5, 0xa0, 0xaf, 0xba, 0xc1, 0x6e, 0x7d,
0x2f, 0x14, 0xce, 0xa2, 0xa2, 0xd1, 0x57, 0x35, 0x57, 0x35, 0xba, 0x2a, 0x00, 0xfe, 0x00, 0x1a,
0x54, 0x41, 0x93, 0x65, 0xf1, 0xd6, 0x1c, 0xf9, 0xaf, 0x01, 0xdc, 0x1b, 0xca, 0x4b, 0x92, 0xa1,
0xf2, 0x30, 0xc7, 0xd0, 0xce, 0x41, 0x62, 0xaf, 0x3d, 0xda, 0x3f, 0xf0, 0x2d, 0x76, 0xb0, 0xc4,
0x2f, 0x90, 0xa3, 0x44, 0x67, 0x73, 0x51, 0x3c, 0xde, 0xfe, 0x14, 0x36, 0x16, 0x2f, 0x8d, 0x86,
0x33, 0x9c, 0xfb, 0x4a, 0x9f, 0xe1, 0xdc, 0xd4, 0xe4, 0x5c, 0xc6, 0x33, 0x5b, 0xbf, 0x50, 0x58,
0xe3, 0x93, 0xda, 0x47, 0x01, 0xff, 0x0e, 0xd8, 0x61, 0x86, 0x52, 0x23, 0x39, 0x18, 0xa2, 0x52,
0xf2, 0x15, 0xae, 0xfe, 0x0a, 0xb6, 0xb2, 0xb5, 0x72, 0x65, 0x77, 0xa0, 0x3d, 0x50, 0xae, 0xff,
0xe8, 0x4b, 0xb4, 0x44, 0x01, 0xf0, 0x7d, 0x60, 0x7d, 0x8c, 0x51, 0xa3, 0x1b, 0x99, 0x1b, 0xfc,
0xf3, 0x91, 0xd7, 0x72, 0x3b, 0x97, 0x3d, 0x84, 0xd0, 0x4c, 0x0b, 0x49, 0x59, 0x7b, 0xf4, 0x76,
0x51, 0xba, 0x7c, 0x34, 0x05, 0x11, 0x78, 0xe4, 0x9d, 0xba, 0x09, 0xbb, 0x25, 0xc1, 0x6b, 0xda,
0xcc, 0x87, 0xaa, 0x57, 0x43, 0xe5, 0x33, 0xeb, 0x42, 0x7d, 0xe6, 0x73, 0xfd, 0xbf, 0xa1, 0x78,
0xdf, 0xa1, 0xa6, 0x5d, 0x9f, 0x9b, 0x5b, 0xfb, 0x86, 0xce, 0xab, 0x53, 0xae, 0xea, 0xf8, 0x27,
0x70, 0x21, 0xef, 0xe6, 0xa6, 0x52, 0x39, 0xb3, 0x88, 0x7c, 0x63, 0xb9, 0x09, 0xcb, 0x6d, 0xf6,
0x10, 0x9a, 0x14, 0x55, 0x75, 0x43, 0xea, 0xdd, 0xcd, 0x8a, 0x1a, 0xe1, 0xae, 0xcd, 0x38, 0xb9,
0x26, 0x6f, 0xd8, 0x71, 0xb2, 0x16, 0x3b, 0x82, 0xce, 0x20, 0x99, 0xce, 0x74, 0x1f, 0xbf, 0x8f,
0x92, 0x48, 0x47, 0x69, 0xa2, 0xba, 0x4d, 0x72, 0xf5, 0xa0, 0xac, 0x68, 0x81, 0x21, 0x96, 0x9e,
0xf0, 0x9f, 0x03, 0xd8, 0xac, 0x80, 0x2b, 0x92, 0xf6, 0x7a, 0x6b, 0x37, 0xeb, 0x7d, 0x0c, 0xcd,
0x67, 0x11, 0xc6, 0xa7, 0xaa, 0x5b, 0x27, 0x62, 0x6f, 0xa5, 0x1a, 0xa2, 0x09, 0xc7, 0xe6, 0xbf,
0x05, 0xb0, 0x75, 0x1d, 0xe1, 0x5a, 0x35, 0x3d, 0x80, 0xaf, 0xb3, 0x68, 0x22, 0xb3, 0xf9, 0x17,
0x38, 0x77, 0x2b, 0xbc, 0x84, 0xb0, 0x6f, 0xe1, 0x7e, 0xc5, 0xd7, 0xe7, 0x63, 0x5b, 0x22, 0x2b,
0xea, 0xdd, 0x95, 0xa2, 0x2c, 0x4f, 0xac, 0x78, 0xce, 0xff, 0x0d, 0xe0, 0x9d, 0x6b, 0xaf, 0x8a,
0x7e, 0x0c, 0xca, 0xad, 0xbf, 0x0f, 0x9d, 0x97, 0x66, 0x55, 0xf4, 0x51, 0xe9, 0x28, 0x91, 0x86,
0xe9, 0x1a, 0x76, 0x09, 0x67, 0x03, 0x68, 0x11, 0x36, 0x94, 0x53, 0x27, 0xf3, 0x83, 0x5b, 0x64,
0x1e, 0x78, 0xbe, 0xdd, 0x69, 0xf9, 0x73, 0x23, 0x86, 0xb6, 0xae, 0x5f, 0xe1, 0x64, 0x6c, 0x3f,
0x81, 0xf5, 0x85, 0x07, 0x77, 0xda, 0x73, 0x29, 0xec, 0xf8, 0xdd, 0xb2, 0xa0, 0xe4, 0xe6, 0x29,
0xfd, 0x18, 0xa0, 0xa0, 0xba, 0x05, 0x70, 0x43, 0x7f, 0x96, 0xc8, 0xfc, 0x18, 0x76, 0xfc, 0xe2,
0xbb, 0x43, 0x40, 0xdf, 0x2d, 0xb5, 0xa2, 0x5b, 0xb8, 0x04, 0x78, 0x9e, 0x9e, 0xe2, 0x48, 0x4b,
0x3d, 0x53, 0x86, 0x71, 0x9c, 0x2a, 0xed, 0xfb, 0xc9, 0x9c, 0x69, 0x31, 0x6b, 0xa9, 0xf3, 0x65,
0x42, 0x06, 0x7b, 0x1f, 0xde, 0x24, 0xa7, 0xe8, 0xdb, 0x66, 0xb3, 0x32, 0xeb, 0xc2, 0xdf, 0xf3,
0x27, 0xb0, 0x7e, 0x18, 0xcf, 0x94, 0xc6, 0xcc, 0x45, 0xd9, 0x87, 0x86, 0x89, 0xe9, 0x7f, 0x9a,
0xb6, 0x8a, 0x97, 0x85, 0x14, 0x61, 0x29, 0xfc, 0x31, 0xac, 0x51, 0xb7, 0x8c, 0xc6, 0xaf, 0x71,
0x22, 0x69, 0xd4, 0xec, 0x04, 0x05, 0x4b, 0xa3, 0xb6, 0x30, 0x32, 0x23, 0x68, 0xac, 0x1e, 0x11,
0x06, 0x21, 0xfd, 0x79, 0x71, 0x85, 0xa0, 0xff, 0x2d, 0x1d, 0xa8, 0x0f, 0x23, 0xfb, 0x19, 0xea,
0xc2, 0x1c, 0x09, 0x91, 0x97, 0xd4, 0x26, 0x06, 0x91, 0x97, 0x4f, 0x3b, 0xbf, 0x5f, 0xf5, 0x82,
0x3f, 0xae, 0x7a, 0xc1, 0x5f, 0x57, 0xbd, 0xe0, 0x97, 0xbf, 0x7b, 0x6f, 0x9c, 0x34, 0xe9, 0xff,
0xdb, 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x29, 0x07, 0x36, 0x04, 0xd0, 0x09, 0x00, 0x00,
0x10, 0x65, 0xec, 0xb1, 0xb1, 0x2b, 0x24, 0x71, 0x9a, 0xb0, 0xf2, 0x46, 0x91, 0x89, 0xfa, 0xc0,
0x86, 0x48, 0xe4, 0xb0, 0x48, 0x2b, 0x60, 0x39, 0xc0, 0xc6, 0x59, 0xc5, 0x02, 0x2f, 0xd0, 0x5e,
0x2d, 0x37, 0xa4, 0x8e, 0x53, 0xec, 0x8e, 0x32, 0x9e, 0x31, 0x33, 0x3d, 0x49, 0xcc, 0x81, 0x23,
0xdf, 0x80, 0xc4, 0x91, 0x9f, 0xe1, 0x08, 0x7f, 0x80, 0xc2, 0x85, 0x3f, 0xe0, 0xba, 0xea, 0xea,
0xee, 0x99, 0xf1, 0x38, 0x76, 0x94, 0xbd, 0x75, 0xbd, 0x7e, 0x5d, 0xf5, 0xba, 0xa6, 0xaa, 0xa6,
0x61, 0x7d, 0x9a, 0x04, 0x17, 0x52, 0xe1, 0xe1, 0x34, 0x89, 0x55, 0xcc, 0x5a, 0x41, 0xa4, 0x30,
0x89, 0x64, 0xc8, 0xbf, 0x81, 0xf6, 0x20, 0x3a, 0xc3, 0xab, 0x21, 0x2a, 0xc9, 0xf6, 0x60, 0xed,
0x28, 0x0e, 0xb3, 0x49, 0xf4, 0xb5, 0x3c, 0xc5, 0xb0, 0xeb, 0xed, 0x79, 0xfb, 0x6d, 0x51, 0x86,
0x34, 0xe3, 0x79, 0x30, 0xc1, 0xef, 0x32, 0x19, 0xa9, 0x6c, 0xd2, 0xad, 0x19, 0x46, 0x09, 0xe2,
0x7f, 0x7b, 0xd0, 0x7e, 0x9a, 0xc8, 0x09, 0x92, 0xc7, 0x1d, 0x68, 0x89, 0xf8, 0xb2, 0xec, 0x2e,
0xb7, 0xd9, 0x07, 0xb0, 0x31, 0x88, 0x2e, 0x30, 0x49, 0xf1, 0x38, 0x92, 0xa7, 0x21, 0x9e, 0x91,
0xbb, 0x96, 0xa8, 0xa0, 0x6c, 0x17, 0xda, 0x47, 0x72, 0xfc, 0x0a, 0x9f, 0xcf, 0xa6, 0xd8, 0xad,
0x93, 0x93, 0x02, 0xc8, 0x77, 0x47, 0xc1, 0xcf, 0xd8, 0xf5, 0xf7, 0xbc, 0xfd, 0x75, 0x51, 0x00,
0x55, 0xbd, 0x8d, 0x05, 0xbd, 0x8c, 0xc3, 0x3b, 0x42, 0x46, 0x2f, 0x73, 0x0d, 0x4d, 0xd2, 0x30,
0x87, 0x71, 0x0e, 0x1b, 0x83, 0xc9, 0x34, 0x4e, 0x94, 0xc0, 0x74, 0x1a, 0x47, 0x29, 0xb2, 0x0e,
0xd4, 0x8f, 0x93, 0xc4, 0x5e, 0x49, 0x2f, 0xf9, 0x2f, 0xd0, 0x79, 0x12, 0xc6, 0xe3, 0xf3, 0xbe,
0x54, 0x52, 0xe0, 0x4f, 0x19, 0xa6, 0x8a, 0x6d, 0x43, 0x83, 0x92, 0x6b, 0x79, 0xc6, 0xd0, 0x28,
0x25, 0xc8, 0x66, 0xcf, 0x18, 0x1a, 0xa5, 0xf3, 0x74, 0x43, 0x5f, 0x18, 0x43, 0xa3, 0xa3, 0x30,
0x18, 0x9b, 0x9b, 0xf9, 0xc2, 0x18, 0x8c, 0x81, 0xff, 0x22, 0xc0, 0x4b, 0x7b, 0x1d, 0x5a, 0xf3,
0x01, 0x6c, 0x95, 0xe2, 0x5b, 0x99, 0xf7, 0xa0, 0x29, 0xe2, 0xcb, 0x41, 0x3f, 0xed, 0x7a, 0x7b,
0xf5, 0x7d, 0x5f, 0x58, 0x8b, 0x92, 0x46, 0x5f, 0x55, 0x6f, 0xd5, 0x68, 0xab, 0x00, 0xf8, 0x7d,
0x68, 0x50, 0x06, 0xf5, 0x2d, 0x8b, 0xb3, 0x7a, 0xc9, 0x7f, 0xf7, 0x60, 0x6b, 0x28, 0xaf, 0x48,
0x46, 0x9a, 0x87, 0x39, 0x81, 0x76, 0x0e, 0x12, 0x7b, 0xed, 0xe1, 0xc1, 0xa1, 0x2b, 0xb1, 0xc3,
0x05, 0x7e, 0x81, 0x1c, 0x47, 0x2a, 0x99, 0x89, 0xe2, 0xf0, 0xce, 0xe7, 0xb0, 0x31, 0xbf, 0xa9,
0x35, 0x9c, 0xe3, 0xcc, 0x65, 0xfa, 0x1c, 0x67, 0x3a, 0x27, 0x17, 0x32, 0xcc, 0x4c, 0xfe, 0x7c,
0x61, 0x8c, 0xcf, 0x6a, 0x9f, 0x78, 0xfc, 0x07, 0x60, 0x47, 0x09, 0x4a, 0x85, 0xe4, 0x60, 0x88,
0x69, 0x2a, 0x5f, 0xe2, 0xf2, 0xaf, 0x60, 0x32, 0x5b, 0x2b, 0x67, 0x76, 0x17, 0xda, 0x83, 0xd4,
0xd6, 0x1f, 0x7d, 0x89, 0x96, 0x28, 0x00, 0x7e, 0x00, 0xac, 0x8f, 0x21, 0x2a, 0xb4, 0x2d, 0xb3,
0xc2, 0x3f, 0x1f, 0x39, 0x2d, 0xb7, 0x73, 0xd9, 0x03, 0xf0, 0x75, 0xb7, 0x90, 0x94, 0xb5, 0x87,
0xef, 0x16, 0xa9, 0xcb, 0x5b, 0x53, 0x10, 0x81, 0x07, 0xce, 0xa9, 0xed, 0xb0, 0x5b, 0x2e, 0x78,
0x43, 0x99, 0xb9, 0x50, 0xf5, 0x6a, 0xa8, 0xbc, 0x67, 0x6d, 0xa8, 0x2f, 0xdc, 0x5d, 0xdf, 0x34,
0x14, 0xef, 0x5b, 0x54, 0x97, 0xeb, 0x33, 0xbd, 0x6b, 0xce, 0xd0, 0x7a, 0xf9, 0x95, 0xab, 0x3a,
0xfe, 0xf3, 0x6c, 0xc8, 0xbb, 0xb9, 0xa9, 0x64, 0x4e, 0x0f, 0x22, 0x57, 0x58, 0xb6, 0xc3, 0x72,
0x9b, 0x3d, 0x80, 0x26, 0x45, 0x4d, 0xbb, 0x3e, 0xd5, 0xee, 0x66, 0x45, 0x8d, 0xb0, 0xdb, 0xba,
0x9d, 0x6c, 0x91, 0x37, 0x4c, 0x3b, 0x19, 0x8b, 0x1d, 0x43, 0x67, 0x10, 0x4d, 0x33, 0xd5, 0xc7,
0x1f, 0x83, 0x28, 0x50, 0x41, 0x1c, 0xa5, 0xdd, 0x26, 0xb9, 0xba, 0x5f, 0x56, 0x34, 0xc7, 0x10,
0x0b, 0x47, 0xf8, 0xaf, 0x1e, 0x6c, 0x56, 0xc0, 0x25, 0x97, 0x76, 0x7a, 0x6b, 0xab, 0xf5, 0x3e,
0x82, 0xe6, 0xd3, 0x00, 0xc3, 0xb3, 0xb4, 0x5b, 0x27, 0x62, 0x6f, 0xa9, 0x1a, 0xa2, 0x09, 0xcb,
0xe6, 0x7f, 0x78, 0xb0, 0x7d, 0x13, 0xe1, 0x46, 0x35, 0x3d, 0x80, 0x6f, 0x93, 0x60, 0x22, 0x93,
0xd9, 0x57, 0x38, 0xb3, 0x23, 0xbc, 0x84, 0xb0, 0xef, 0xe1, 0x5e, 0xc5, 0xd7, 0x97, 0x63, 0x93,
0x22, 0x23, 0xea, 0xfd, 0xa5, 0xa2, 0x0c, 0x4f, 0x2c, 0x39, 0xce, 0xff, 0xf7, 0xe0, 0xbd, 0x1b,
0xb7, 0x8a, 0x7a, 0xf4, 0xca, 0xa5, 0x7f, 0x00, 0x9d, 0x17, 0x7a, 0x54, 0xf4, 0x31, 0x55, 0x41,
0x24, 0x35, 0xd3, 0x16, 0xec, 0x02, 0xce, 0x06, 0xd0, 0x22, 0x6c, 0x28, 0xa7, 0x56, 0xe6, 0x47,
0xb7, 0xc8, 0x3c, 0x74, 0x7c, 0x33, 0xd3, 0xf2, 0xe3, 0x5a, 0x0c, 0x4d, 0x5d, 0x37, 0xc2, 0xc9,
0xd8, 0x79, 0x0c, 0xeb, 0x73, 0x07, 0xee, 0x34, 0xe7, 0x62, 0xd8, 0x75, 0xb3, 0x65, 0x4e, 0xc9,
0xea, 0x2e, 0xfd, 0x14, 0xa0, 0xa0, 0xda, 0x01, 0xb0, 0xa2, 0x3e, 0x4b, 0x64, 0x7e, 0x02, 0xbb,
0x6e, 0xf0, 0xdd, 0x21, 0xa0, 0xab, 0x96, 0x5a, 0x51, 0x2d, 0x5c, 0x02, 0x3c, 0x8b, 0xcf, 0x70,
0xa4, 0xa4, 0xca, 0x52, 0xcd, 0x38, 0x89, 0x53, 0xe5, 0xea, 0x49, 0xaf, 0x69, 0x30, 0x2b, 0xa9,
0xf2, 0x61, 0x42, 0x06, 0xfb, 0x10, 0xde, 0x26, 0xa7, 0xe8, 0xca, 0x66, 0xb3, 0xd2, 0xeb, 0xc2,
0xed, 0xf3, 0xc7, 0xb0, 0x7e, 0x14, 0x66, 0xa9, 0xc2, 0xc4, 0x46, 0x39, 0x80, 0x86, 0x8e, 0xe9,
0x7e, 0x4d, 0xdb, 0xc5, 0xc9, 0x42, 0x8a, 0x30, 0x14, 0xfe, 0x08, 0xd6, 0xa8, 0x5a, 0x46, 0xe3,
0x57, 0x38, 0x91, 0xd4, 0x6a, 0xa6, 0x83, 0xbc, 0x85, 0x56, 0x9b, 0x6b, 0x99, 0x11, 0x34, 0x96,
0xb7, 0x08, 0x03, 0x9f, 0x1e, 0x2f, 0x36, 0x11, 0xf4, 0x6e, 0xe9, 0x40, 0x7d, 0x18, 0x98, 0xcf,
0x50, 0x17, 0x7a, 0x49, 0x88, 0xbc, 0xa2, 0x32, 0xd1, 0x88, 0xd4, 0xff, 0x90, 0x2d, 0x93, 0x76,
0xfd, 0x87, 0x7f, 0x93, 0x69, 0xef, 0x1e, 0x0a, 0xf5, 0xe2, 0xa1, 0xf0, 0xa4, 0xf3, 0xe7, 0x75,
0xcf, 0xfb, 0xeb, 0xba, 0xe7, 0xfd, 0x73, 0xdd, 0xf3, 0x7e, 0xfb, 0xb7, 0xf7, 0xd6, 0x69, 0x93,
0x1e, 0x85, 0x1f, 0xbf, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x9f, 0xcc, 0xe2, 0x4c, 0x25, 0x0a, 0x00,
0x00,
}

View file

@ -131,3 +131,9 @@ message Field {
int64 Min = 3;
int64 Max = 4;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string View = 3;
}

View file

@ -11,6 +11,7 @@
It has these top-level messages:
Bitmap
Pair
SumCount
Bit
ColumnAttrSet
Attr
@ -66,6 +67,16 @@ func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} }
type SumCount struct {
Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"`
Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
func (m *SumCount) Reset() { *m = SumCount{} }
func (m *SumCount) String() string { return proto.CompactTextString(m) }
func (*SumCount) ProtoMessage() {}
func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
type Bit struct {
RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"`
ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"`
@ -75,7 +86,7 @@ type Bit struct {
func (m *Bit) Reset() { *m = Bit{} }
func (m *Bit) String() string { return proto.CompactTextString(m) }
func (*Bit) ProtoMessage() {}
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
type ColumnAttrSet struct {
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
@ -85,7 +96,7 @@ type ColumnAttrSet struct {
func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} }
func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) }
func (*ColumnAttrSet) ProtoMessage() {}
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} }
func (m *ColumnAttrSet) GetAttrs() []*Attr {
if m != nil {
@ -106,7 +117,7 @@ type Attr struct {
func (m *Attr) Reset() { *m = Attr{} }
func (m *Attr) String() string { return proto.CompactTextString(m) }
func (*Attr) ProtoMessage() {}
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} }
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} }
type AttrMap struct {
Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"`
@ -115,7 +126,7 @@ type AttrMap struct {
func (m *AttrMap) Reset() { *m = AttrMap{} }
func (m *AttrMap) String() string { return proto.CompactTextString(m) }
func (*AttrMap) ProtoMessage() {}
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} }
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
func (m *AttrMap) GetAttrs() []*Attr {
if m != nil {
@ -125,16 +136,18 @@ func (m *AttrMap) GetAttrs() []*Attr {
}
type QueryRequest struct {
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"`
ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} }
type QueryResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
@ -145,7 +158,7 @@ type QueryResponse struct {
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} }
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} }
func (m *QueryResponse) GetResults() []*QueryResult {
if m != nil {
@ -162,16 +175,17 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet {
}
type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
}
func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} }
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} }
func (m *QueryResult) GetBitmap() *Bitmap {
if m != nil {
@ -187,6 +201,13 @@ func (m *QueryResult) GetPairs() []*Pair {
return nil
}
func (m *QueryResult) GetSumCount() *SumCount {
if m != nil {
return m.SumCount
}
return nil
}
type ImportRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
@ -199,11 +220,12 @@ type ImportRequest struct {
func (m *ImportRequest) Reset() { *m = ImportRequest{} }
func (m *ImportRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRequest) ProtoMessage() {}
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} }
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} }
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*SumCount)(nil), "internal.SumCount")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
@ -288,6 +310,34 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *SumCount) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *SumCount) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Sum != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Sum))
}
if m.Count != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Count))
}
return i, nil
}
func (m *Bit) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@ -499,6 +549,26 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.ExcludeAttrs {
dAtA[i] = 0x30
i++
if m.ExcludeAttrs {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if m.ExcludeBits {
dAtA[i] = 0x38
i++
if m.ExcludeBits {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -602,6 +672,16 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.SumCount != nil {
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size()))
n6, err := m.SumCount.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n6
}
return i, nil
}
@ -638,56 +718,56 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
}
if len(m.RowIDs) > 0 {
dAtA7 := make([]byte, len(m.RowIDs)*10)
var j6 int
dAtA8 := make([]byte, len(m.RowIDs)*10)
var j7 int
for _, num := range m.RowIDs {
for num >= 1<<7 {
dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80)
dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j6++
j7++
}
dAtA7[j6] = uint8(num)
j6++
dAtA8[j7] = uint8(num)
j7++
}
dAtA[i] = 0x22
i++
i = encodeVarintPublic(dAtA, i, uint64(j6))
i += copy(dAtA[i:], dAtA7[:j6])
i = encodeVarintPublic(dAtA, i, uint64(j7))
i += copy(dAtA[i:], dAtA8[:j7])
}
if len(m.ColumnIDs) > 0 {
dAtA9 := make([]byte, len(m.ColumnIDs)*10)
var j8 int
dAtA10 := make([]byte, len(m.ColumnIDs)*10)
var j9 int
for _, num := range m.ColumnIDs {
for num >= 1<<7 {
dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80)
dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j8++
j9++
}
dAtA9[j8] = uint8(num)
j8++
dAtA10[j9] = uint8(num)
j9++
}
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(j8))
i += copy(dAtA[i:], dAtA9[:j8])
i = encodeVarintPublic(dAtA, i, uint64(j9))
i += copy(dAtA[i:], dAtA10[:j9])
}
if len(m.Timestamps) > 0 {
dAtA11 := make([]byte, len(m.Timestamps)*10)
var j10 int
dAtA12 := make([]byte, len(m.Timestamps)*10)
var j11 int
for _, num1 := range m.Timestamps {
num := uint64(num1)
for num >= 1<<7 {
dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80)
dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j10++
j11++
}
dAtA11[j10] = uint8(num)
j10++
dAtA12[j11] = uint8(num)
j11++
}
dAtA[i] = 0x32
i++
i = encodeVarintPublic(dAtA, i, uint64(j10))
i += copy(dAtA[i:], dAtA11[:j10])
i = encodeVarintPublic(dAtA, i, uint64(j11))
i += copy(dAtA[i:], dAtA12[:j11])
}
return i, nil
}
@ -750,6 +830,18 @@ func (m *Pair) Size() (n int) {
return n
}
func (m *SumCount) Size() (n int) {
var l int
_ = l
if m.Sum != 0 {
n += 1 + sovPublic(uint64(m.Sum))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
}
return n
}
func (m *Bit) Size() (n int) {
var l int
_ = l
@ -838,6 +930,12 @@ func (m *QueryRequest) Size() (n int) {
if m.Remote {
n += 2
}
if m.ExcludeAttrs {
n += 2
}
if m.ExcludeBits {
n += 2
}
return n
}
@ -882,6 +980,10 @@ func (m *QueryResult) Size() (n int) {
if m.Changed {
n += 2
}
if m.SumCount != nil {
l = m.SumCount.Size()
n += 1 + l + sovPublic(uint64(l))
}
return n
}
@ -1167,6 +1269,94 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *SumCount) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SumCount: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType)
}
m.Sum = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Sum |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType)
}
m.Count = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Count |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Bit) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@ -1799,6 +1989,46 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
}
}
m.Remote = bool(v != 0)
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.ExcludeAttrs = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.ExcludeBits = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -2093,6 +2323,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
}
}
m.Changed = bool(v != 0)
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.SumCount == nil {
m.SumCount = &SumCount{}
}
if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -2535,41 +2798,44 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 563 bytes of a gzipped FileDescriptorProto
// 621 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57,
0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xe8, 0x0c, 0xec, 0x3d, 0x33, 0xad,
0xc1, 0x92, 0x7f, 0x74, 0xb7, 0x81, 0x1c, 0x80, 0x13, 0xb0, 0xe1, 0x06, 0x70, 0x14, 0x96, 0x1c,
0x01, 0x85, 0x8b, 0xa0, 0xea, 0x76, 0xc7, 0x1e, 0x16, 0x68, 0x76, 0xfd, 0x5e, 0x75, 0xb5, 0xeb,
0xd5, 0xab, 0x32, 0x4c, 0xab, 0xfa, 0x32, 0x4b, 0xaf, 0x56, 0x95, 0x28, 0x55, 0x49, 0x47, 0x69,
0xa1, 0xb8, 0x28, 0x92, 0x2c, 0x88, 0x60, 0x10, 0xa5, 0x2a, 0x4f, 0x2a, 0x4a, 0xc1, 0x8d, 0x52,
0x25, 0x3d, 0xe2, 0x3b, 0xa1, 0xcb, 0xf4, 0x99, 0x3e, 0x85, 0xfe, 0x0b, 0xa5, 0x84, 0xf4, 0x7a,
0xbe, 0x13, 0x4e, 0x4e, 0xe7, 0x2b, 0x9b, 0xb7, 0x42, 0x9a, 0x99, 0x60, 0xb0, 0x02, 0xf7, 0x75,
0x92, 0x0a, 0xba, 0x00, 0xe7, 0x25, 0xdf, 0x7b, 0xc4, 0x27, 0xa1, 0xcb, 0xf0, 0x48, 0xef, 0x43,
0x7f, 0x5d, 0xd6, 0x85, 0xf2, 0x7a, 0x9a, 0x33, 0x20, 0x78, 0x0b, 0x4e, 0x94, 0x2a, 0x0c, 0xb2,
0xf2, 0x53, 0xbc, 0x69, 0x12, 0x0c, 0xa0, 0x8f, 0x61, 0xb4, 0x2e, 0xb3, 0x3a, 0x2f, 0xe2, 0x4d,
0x93, 0x75, 0xc4, 0xf4, 0x09, 0x8c, 0x2f, 0xd2, 0x9c, 0x4b, 0x95, 0xe4, 0x95, 0xe7, 0xf8, 0x24,
0x74, 0x58, 0x4b, 0x04, 0x5b, 0x98, 0x99, 0x9b, 0x58, 0xd5, 0x8e, 0x2b, 0x3a, 0x87, 0xde, 0xf1,
0xf5, 0x5e, 0xbc, 0xb9, 0xa3, 0x9a, 0x1f, 0x04, 0x5c, 0x3c, 0x75, 0xe5, 0x8c, 0x8d, 0x1c, 0x0a,
0xee, 0xc5, 0xbe, 0xe2, 0x4d, 0x5d, 0xfa, 0x4c, 0x7d, 0x98, 0xec, 0x94, 0x48, 0x8b, 0x9b, 0x77,
0x49, 0x56, 0x73, 0x5d, 0xd5, 0x98, 0x75, 0x29, 0x54, 0x14, 0x17, 0xca, 0x84, 0x5d, 0x5d, 0xf4,
0x11, 0xa3, 0xa2, 0xa8, 0x2c, 0x33, 0x13, 0xec, 0xfb, 0x24, 0x1c, 0xb1, 0x96, 0xa0, 0x4b, 0x80,
0xb3, 0xac, 0x4c, 0x9a, 0xdc, 0x81, 0x4f, 0x42, 0xc2, 0x3a, 0x4c, 0x70, 0x02, 0x43, 0xac, 0xf4,
0x55, 0x52, 0xb5, 0xda, 0xc8, 0xff, 0xb4, 0x7d, 0x84, 0xe9, 0x9b, 0x9a, 0x8b, 0x3d, 0xe3, 0x1f,
0x6a, 0x2e, 0xb5, 0x05, 0x1a, 0x37, 0x22, 0x0d, 0xa0, 0x0f, 0x61, 0xb0, 0xcb, 0xd2, 0x2b, 0x6e,
0x1a, 0xe5, 0xb2, 0x06, 0xa1, 0xd4, 0xb6, 0xc1, 0x52, 0x4b, 0x1d, 0xb1, 0x2e, 0x85, 0x99, 0x8c,
0xe7, 0xa5, 0xb2, 0x5a, 0x1a, 0x14, 0x7c, 0x25, 0x30, 0x6b, 0x3e, 0x2c, 0xab, 0xb2, 0x90, 0x1c,
0x9b, 0xbb, 0x15, 0xc2, 0x36, 0x77, 0x2b, 0x04, 0x3d, 0x81, 0x21, 0xe3, 0xb2, 0xce, 0x94, 0xf5,
0xe7, 0x41, 0xab, 0xc1, 0xe6, 0xd6, 0x99, 0x62, 0xf6, 0x16, 0x7d, 0x0e, 0xf3, 0x5b, 0x7e, 0x63,
0x45, 0x98, 0xf7, 0xa8, 0xcd, 0xbb, 0x15, 0x67, 0xff, 0x5c, 0x0f, 0xbe, 0x10, 0x98, 0x74, 0x5e,
0xa6, 0xa1, 0xdd, 0x05, 0x5d, 0xd6, 0xe4, 0x74, 0xd1, 0x3e, 0x64, 0x78, 0x66, 0x77, 0x65, 0x0a,
0xe4, 0xbc, 0x99, 0x02, 0x72, 0x8e, 0xbd, 0xc7, 0xf9, 0xb7, 0xdf, 0xef, 0xf4, 0x1e, 0x69, 0x66,
0x82, 0xd4, 0x83, 0xe1, 0xfa, 0x7d, 0x52, 0xdc, 0xf0, 0x6b, 0x3d, 0x05, 0x23, 0x66, 0x61, 0xf0,
0x9d, 0xc0, 0x2c, 0xce, 0xab, 0x52, 0xa8, 0x8e, 0x2f, 0x71, 0x71, 0xcd, 0x3f, 0x5b, 0x5f, 0x34,
0x40, 0xf6, 0x4c, 0x24, 0xb9, 0x99, 0xbf, 0x31, 0x33, 0x00, 0x59, 0xed, 0x8f, 0xf6, 0xc3, 0x65,
0x06, 0x68, 0x27, 0x70, 0x9f, 0xa4, 0xe7, 0x1a, 0x0f, 0x0d, 0xc2, 0x81, 0xb3, 0xeb, 0x24, 0xbd,
0xbe, 0x0e, 0xb5, 0x04, 0x0e, 0xdc, 0x71, 0x9f, 0xa4, 0x37, 0xf0, 0x9d, 0xd0, 0x61, 0x1d, 0x26,
0x5a, 0xfc, 0x3c, 0x2c, 0xc9, 0xaf, 0xc3, 0x92, 0xfc, 0x3e, 0x2c, 0xc9, 0xb7, 0x3f, 0xcb, 0x7b,
0x97, 0x03, 0xfd, 0x43, 0x79, 0xf6, 0x37, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x6d, 0x5d, 0xc9, 0x60,
0x04, 0x00, 0x00,
0x10, 0xa5, 0x63, 0x27, 0x71, 0x2a, 0x99, 0x51, 0xd4, 0xe2, 0x63, 0x21, 0x14, 0x59, 0x16, 0x0b,
0xaf, 0x32, 0x52, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xe8, 0x0c, 0xec, 0x3d, 0x49,
0x6b, 0xb0, 0xe4, 0x1f, 0xed, 0xb6, 0x98, 0x9c, 0x83, 0x0d, 0x37, 0x80, 0x0b, 0xb0, 0xe3, 0x00,
0x2c, 0x39, 0x02, 0x0a, 0x17, 0x41, 0xd5, 0xed, 0x8e, 0x1d, 0x16, 0x88, 0x5d, 0xbd, 0x57, 0x5d,
0xe5, 0xfa, 0xbc, 0x32, 0x4c, 0xca, 0xfa, 0x26, 0x4d, 0xb6, 0xf3, 0x52, 0x14, 0xb2, 0xa0, 0x4e,
0x92, 0x4b, 0x2e, 0xf2, 0x38, 0xf5, 0x43, 0x18, 0x84, 0x89, 0xcc, 0xe2, 0x92, 0x52, 0xb0, 0xc3,
0x44, 0x56, 0x2e, 0xf1, 0xac, 0xc0, 0x66, 0xca, 0xa6, 0x4f, 0xa1, 0xff, 0x42, 0x4a, 0x51, 0xb9,
0x3d, 0xcf, 0x0a, 0xc6, 0x8b, 0xf3, 0xb9, 0x89, 0x9b, 0x23, 0xcd, 0xb4, 0xd3, 0x9f, 0x83, 0xfd,
0x3a, 0x4e, 0x04, 0x9d, 0x82, 0xf5, 0x92, 0xef, 0x5d, 0xe2, 0x91, 0xc0, 0x66, 0x68, 0xd2, 0xfb,
0xd0, 0x5f, 0x16, 0x75, 0x2e, 0xdd, 0x9e, 0xe2, 0x34, 0xf0, 0x17, 0xe0, 0x6c, 0xea, 0x4c, 0xd9,
0x18, 0xb3, 0xa9, 0x33, 0x15, 0x63, 0x31, 0x34, 0x4f, 0x63, 0x2c, 0x13, 0xf3, 0x16, 0xac, 0x30,
0x91, 0xe8, 0x64, 0xc5, 0xc7, 0x68, 0xd5, 0x7c, 0x44, 0x03, 0xfa, 0x18, 0x9c, 0x65, 0x91, 0xd6,
0x59, 0x1e, 0xad, 0x9a, 0x2f, 0x1d, 0x31, 0x7d, 0x02, 0xa3, 0xeb, 0x24, 0xe3, 0x95, 0x8c, 0xb3,
0xd2, 0xb5, 0x54, 0xca, 0x96, 0xf0, 0xd7, 0x70, 0xa6, 0x5f, 0x62, 0x27, 0x1b, 0x2e, 0xe9, 0x39,
0xf4, 0x8e, 0xd9, 0x7b, 0xd1, 0xea, 0x3f, 0x27, 0xf0, 0x95, 0x80, 0x8d, 0x56, 0x77, 0x04, 0x23,
0x3d, 0x02, 0x0a, 0xf6, 0xf5, 0xbe, 0xe4, 0x4d, 0x5d, 0xca, 0xa6, 0x1e, 0x8c, 0x37, 0x52, 0x24,
0xf9, 0xed, 0xbb, 0x38, 0xad, 0xb9, 0xaa, 0x6a, 0xc4, 0xba, 0x14, 0x76, 0x14, 0xe5, 0x52, 0xbb,
0x6d, 0x55, 0xf4, 0x11, 0x63, 0x47, 0x61, 0x51, 0xa4, 0xda, 0xd9, 0xf7, 0x48, 0xe0, 0xb0, 0x96,
0xa0, 0x33, 0x80, 0xcb, 0xb4, 0x88, 0x9b, 0xd8, 0x81, 0x47, 0x02, 0xc2, 0x3a, 0x8c, 0x7f, 0x01,
0x43, 0xac, 0xf4, 0x55, 0x5c, 0xb6, 0xbd, 0x91, 0x7f, 0xf5, 0xf6, 0x9d, 0xc0, 0xe4, 0x4d, 0xcd,
0xc5, 0x9e, 0xf1, 0x0f, 0x35, 0xaf, 0xd4, 0x0e, 0x14, 0x6e, 0xba, 0xd4, 0x80, 0x3e, 0x84, 0xc1,
0x26, 0x4d, 0xb6, 0x5c, 0x4f, 0xca, 0x66, 0x0d, 0xc2, 0x5e, 0xdb, 0x09, 0x57, 0xaa, 0x57, 0x87,
0x75, 0x29, 0x8c, 0x64, 0x3c, 0x2b, 0xa4, 0x69, 0xa6, 0x41, 0xd4, 0x87, 0xc9, 0xfa, 0x6e, 0x9b,
0xd6, 0x3b, 0xae, 0x43, 0x07, 0xca, 0x7b, 0xc2, 0x61, 0xf6, 0x06, 0x2b, 0xed, 0x0e, 0x75, 0xf6,
0x0e, 0xe5, 0x7f, 0x22, 0x70, 0xd6, 0x94, 0x5f, 0x95, 0x45, 0x5e, 0x71, 0xdc, 0xd1, 0x5a, 0x08,
0xb3, 0xa3, 0xb5, 0x10, 0xf4, 0x02, 0x86, 0x8c, 0x57, 0x75, 0x2a, 0xcd, 0x9a, 0x1f, 0xb4, 0xa3,
0x30, 0xb1, 0x75, 0x2a, 0x99, 0x79, 0x45, 0x9f, 0xc3, 0xf9, 0x89, 0x6c, 0xb0, 0x2f, 0x8c, 0x7b,
0xd4, 0xc6, 0x9d, 0xf8, 0xd9, 0x5f, 0xcf, 0xfd, 0x6f, 0x04, 0xc6, 0x9d, 0xcc, 0x34, 0x30, 0x67,
0xa8, 0xca, 0x1a, 0x2f, 0xa6, 0x6d, 0x22, 0xcd, 0x33, 0x73, 0xa6, 0x13, 0x20, 0x57, 0x8d, 0x98,
0xc8, 0x15, 0xae, 0x10, 0x4f, 0xcf, 0x7c, 0xbf, 0xb3, 0x42, 0xa4, 0x99, 0x76, 0x52, 0x17, 0x86,
0xcb, 0xf7, 0x71, 0x7e, 0xcb, 0x77, 0x4a, 0x4c, 0x0e, 0x33, 0x90, 0xce, 0xdb, 0x53, 0x54, 0xd3,
0x1f, 0x2f, 0x68, 0x9b, 0xc2, 0x78, 0xd8, 0xf1, 0x8d, 0xff, 0x85, 0xc0, 0x59, 0x94, 0x95, 0x85,
0x90, 0x1d, 0x35, 0x44, 0xf9, 0x8e, 0xdf, 0x19, 0x35, 0x28, 0x80, 0xec, 0xa5, 0x88, 0x33, 0x2d,
0xfb, 0x11, 0xd3, 0x00, 0x59, 0xa5, 0x0a, 0xa5, 0x02, 0x9b, 0x69, 0xa0, 0xf6, 0x8f, 0x67, 0x5c,
0xb9, 0xb6, 0x56, 0x8e, 0x46, 0xa8, 0x73, 0x73, 0xc5, 0x95, 0xdb, 0x57, 0xae, 0x96, 0x40, 0x9d,
0x1f, 0xcf, 0x18, 0xb5, 0x61, 0x05, 0x16, 0xeb, 0x30, 0xe1, 0xf4, 0xc7, 0x61, 0x46, 0x7e, 0x1e,
0x66, 0xe4, 0xd7, 0x61, 0x46, 0x3e, 0xff, 0x9e, 0xdd, 0xbb, 0x19, 0xa8, 0x7f, 0xdf, 0xb3, 0x3f,
0x01, 0x00, 0x00, 0xff, 0xff, 0x27, 0x5d, 0xef, 0xb2, 0x0b, 0x05, 0x00, 0x00,
}

View file

@ -12,6 +12,11 @@ message Pair {
uint64 Count = 2;
}
message SumCount {
int64 Sum = 1;
int64 Count = 2;
}
message Bit {
uint64 RowID = 1;
uint64 ColumnID = 2;
@ -41,6 +46,8 @@ message QueryRequest {
repeated uint64 Slices = 2;
bool ColumnAttrs = 3;
bool Remote = 5;
bool ExcludeAttrs = 6;
bool ExcludeBits = 7;
}
message QueryResponse {
@ -53,6 +60,7 @@ message QueryResult {
Bitmap Bitmap = 1;
uint64 N = 2;
repeated Pair Pairs = 3;
SumCount SumCount = 5;
bool Changed = 4;
}

View file

@ -198,6 +198,16 @@ func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
return false
}
// HasConditionArg returns true if any arg is a conditional.
func (c *Call) HasConditionArg() bool {
for _, v := range c.Args {
if _, ok := v.(*Condition); ok {
return true
}
}
return false
}
// Condition represents an operation & value.
// When used in an argument map it represents a binary expression.
type Condition struct {

View file

@ -197,6 +197,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
if len(b.keys) == 0 {
return
}
skey := highbits(start)
ekey := highbits(end)
@ -208,31 +209,28 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end))))
}
// Count first partial container.
if i < 0 {
// start is before container, so we should start counting
// at first container that has value
if skey < b.keys[0] {
i = -1
} else {
i = -i
}
// start's container did not exist
// set i to the index of the first container we have with values higher than start
i = -i - 1
} else {
// Count first partial container and advance i so we don't recount it
n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1))
i += 1
}
// Count last container.
if j < 0 {
j = -j
if j > len(b.containers) {
j = len(b.containers)
}
// end's container did not exist
// set j to the index of the first container with values higher than end (or len(containers))
j = -j - 1
} else {
// end's container exists, count it up to end
n += uint64(b.containers[j].countRange(0, int(lowbits(end))))
}
// Count containers in between.
for x := i + 1; x < j; x++ {
for x := i; x < j; x++ {
n += uint64(b.containers[x].n)
}
@ -1994,6 +1992,10 @@ func intersectBitmapRun(a, b *container) *container {
if a.bitmapContains(i) {
output.array = append(output.array, i)
}
// If the run ends the container, break to avoid an infinite loop.
if i == 65535 {
break
}
}
}
output.n = len(output.array)
@ -2506,7 +2508,7 @@ func differenceRunBitmap(a, b *container) *container {
func differenceRunIterator(a *container, itr containerIterator) *container {
output := &container{runs: make([]interval16, 0, a.n)}
output := &container{runs: make([]interval16, 0, a.n), container_type: ContainerRun}
vb, eof := itr.next()
j := 0
@ -2574,7 +2576,7 @@ func differenceRunRun(a, b *container) *container {
alen := len(a.runs)
blen := len(b.runs)
output := &container{runs: make([]interval16, 0, alen+blen)} // TODO allocate max then truncate? or something else
output := &container{runs: make([]interval16, 0, alen+blen), container_type: ContainerRun} // TODO allocate max then truncate? or something else
// cardinality upper bound: sum of number of runs
// each B-run could split an A-run in two, up to len(b.runs) times
@ -2619,6 +2621,7 @@ func differenceRunRun(a, b *container) *container {
}
}
output.n = output.count()
return output
}
@ -2867,7 +2870,8 @@ func (*op) size() int { return 1 + 8 + 4 }
func highbits(v uint64) uint64 { return uint64(v >> 16) }
func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
// search32 returns the index of v in a.
// search32 returns the index of value in a. If value is not found, it works the
// same way as search64.
func search32(a []uint16, value uint16) int {
// Optimize for elements and the last element.
n := len(a)
@ -2904,7 +2908,13 @@ func search32(a []uint16, value uint16) int {
return -(lo + 1)
}
// search64 returns the index of v in a.
// search64 returns the index of value in a. If value is not found, -1 * (1 +
// the index where v would be if it were inserted) is returned. This is done in
// order to both signal that value was not found (negative number), and also
// return information about where v would go if it were inserted. The +1 offset
// is necessary due to the case where v is not found, but would go at index 0.
// since negative 0 is no different from positive 0, we offset the returned
// negative indices by 1. See the test for this function for examples.
func search64(a []uint64, value uint64) int {
// Optimize for elements and the last element.
n := len(a)
@ -3147,8 +3157,9 @@ func xorArrayRun(a, b *container) *container {
} else if va > vb.start {
if va < vb.last {
output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1})
vb.start = va + 1
i++
vb.start = va + 1
if vb.start > vb.last {
j++
}
@ -3157,15 +3168,22 @@ func xorArrayRun(a, b *container) *container {
j++
} else { // va == vb.last
vb.last--
if vb.start < vb.last {
if vb.start <= vb.last {
output.n += output.runAppendInterval(vb)
}
j++
i++
}
} else {
vb.start++
} else { // we know va == vb.start
if vb.start == maxContainerVal { // protect overflow
j++
} else {
vb.start++
if vb.start > vb.last {
j++
}
}
i++
}
}
@ -3213,9 +3231,15 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) {
r1 = interval16{start: x.va.start, last: x.vb.start - 1}
has_data = true
}
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
if x.vb.last == maxContainerVal { // Check for overflow
x.va_valid = false
} else {
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
x.va_valid = false
}
}
} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside
@ -3225,26 +3249,39 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) {
has_data = true
}
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
if x.va.last == maxContainerVal { //check for overflow
x.vb_valid = false
} else {
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
x.vb_valid = false
}
}
} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap
x.va_valid = false
r1 = interval16{start: x.va.start, last: x.vb.start - 1}
has_data = true
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
if x.va.last == maxContainerVal { // check for overflow
x.vb_valid = false
} else {
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
x.vb_valid = false
}
}
} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap
x.vb_valid = false
r1 = interval16{start: x.vb.start, last: x.va.start - 1}
has_data = true
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
if x.vb.last == maxContainerVal { // check for overflow
x.va_valid = false
} else {
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
x.va_valid = false
}
}
}
return

View file

@ -623,7 +623,7 @@ func TestIntersectBitmapRunArray(t *testing.T) {
},
{
bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
runs: []interval16{{start: 63, last: 10000}},
runs: []interval16{{start: 63, last: 10000}, {start: 65000, last: 65535}},
exp: []uint16{63, 64, 128, 192, 257, 259, 320, 384, 512},
expN: 9,
},
@ -1601,6 +1601,7 @@ func TestDifferenceRunRun(t *testing.T) {
aruns []interval16
bruns []interval16
exp []interval16
expn int
}{
{
// this tests all six overlap combinations
@ -1609,6 +1610,7 @@ func TestDifferenceRunRun(t *testing.T) {
aruns: []interval16{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}},
bruns: []interval16{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}},
exp: []interval16{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}},
expn: 13,
},
}
for i, test := range tests {
@ -1620,6 +1622,9 @@ func TestDifferenceRunRun(t *testing.T) {
if !reflect.DeepEqual(ret.runs, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
}
if ret.n != test.expn {
t.Fatalf("test #%v expected n=%v, but got n=%v", i, test.expn, ret.n)
}
}
}
@ -1711,31 +1716,43 @@ func TestWriteReadRun(t *testing.T) {
}
func TestXorArrayRun(t *testing.T) {
a := &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray}
b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
exp := []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
//ret := xorArrayRun(a, b)
ret := xor(a, b)
if !reflect.DeepEqual(ret.array, exp) {
t.Fatalf("test #1 expected %v, but got %v", exp, ret.array)
tests := []struct {
a *container
b *container
exp *container
}{
{
a: &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, container_type: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, container_type: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65534, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{65534}, container_type: ContainerArray, n: 1},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65535, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{}, container_type: ContainerArray, n: 0},
},
}
ret = xor(b, a)
if !reflect.DeepEqual(ret.array, exp) {
t.Fatalf("test #2 expected %v, but got %v", exp, ret.array)
}
c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}
// exp = []int16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
expr := []interval16{{start: 1, last: 4}, {start: 6, last: 9}, {start: 11, last: 11}, {start: 14, last: 16}}
ret = xor(b, c)
if !reflect.DeepEqual(ret.runs, expr) {
t.Fatalf("test #3 expected %v, but got %v", exp, ret.runs)
}
ret = xor(c, b)
if !reflect.DeepEqual(ret.runs, expr) {
t.Fatalf("test #4 expected %v, but got %v", exp, ret.array)
for i, test := range tests {
test.a.n = test.a.count()
test.b.n = test.b.count()
ret := xor(test.a, test.b)
if !reflect.DeepEqual(ret, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret)
}
ret = xor(test.b, test.a)
if !reflect.DeepEqual(ret, test.exp) {
t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret)
}
}
}
//special case that didn't fit the xorrunrun table testing below.
@ -1832,6 +1849,11 @@ func TestXorRunRun(t *testing.T) {
bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
},
{
aruns: []interval16{{start: 65530, last: 65535}},
bruns: []interval16{{start: 65532, last: 65535}},
exp: []interval16{{start: 65530, last: 65531}},
},
}
for i, test := range tests {
a.runs = test.aruns
@ -2310,3 +2332,81 @@ func Test_BufBitmapIterator_UnreadPanic(t *testing.T) {
itr.unread()
itr.unread()
}
func TestSearc64(t *testing.T) {
tests := []struct {
a []uint64
value uint64
exp int
}{
{
a: []uint64{1, 5, 10, 12},
value: 5,
exp: 1,
},
{
a: []uint64{1, 5, 10, 12},
value: 1,
exp: 0,
},
{
a: []uint64{1, 5, 10, 12},
value: 0,
exp: -1,
},
{
a: []uint64{1, 5, 10, 12},
value: 2,
exp: -2,
},
{
a: []uint64{1, 5, 10, 12},
value: 7,
exp: -3,
},
{
a: []uint64{1, 5, 10, 12},
value: 11,
exp: -4,
},
{
a: []uint64{1, 5, 10, 12},
value: 13,
exp: -5,
},
{
a: []uint64{1, 5, 10, 12},
value: 3843534,
exp: -5,
},
{
a: []uint64{},
value: 3843534,
exp: -1,
},
{
a: []uint64{},
value: 0,
exp: -1,
},
{
a: []uint64{0},
value: 0,
exp: 0,
},
{
a: []uint64{0},
value: 1,
exp: -2,
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%d in %v", test.value, test.a), func(t *testing.T) {
actual := search64(test.a, test.value)
if actual != test.exp {
t.Errorf("got: %d, exp: %d", actual, test.exp)
}
})
}
}

View file

@ -54,6 +54,104 @@ func TestContainerCount(t *testing.T) {
}
}
func TestCountRange(t *testing.T) {
tests := []struct {
name string
bitmap []uint64
start uint64
end uint64
exp uint64
}{
{
name: "j < 0 : 1",
bitmap: []uint64{0, 1, 2, 3 * 65536},
start: 0,
end: 65536,
exp: 3,
},
{
name: "i < 0 : 1",
bitmap: []uint64{0, 1, 2, 2 * 65536, 3 * 65536},
start: 65536,
end: 3 * 65536,
exp: 1,
},
{
name: "single-container-run",
bitmap: []uint64{0, 2, 3, 4, 5, 2 * 65536, 3 * 65536},
start: 2,
end: 5,
exp: 3,
},
{
name: "single-container-beg",
bitmap: []uint64{1, 2, 3, 4, 5, 2 * 65536, 3 * 65536},
start: 1,
end: 4,
exp: 3,
},
{
name: "partial-start",
bitmap: []uint64{1, 2, 3, 4, 5, 2 * 65536, 3 * 65536},
start: 5,
end: 3 * 65536,
exp: 2,
},
{
name: "partial-end",
bitmap: []uint64{1, 2 * 65536, 3 * 65536, 3*65536 + 1, 3*65536 + 2},
start: 0,
end: (3 * 65536) + 1,
exp: 3,
},
{
name: "partial-both",
bitmap: []uint64{65536, 65537, 65538, 2 * 65536, 2*65536 + 1, 2*65536 + 2},
start: 65537,
end: (2 * 65536) + 1,
exp: 3,
},
{
name: "partial-both-bookends",
bitmap: []uint64{0, 65535, 65536, 65537, 65538, 2 * 65536, 2*65536 + 1, 2*65536 + 2, 3 * 65536},
start: 65537,
end: (2 * 65536) + 1,
exp: 3,
},
{
name: "empty-bookends",
bitmap: []uint64{1, 65535, 5 * 65536, 5*65536 + 1},
start: 65536,
end: 5 * 65536,
exp: 0,
},
{
name: "i not found, j found",
bitmap: []uint64{1, 65535, 5 * 65536},
start: 2 * 65535,
end: 5*65536 + 1,
exp: 1,
},
{
name: "i not found, j not found",
bitmap: []uint64{1, 65535, 5 * 65536, 7 * 65536},
start: 2 * 65535,
end: 6 * 65536,
exp: 1,
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%s: %d to %d in '%v'", test.name, test.start, test.end, test.bitmap), func(t *testing.T) {
b := roaring.NewBitmap(test.bitmap...)
actual := b.CountRange(test.start, test.end)
if actual != test.exp {
t.Errorf("got: %d, exp: %d", actual, test.exp)
}
})
}
}
func TestCheckBitmap(t *testing.T) {
b := roaring.NewBitmap()
x := 0

View file

@ -339,6 +339,15 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err != nil {
return err
}
case *internal.DeleteViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
}
err := f.DeleteView(obj.View)
if err != nil {
return err
}
}
return nil
}

View file

@ -43,12 +43,9 @@ func (h *Holder) Close() error {
return h.Holder.Close()
}
// Reopen closes the holder and instantiates and opens a new holder.
// Reopen instantiates and opens a new holder.
// Note that the holder must be Closed first.
func (h *Holder) Reopen() error {
if err := h.Holder.Close(); err != nil {
return err
}
path, logOutput := h.Path, h.Holder.LogOutput
h.Holder = pilosa.NewHolder()
h.Holder.Path = path

20
view.go
View file

@ -25,6 +25,7 @@ import (
"sync"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
)
// View layout modes.
@ -162,7 +163,9 @@ func (v *View) Close() error {
// Close all fragments.
for _, frag := range v.fragments {
_ = frag.Close()
if err := frag.Close(); err != nil {
return err
}
}
v.fragments = make(map[uint64]*Fragment)
@ -300,8 +303,21 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan
return frag.SetFieldValue(columnID, bitDepth, value)
}
// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
for _, f := range v.Fragments() {
fsum, fcount, err := f.FieldSum(filter, bitDepth)
if err != nil {
return sum, count, err
}
sum += fsum
count += fcount
}
return sum, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
bm := NewBitmap()
for _, frag := range v.Fragments() {
other, err := frag.FieldRange(op, bitDepth, predicate)