From 38e3ce10fe626a257c4b9fcfa2ffe421923a5c71 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 31 Aug 2018 10:32:33 -0500 Subject: [PATCH 01/45] removing bounds check --- roaring/roaring.go | 103 +++++++++++++++++++++++-------- roaring/roaring_internal_test.go | 4 +- 2 files changed, 78 insertions(+), 29 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index e7333325c..c39c8b2e0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2136,15 +2136,26 @@ func intersectArrayBitmap(a, b *Container) *Container { } func intersectBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + var ( - for i := range a.bitmap { - v := a.bitmap[i] & b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob =buf[:bitmapN] + n int +) + for i := 0; i < bitmapN; i++ { + v := ab[i] & bb[i] + ob[i] = v + n += int(popcount(v)) + } + + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, } - output.optimize() return output } @@ -2434,17 +2445,26 @@ func unionArrayBitmap(a, b *Container) *Container { } func unionBitmapBitmap(a, b *Container) *Container { - output := &Container{ - bitmap: make([]uint64, bitmapN), - containerType: containerBitmap, - } + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob =buf[:bitmapN] + + n int + ) for i := 0; i < bitmapN; i++ { - v := a.bitmap[i] | b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) + v := ab[i] | bb[i] + ob[i] = v + n += int(popcount(v)) } + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, + } return output } @@ -2780,13 +2800,25 @@ func differenceBitmapArray(a, b *Container) *Container { } func differenceBitmapBitmap(a, b *Container) *Container { - output := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap} + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob =buf[:bitmapN] - for i := range a.bitmap { - v := a.bitmap[i] & (^b.bitmap[i]) - output.bitmap[i] = v - output.n += int(popcount(v)) + n int + ) + for i := 0; i < bitmapN; i++ { + v := ab[i] & (^bb[i]) + ob[i] = v + n += int(popcount(v)) + } + + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, } if output.n < ArrayMaxSize { output.bitmapToArray() @@ -2871,16 +2903,26 @@ func xorArrayBitmap(a, b *Container) *Container { } func xorBitmapBitmap(a, b *Container) *Container { - output := &Container{ - bitmap: make([]uint64, bitmapN), - containerType: containerBitmap, - } + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob =buf[:bitmapN] + + n int + ) + for i := 0; i < bitmapN; i++ { - v := a.bitmap[i] ^ b.bitmap[i] - output.bitmap[i] = v - output.n += int(popcount(v)) + v := ab[i] ^ bb[i] + ob[i] = v + n += int(popcount(v)) } + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, + } if output.count() < ArrayMaxSize { output.bitmapToArray() } @@ -3333,9 +3375,16 @@ func popcount(x uint64) uint64 { } func popcountAndSlice(s, m []uint64) uint64 { + var ( + a=s[:bitmapN] + b=m[:bitmapN] + ) + _ = a[bitmapN-1] + _ = b[bitmapN-1] + cnt := uint64(0) - for i := range s { - cnt += popcount(s[i] & m[i]) + for i:=0;i Date: Fri, 14 Sep 2018 10:27:33 -0500 Subject: [PATCH 02/45] updated comments and gofmt --- roaring/roaring.go | 67 ++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index c39c8b2e0..c98c13621 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2136,15 +2136,15 @@ func intersectArrayBitmap(a, b *Container) *Container { } func intersectBitmapBitmap(a, b *Container) *Container { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html var ( - - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] - buf = make([]uint64, bitmapN) - ob =buf[:bitmapN] - - n int -) + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + n int + ) for i := 0; i < bitmapN; i++ { v := ab[i] & bb[i] ob[i] = v @@ -2152,8 +2152,8 @@ func intersectBitmapBitmap(a, b *Container) *Container { } output := &Container{ - bitmap: ob, - n: n, + bitmap: ob, + n: n, containerType: containerBitmap, } return output @@ -2445,11 +2445,14 @@ func unionArrayBitmap(a, b *Container) *Container { } func unionBitmapBitmap(a, b *Container) *Container { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] buf = make([]uint64, bitmapN) - ob =buf[:bitmapN] + ob = buf[:bitmapN] n int ) @@ -2461,8 +2464,8 @@ func unionBitmapBitmap(a, b *Container) *Container { } output := &Container{ - bitmap: ob, - n: n, + bitmap: ob, + n: n, containerType: containerBitmap, } return output @@ -2800,11 +2803,14 @@ func differenceBitmapArray(a, b *Container) *Container { } func differenceBitmapBitmap(a, b *Container) *Container { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] buf = make([]uint64, bitmapN) - ob =buf[:bitmapN] + ob = buf[:bitmapN] n int ) @@ -2816,8 +2822,8 @@ func differenceBitmapBitmap(a, b *Container) *Container { } output := &Container{ - bitmap: ob, - n: n, + bitmap: ob, + n: n, containerType: containerBitmap, } if output.n < ArrayMaxSize { @@ -2903,11 +2909,14 @@ func xorArrayBitmap(a, b *Container) *Container { } func xorBitmapBitmap(a, b *Container) *Container { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + var ( - ab = a.bitmap[:bitmapN] - bb = b.bitmap[:bitmapN] + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] buf = make([]uint64, bitmapN) - ob =buf[:bitmapN] + ob = buf[:bitmapN] n int ) @@ -2919,8 +2928,8 @@ func xorBitmapBitmap(a, b *Container) *Container { } output := &Container{ - bitmap: ob, - n: n, + bitmap: ob, + n: n, containerType: containerBitmap, } if output.count() < ArrayMaxSize { @@ -3376,14 +3385,14 @@ func popcount(x uint64) uint64 { func popcountAndSlice(s, m []uint64) uint64 { var ( - a=s[:bitmapN] - b=m[:bitmapN] + a = s[:bitmapN] + b = m[:bitmapN] ) _ = a[bitmapN-1] _ = b[bitmapN-1] - + cnt := uint64(0) - for i:=0;i Date: Fri, 14 Sep 2018 14:46:02 -0500 Subject: [PATCH 03/45] code cleanup --- roaring/roaring.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 14be4ea0b..87b6eac3d 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2148,9 +2148,8 @@ func intersectBitmapBitmap(a, b *Container) *Container { n int ) for i := 0; i < bitmapN; i++ { - v := ab[i] & bb[i] - ob[i] = v - n += int(popcount(v)) + ob[i] = ab[i] & bb[i] + n += int(popcount(ob[i])) } output := &Container{ @@ -2460,9 +2459,8 @@ func unionBitmapBitmap(a, b *Container) *Container { ) for i := 0; i < bitmapN; i++ { - v := ab[i] | bb[i] - ob[i] = v - n += int(popcount(v)) + ob[i] =ab[i] | bb[i] + n += int(popcount(ob[i])) } output := &Container{ @@ -2818,9 +2816,8 @@ func differenceBitmapBitmap(a, b *Container) *Container { ) for i := 0; i < bitmapN; i++ { - v := ab[i] & (^bb[i]) - ob[i] = v - n += int(popcount(v)) + ob[i] = ab[i] & (^bb[i]) + n += int(popcount(ob[i])) } output := &Container{ @@ -2924,9 +2921,8 @@ func xorBitmapBitmap(a, b *Container) *Container { ) for i := 0; i < bitmapN; i++ { - v := ab[i] ^ bb[i] - ob[i] = v - n += int(popcount(v)) + ob[i] = ab[i] ^ bb[i] + n += int(popcount(ob[i])) } output := &Container{ From fe8756927cd6e6bcd9bf4b57e8a90ba944750b9f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 14 Sep 2018 15:40:37 -0500 Subject: [PATCH 04/45] manual gofmt --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 87b6eac3d..5e82364b1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2459,7 +2459,7 @@ func unionBitmapBitmap(a, b *Container) *Container { ) for i := 0; i < bitmapN; i++ { - ob[i] =ab[i] | bb[i] + ob[i] = ab[i] | bb[i] n += int(popcount(ob[i])) } From 7a32745b28e3e5ddd48d65e7ab75950b84e0c987 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 19 Sep 2018 12:35:00 -0500 Subject: [PATCH 05/45] Make translate map size configurable. --- ctl/server.go | 1 + holder.go | 23 +++++++++++++++++-- server.go | 12 ++++++++++ server/config.go | 3 ++- server/server.go | 10 +++++++++ test/pilosa.go | 15 +++++++++++-- translate.go | 33 +++++++++++++++++++++++++--- translate_mapsize_386.go | 5 ----- translate_mapsize_all64bitsystems.go | 8 ------- 9 files changed, 89 insertions(+), 21 deletions(-) delete mode 100644 translate_mapsize_386.go delete mode 100644 translate_mapsize_all64bitsystems.go diff --git a/ctl/server.go b/ctl/server.go index 90fdfb46e..194633a48 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") + flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size of mmap to allocate for key translation.") // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") diff --git a/holder.go b/holder.go index 794104e53..5cffeea9b 100644 --- a/holder.go +++ b/holder.go @@ -77,9 +77,19 @@ type Holder struct { Logger Logger } +// HolderOption is a functional option type for pilosa.Holder +type HolderOption func(f *Holder) error + +func OptHolderTranslateFileMapSize(mapSize int) HolderOption { + return func(h *Holder) error { + h.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) + return nil + } +} + // NewHolder returns a new instance of Holder. -func NewHolder() *Holder { - return &Holder{ +func NewHolder(opts ...HolderOption) *Holder { + h := &Holder{ indexes: make(map[string]*Index), closing: make(chan struct{}), @@ -97,6 +107,15 @@ func NewHolder() *Holder { Logger: NopLogger, } + + for _, opt := range opts { + err := opt(h) + if err != nil { + // TODO (2.0): Change func signature to return error + panic(errors.Wrap(err, "applying option")) + } + } + return h } // Open initializes the root data directory for the holder. diff --git a/server.go b/server.go index cde1dd406..708fd853b 100644 --- a/server.go +++ b/server.go @@ -234,6 +234,18 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } +func OptServerHolderOptions(opts ...HolderOption) ServerOption { + return func(s *Server) error { + for _, opt := range opts { + err := opt(s.holder) + if err != nil { + return errors.Wrap(err, "applying option") + } + } + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ diff --git a/server/config.go b/server/config.go index 8367eb835..d255e4bb6 100644 --- a/server/config.go +++ b/server/config.go @@ -71,8 +71,9 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` - // DEPRECATED: Translation config supports translation store replication. Translation struct { + MapSize int `toml:"map-size"` + // DEPRECATED: Translation config supports translation store replication. PrimaryURL string `toml:"primary-url"` } `toml:"translation"` diff --git a/server/server.go b/server/server.go index 4fdd711fe..61942522f 100644 --- a/server/server.go +++ b/server/server.go @@ -283,6 +283,16 @@ func (m *Command) SetupServer() error { coordinatorOpt, } + if m.Config.Translation.MapSize > 0 { + serverOptions = append( + serverOptions, + pilosa.OptServerHolderOptions( + pilosa.OptHolderTranslateFileMapSize( + m.Config.Translation.MapSize, + ), + ), + ) + } serverOptions = append(serverOptions, m.serverOptions...) m.Server, err = pilosa.NewServer(serverOptions...) diff --git a/test/pilosa.go b/test/pilosa.go index 9dbd78057..835291cbd 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -55,11 +55,22 @@ func newCommand(opts ...server.CommandOption) *Command { panic(err) } - // set aggressive close timeout by default to avoid hanging tests. This was + // Set aggressive close timeout by default to avoid hanging tests. This was // a problem with PDK tests which used go-pilosa as well. We put it at the // beginning of the option slice so that it can be overridden by user-passed // options. - opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...) + // Also set TranslateFile MapSize to a smaller number so memory allocation + // does not fail on 32-bit systems. + opts = append([]server.CommandOption{ + server.OptCommandCloseTimeout(time.Millisecond * 2), + server.OptCommandServerOptions( + pilosa.OptServerHolderOptions( + pilosa.OptHolderTranslateFileMapSize( + 2 << 26, + ), + ), + ), + }, opts...) m := &Command{commandOptions: opts} m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) m.Config.DataDir = path diff --git a/translate.go b/translate.go index 41f3b4852..f1eddfbfe 100644 --- a/translate.go +++ b/translate.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "io" "io/ioutil" @@ -17,6 +16,7 @@ import ( "time" "github.com/cespare/xxhash" + "github.com/pkg/errors" ) const ( @@ -81,9 +81,26 @@ type TranslateFile struct { replicationRetryInterval time.Duration } +// TranslateFileOption is a functional option type for pilosa.TranslateFile +type TranslateFileOption func(f *TranslateFile) error + +func OptTranslateFileMapSize(mapSize int) TranslateFileOption { + return func(f *TranslateFile) error { + f.mapSize = mapSize + return nil + } +} + // NewTranslateFile returns a new instance of TranslateFile. -func NewTranslateFile() *TranslateFile { - return &TranslateFile{ +func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { + // 10GB default map size + defaultMapSize := 10 * (1 << 30) + // Use 2GB default map size on 32-bit systems + if 32<<(^uint(0)>>32&1) == 32 { + defaultMapSize = (1 << 31) - 1 // 2GB + } + + f := &TranslateFile{ writeNotify: make(chan struct{}), closing: make(chan struct{}), cols: make(map[string]*index), @@ -96,6 +113,16 @@ func NewTranslateFile() *TranslateFile { replicationRetryInterval: defaultReplicationRetryInterval, } + + for _, opt := range opts { + err := opt(f) + if err != nil { + // TODO (2.0): Change func signature to return error + panic(errors.Wrap(err, "applying option")) + } + } + + return f } func (s *TranslateFile) Open() (err error) { diff --git a/translate_mapsize_386.go b/translate_mapsize_386.go deleted file mode 100644 index 259b735ac..000000000 --- a/translate_mapsize_386.go +++ /dev/null @@ -1,5 +0,0 @@ -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so must be < 2^31 -const defaultMapSize = (1 << 31) - 1 // 2GB diff --git a/translate_mapsize_all64bitsystems.go b/translate_mapsize_all64bitsystems.go deleted file mode 100644 index 605d6270a..000000000 --- a/translate_mapsize_all64bitsystems.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !386 - -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on -// 64bit systems. -const defaultMapSize = 10 * (1 << 30) // 10GB From fcdc3b742708b66101a0d604a50e9c997f49033b Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 19 Sep 2018 20:54:06 +0300 Subject: [PATCH 06/45] trivial comment fix --- roaring/roaring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index c3f2cc3db..c17c297f5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -46,7 +46,7 @@ const ( // at the beginning of every serialized run container. runCountHeaderSize = 2 - // interval32Size is the size of a single run in a container.runs. + // interval16Size is the size of a single run in a container.runs. interval16Size = 4 // bitmapN is the number of values in a container.bitmap. @@ -1698,7 +1698,7 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) { // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) //} - // Write sizeof(uint32) * cardinality bytes. + // Write sizeof(uint16) * cardinality bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n]) return int64(nn), err } From 2394d3610848aac94355c7ebeb81dfc9ccaec303 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 19 Sep 2018 13:30:22 -0500 Subject: [PATCH 07/45] Adjust tests, fix 32-bit config --- cmd/server_test.go | 8 +++++--- server/server_test.go | 2 ++ test/pilosa.go | 2 +- translate.go | 14 +++++++++----- translate_test.go | 6 +++--- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index e58f8af4d..84b2297d4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) { tests := []commandTest{ // TEST 0 { - args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, + args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"}, env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" @@ -65,13 +65,14 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) + v.Check(cmd.Server.Config.Translation.MapSize, 100000) return v.Error() }, }, // TEST 1 { args: []string{"server", "--anti-entropy.interval", "9m0s"}, - env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110"}, + env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"}, cfgFileContent: ` bind = "localhost:0" data-dir = "` + actualDataDir + `" @@ -85,12 +86,13 @@ func TestServerConfig(t *testing.T) { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) + v.Check(cmd.Server.Config.Translation.MapSize, 100000) return v.Error() }, }, // TEST 2 { - args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, + args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true", "--translation.map-size", "100000"}, env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" diff --git a/server/server_test.go b/server/server_test.go index ab860dc1b..a15bb65e4 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -403,6 +403,7 @@ func TestClusteringNodesReplica1(t *testing.T) { // Create new main with the same config. config := cluster[2].Command.Config + config.Translation.MapSize = 100000 // config.Bind = cluster[2].API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster @@ -476,6 +477,7 @@ func TestClusteringNodesReplica2(t *testing.T) { // Create new main with the same config. config := cluster[2].Command.Config + config.Translation.MapSize = 100000 // config.Bind = cluster[2].API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster diff --git a/test/pilosa.go b/test/pilosa.go index 835291cbd..a2e6c1c58 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -66,7 +66,7 @@ func newCommand(opts ...server.CommandOption) *Command { server.OptCommandServerOptions( pilosa.OptServerHolderOptions( pilosa.OptHolderTranslateFileMapSize( - 2 << 26, + 2 << 25, ), ), ), diff --git a/translate.go b/translate.go index f1eddfbfe..fc4a9d3c7 100644 --- a/translate.go +++ b/translate.go @@ -93,11 +93,15 @@ func OptTranslateFileMapSize(mapSize int) TranslateFileOption { // NewTranslateFile returns a new instance of TranslateFile. func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { - // 10GB default map size - defaultMapSize := 10 * (1 << 30) - // Use 2GB default map size on 32-bit systems - if 32<<(^uint(0)>>32&1) == 32 { - defaultMapSize = (1 << 31) - 1 // 2GB + var defaultMapSize64 int64 = 1 << 33 + var defaultMapSize int + + if ^uint(0)>>32 > 0 { + // 10GB default map size + defaultMapSize = int(defaultMapSize64) + } else { + // Use 2GB default map size on 32-bit systems + defaultMapSize = (1 << 31) - 1 } f := &TranslateFile{ diff --git a/translate_test.go b/translate_test.go index 706fe365f..1aea26cfe 100644 --- a/translate_test.go +++ b/translate_test.go @@ -367,7 +367,7 @@ func TestPrintTranslateFile(t *testing.T) { } f.Close() - s := pilosa.NewTranslateFile() + s := pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25)) s.Path = f.Name() err = s.Open() if err != nil { @@ -809,7 +809,7 @@ func NewTranslateFile() *TranslateFile { } f.Close() - s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25))} s.Path = f.Name() return s } @@ -847,7 +847,7 @@ func (s *TranslateFile) Reopen() error { } s.lock.Lock() - s.TranslateFile = pilosa.NewTranslateFile() + s.TranslateFile = pilosa.NewTranslateFile(pilosa.OptTranslateFileMapSize(2 << 25)) s.lock.Unlock() s.Path = prev.Path s.SetPrimaryStore("restored-primary", prev.PrimaryTranslateStore) From ebc4d7fc13458e11e33a2f725cccbb0ec42d4dfa Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 19 Sep 2018 13:37:42 -0500 Subject: [PATCH 08/45] Set low map size for 32-bit --- server/server_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/server_test.go b/server/server_test.go index a15bb65e4..faca9c9d0 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -498,6 +498,7 @@ func TestClusteringNodesReplica2(t *testing.T) { // Create new main with the same config. config = cluster[1].Command.Config // config.Bind = cluster[1].API.Node().URI.HostPort() + config.Translation.MapSize = 100000 // this isn't necessary, but makes the test run way faster config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port)) From 42293c58a5ee088e88d51aba32be80f1bf8fa497 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 19 Sep 2018 14:11:33 -0500 Subject: [PATCH 09/45] Do not run prerelease in CI if this is a pull request --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5bf059e2..b42b2aa64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,6 +68,7 @@ jobs: docker: - image: circleci/python:2.7-jessie steps: + - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR - *fast-checkout - run: sudo pip install awscli - run: make prerelease-upload From e7481f4fd2e6446aba020933698a4bc3b4f96e92 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 18 Sep 2018 16:38:24 -0500 Subject: [PATCH 10/45] treat import timestamps as UTC --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index 7736b12e5..a8783f502 100644 --- a/api.go +++ b/api.go @@ -732,7 +732,7 @@ func (api *API) Import(_ context.Context, req *ImportRequest) error { if ts == 0 { continue } - t := time.Unix(0, ts) + t := time.Unix(0, ts).UTC() timestamps[i] = &t } From b86478c613409ab8258b5bd2d364f61d9acd7b09 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 19 Sep 2018 16:14:34 -0500 Subject: [PATCH 11/45] test to ensure views match UTC time --- server/server_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/server/server_test.go b/server/server_test.go index ab860dc1b..67440c21e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "math/rand" "reflect" "sort" @@ -564,4 +565,57 @@ func TestRemoveNodeAfterItDies(t *testing.T) { } } +// Ensure program imports timestamps as UTC. +func TestMain_ImportTimestamp(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + indexName := "i" + fieldName := "f" + + // Create index. + if _, err := m.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + + // Create field. + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil { + t.Fatal(err) + } + + data := pilosa.ImportRequest{ + Index: indexName, + Field: fieldName, + Shard: 0, + RowIDs: []uint64{1, 2}, + ColumnIDs: []uint64{1, 2}, + Timestamps: []int64{1514764800000000000, 1577833200000000000}, // 2018-01-01T00:00, 2019-12-31T23:00 + } + + // Import data. + if err := m.API.Import(context.Background(), &data); err != nil { + t.Fatal(err) + } + + // Ensure the correct views were created. + dir := fmt.Sprintf("%s/%s/%s/views", m.Config.DataDir, indexName, fieldName) + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + exp := []string{ + "standard", "standard_2018", "standard_201801", "standard_20180101", + "standard_2019", "standard_201912", "standard_20191231", + } + got := []string{} + for _, f := range files { + got = append(got, f.Name()) + } + + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected %v, but got %v", exp, got) + } +} + // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address. From 995a24d0afbe556dbfd00a316c695f5f07c88c11 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 19 Sep 2018 15:05:16 -0500 Subject: [PATCH 12/45] ensure mutex imports unset previous columns --- fragment.go | 121 +++++++++++++++++++++++++++++++++++--- fragment_internal_test.go | 70 ++++++++++++++++++++++ 2 files changed, 182 insertions(+), 9 deletions(-) diff --git a/fragment.go b/fragment.go index 7f651b135..7e3cd26d9 100644 --- a/fragment.go +++ b/fragment.go @@ -1330,16 +1330,29 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } + if f.mutexVector != nil { + return f.bulkImportMutex(rowIDs, columnIDs) + } + return f.bulkImportStandard(rowIDs, columnIDs) +} + +// bulkImportStandard performs a bulk import on a standard fragment. +func (f *fragment) bulkImportStandard(rowIDs, columnIDs []uint64) error { // Create a temporary bitmap which will be populated by rowIDs and columnIDs // and then merged into the existing fragment's bitmap. localBitmap := roaring.NewBitmap() + // Disconnect op writer so we don't append updates. localBitmap.OpWriter = nil - // Process every bit. - // If an error occurs then reopen the storage. - lastID := uint64(0) + // rowSet maintains the set of rowIDs present in this import. + // It allows the cache to be updated once per row, instead of once + // per bit. rowSet := make(map[uint64]struct{}) + lastRowID := uint64(0) + + // Process every bit by writing to a local bitmap, + // to be merged with fragment storage next. for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] @@ -1349,18 +1362,18 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { return err } - // Write to storage. + // Write to local storage. _, err = localBitmap.Add(pos) if err != nil { return err } + // Reduce the StatsD rate for high volume stats f.stats.Count("ImportBit", 1, 0.0001) - // import optimization to avoid linear foreach calls - // slight risk of concurrent cache counter being off but - // no real danger - if i == 0 || rowID != lastID { - lastID = rowID + + // Add row to rowSet. + if i == 0 || rowID != lastRowID { + lastRowID = rowID rowSet[rowID] = struct{}{} } @@ -1389,6 +1402,96 @@ func (f *fragment) bulkImport(rowIDs, columnIDs []uint64) error { return unprotectedWriteToFragment(f, results) } +// bulkImportMutex performs a bulk import on a fragment while ensuring +// mutex restrictions. Because the mutex requirements must be checked +// against storage, this method must acquire a write lock on the fragment +// during the entire process, and it handles every bit independently. +func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Disconnect op writer so we don't append updates. + f.storage.OpWriter = nil + + // If an error occurs then reopen the storage. + if err := func() error { + // rowSet maintains the set of rowIDs present in this import. + // It allows the cache to be updated once per row, instead of once + // per bit. + rowSet := make(map[uint64]struct{}) + lastRowID := uint64(0) + + // Process every bit. + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] + + // Handle mutex vector (i.e. clear an existing row). + if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID { + // Determine the position of the bit in the storage. + pos, err := f.pos(existingRowID, columnID) + if err != nil { + return err + } + + // Clear storage. + _, err = f.storage.Remove(pos) + if err != nil { + return err + } + + rowSet[existingRowID] = struct{}{} + } + + // Determine the position of the bit in the storage. + pos, err := f.pos(rowID, columnID) + if err != nil { + return err + } + + // Write to storage. + _, err = f.storage.Add(pos) + if err != nil { + return err + } + + // Reduce the StatsD rate for high volume stats + f.stats.Count("ImportBit", 1, 0.0001) + + // Add row to rowSet. + if i == 0 || rowID != lastRowID { + lastRowID = rowID + rowSet[rowID] = struct{}{} + } + + // Invalidate block checksum. + delete(f.checksums, int(rowID/HashBlockSize)) + } + + // Update cache counts for all rows. + for rowID := range rowSet { + // Import should ALWAYS have row() load a new bm from fragment.storage + // because the row that's in rowCache hasn't been updated with + // this import's data. + f.cache.BulkAdd(rowID, f.unprotectedRow(rowID).Count()) + } + + f.cache.Invalidate() + + return nil + }(); err != nil { + _ = f.closeStorage() + _ = f.openStorage() + return err + } + + // Write the storage to disk and reload. + if err := f.snapshot(); err != nil { + return err + } + + return nil +} + // importValue bulk imports a set of range-encoded values. func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() diff --git a/fragment_internal_test.go b/fragment_internal_test.go index dd2d00b41..e0865c606 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1178,6 +1178,76 @@ func TestFragment_SetMutex(t *testing.T) { } } +// Ensure a fragment can import mutually exclusive values. +func TestFragment_ImportMutex(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + exp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 2, 2}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 1: {0, 2, 3}, + 2: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {8}, + }, + }, + { + []uint64{1, 2, 3}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 1: {}, + 2: {}, + 3: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + // Check for expected results. + for k, v := range test.exp { + cols := f.row(k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func BenchmarkFragment_Snapshot(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") From 3ca7944fe33f6422789372ddd82c94a0b0c1636c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 20 Sep 2018 12:37:28 -0500 Subject: [PATCH 13/45] remove unecessary lines confirmed that bounds checks are still avoided by go test -gcflags="-d=ssa/check_bce/debug=1" ./roaring.go:3387:8: Found IsSliceInBounds ./roaring.go:3388:8: Found IsSliceInBounds ./roaring.go:3436:21: Found IsSliceInBounds --- roaring/roaring.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 4a5eea5fc..6eaad740d 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3387,8 +3387,6 @@ func popcountAndSlice(s, m []uint64) uint64 { a = s[:bitmapN] b = m[:bitmapN] ) - _ = a[bitmapN-1] - _ = b[bitmapN-1] cnt := uint64(0) for i := 0; i < bitmapN; i++ { From 52ba336461be7d5ec2f4caba98cf4b38ce05ddda Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 20 Sep 2018 16:05:28 -0500 Subject: [PATCH 14/45] Address code review feedback --- ctl/server.go | 2 +- docs/configuration.md | 12 ++++++++++++ holder.go | 23 ++--------------------- server.go | 9 ++------- server/server.go | 6 ++---- test/pilosa.go | 6 ++---- translate.go | 3 +-- 7 files changed, 22 insertions(+), 39 deletions(-) diff --git a/ctl/server.go b/ctl/server.go index 194633a48..7f384ec7b 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -45,7 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") - flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size of mmap to allocate for key translation.") + flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size in bytes of mmap to allocate for key translation.") // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") diff --git a/docs/configuration.md b/docs/configuration.md index 22a67ddcf..5750441ef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -307,6 +307,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h skip-verify = true ``` +#### Translation Map Size + +* Description: Size in bytes of mmap to allocate for key translation +* Flag: `translation.map-size` +* Env: `PILOSA_TRANSLATION_MAP_SIZE` +* Config: + + ```toml + [translation] + map-size = 10737418240 + ``` + ### Example Cluster Configuration A three node cluster running on different hosts could be minimally configured as follows: diff --git a/holder.go b/holder.go index 5cffeea9b..794104e53 100644 --- a/holder.go +++ b/holder.go @@ -77,19 +77,9 @@ type Holder struct { Logger Logger } -// HolderOption is a functional option type for pilosa.Holder -type HolderOption func(f *Holder) error - -func OptHolderTranslateFileMapSize(mapSize int) HolderOption { - return func(h *Holder) error { - h.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) - return nil - } -} - // NewHolder returns a new instance of Holder. -func NewHolder(opts ...HolderOption) *Holder { - h := &Holder{ +func NewHolder() *Holder { + return &Holder{ indexes: make(map[string]*Index), closing: make(chan struct{}), @@ -107,15 +97,6 @@ func NewHolder(opts ...HolderOption) *Holder { Logger: NopLogger, } - - for _, opt := range opts { - err := opt(h) - if err != nil { - // TODO (2.0): Change func signature to return error - panic(errors.Wrap(err, "applying option")) - } - } - return h } // Open initializes the root data directory for the holder. diff --git a/server.go b/server.go index 708fd853b..ce6319f45 100644 --- a/server.go +++ b/server.go @@ -234,14 +234,9 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } -func OptServerHolderOptions(opts ...HolderOption) ServerOption { +func OptServerTranslateFileMapSize(mapSize int) ServerOption { return func(s *Server) error { - for _, opt := range opts { - err := opt(s.holder) - if err != nil { - return errors.Wrap(err, "applying option") - } - } + s.holder.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) return nil } } diff --git a/server/server.go b/server/server.go index 61942522f..b4ac595fa 100644 --- a/server/server.go +++ b/server/server.go @@ -286,10 +286,8 @@ func (m *Command) SetupServer() error { if m.Config.Translation.MapSize > 0 { serverOptions = append( serverOptions, - pilosa.OptServerHolderOptions( - pilosa.OptHolderTranslateFileMapSize( - m.Config.Translation.MapSize, - ), + pilosa.OptServerTranslateFileMapSize( + m.Config.Translation.MapSize, ), ) } diff --git a/test/pilosa.go b/test/pilosa.go index a2e6c1c58..f8f60eebe 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -64,10 +64,8 @@ func newCommand(opts ...server.CommandOption) *Command { opts = append([]server.CommandOption{ server.OptCommandCloseTimeout(time.Millisecond * 2), server.OptCommandServerOptions( - pilosa.OptServerHolderOptions( - pilosa.OptHolderTranslateFileMapSize( - 2 << 25, - ), + pilosa.OptServerTranslateFileMapSize( + 2 << 25, ), ), }, opts...) diff --git a/translate.go b/translate.go index fc4a9d3c7..2ef289f45 100644 --- a/translate.go +++ b/translate.go @@ -93,7 +93,7 @@ func OptTranslateFileMapSize(mapSize int) TranslateFileOption { // NewTranslateFile returns a new instance of TranslateFile. func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { - var defaultMapSize64 int64 = 1 << 33 + var defaultMapSize64 int64 = 10 * (1 << 30) var defaultMapSize int if ^uint(0)>>32 > 0 { @@ -103,7 +103,6 @@ func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { // Use 2GB default map size on 32-bit systems defaultMapSize = (1 << 31) - 1 } - f := &TranslateFile{ writeNotify: make(chan struct{}), closing: make(chan struct{}), From 4f17dbdbf1872742720c890baea3542eed7abe47 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 18 Sep 2018 10:34:31 -0500 Subject: [PATCH 15/45] add support for Bool fields prevent import of non-boolean row values to bool fields --- ctl/import_test.go | 46 +++++++++++++++++++++++ docs/administration.md | 14 +++++++ docs/api-reference.md | 2 + docs/data-model.md | 6 ++- docs/glossary.md | 2 +- docs/tutorials.md | 2 +- executor.go | 75 +++++++++++++++++++++++++------------- executor_test.go | 74 ++++++++++++++++++++++++++++++++++++- field.go | 37 +++++++++++++++++++ fragment.go | 34 +++++++++++++++++ fragment_internal_test.go | 77 +++++++++++++++++++++++++++++++++++++++ http/handler.go | 16 ++++++++ pql/ast.go | 17 +++++++++ view.go | 2 + 14 files changed, 375 insertions(+), 29 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index cd75a5d0a..2913005fb 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -279,3 +279,49 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) { t.Fatalf("Import Run with values doesn't work: %s", err) } } + +// Ensure that import into bool field runs. +func TestImportCommand_RunBool(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + ctx := context.Background() + + cmd := test.MustRunCluster(t, 1)[0] + cm.Host = cmd.API.Node().URI.HostPort() + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "bool"}}`))) + + cm.Index = "i" + cm.Field = "f" + + t.Run("Valid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run to bool field doesn't work: %s", err) + } + }) + + // Ensure that invalid bool values return an error. + t.Run("Invalid", func(t *testing.T) { + file, err := ioutil.TempFile("", "import-invalid-bool.csv") + if err != nil { + t.Fatal(err) + } + file.Write([]byte("0,1\n1,2\n1,3\n2,4")) + + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if !strings.Contains(err.Error(), "bool field imports only support values 0 and 1") { + t.Fatalf("expect error: bool field imports only support values 0 and 1, actual: %s", err) + } + }) +} diff --git a/docs/administration.md b/docs/administration.md index cd4e30184..e626cc0e5 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -64,6 +64,20 @@ If you are using [integer](../data-model/#bsi-range-encoding) field values, the pilosa import -i project -f stargazer-counts project-stargazer-counts.csv ``` +##### Importing Boolean Values + +If you are using a [boolean](../data-model/#boolean) field, the CSV file should be in the format `Boolean,Value`, where `Boolean` is either `0` (false) or `1` (true). + +For example, importing a file with the following contents will result in columns 3 and 9 being set in the `false` row, and columns 1, 2, 4, and 8 being set in the `true` row. +``` +0,3 +0,9 +1,1 +1,2 +1,4 +1,8 +``` +

Note that you must first create a field. View Create Field for more details. The `-e` flag can create the necessary schema when using a field of type "set".

diff --git a/docs/api-reference.md b/docs/api-reference.md index 890b37941..216a01bca 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -108,6 +108,8 @@ The request payload is in JSON, and may contain the `options` field. The `option * `int` * `min` (int): Minimum integer value allowed for the field. * `max` (int): Maximum integer value allowed for the field. +* `bool` + * (boolean fields take no arguments) * `time` * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this field. * `mutex` diff --git a/docs/data-model.md b/docs/data-model.md index ba7c12024..bababdd34 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -117,7 +117,7 @@ Query operations run in parallel, and they are evenly distributed across a clust ### Field Type -Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `time`, and `mutex`. +Upon creation, fields are configured to be of a certain type. Pilosa supports the following field types: `set`, `int`, `bool`, `time`, and `mutex`. #### Set @@ -192,3 +192,7 @@ Set(3, A=8, 2017-05-19T00:00) #### Mutex Mutex fields are similar to `set` fields, with the distinction of requiring the row value for each column to be mutually exclusive. In other words, each column can only have a single value for the field. If the field value for a column is updated on a `mutex` field, then the previous field value for that column will be cleared. This field type is like a field in an RDBMS table where every record contains a single value for a particular field. + +#### Boolean + +A boolean field is similar to a `mutex` field tracking only two values: `true` and `false`. Boolean fields do not maintain a sorted cache, nor do they support key values. diff --git a/docs/glossary.md b/docs/glossary.md index c39a0bf6f..fccbe0ae8 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -22,7 +22,7 @@ nav = [] Fragment: A Fragment is the intersection of a [field](#field) and a [shard](#shard) in an [index](#index). -[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). +[Field](../data-model/#field): Fields are used to group [rows](#row) into different categories. Row IDs are namespaced by field such that the same row ID in a different field refers to a different row. For [ranked](#topn) fields, rows are kept in sorted order within the field. Fields are one of four types: set, [int](#bsi), bool, time, and mutex. For more information, see [data model](../data-model/) and [Creating fields](../api-reference/#create-field). [Frame](../data-model/#field): Prior to Pilosa 1.0, fields were known as frames. diff --git a/docs/tutorials.md b/docs/tutorials.md index 040d85a1c..47344db4f 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -483,7 +483,7 @@ curl localhost:10101/index/patients/field/tcells \ {"success":true} ``` -Next, let's populate our fields with data. There are two ways to get data into fields: use the `SetFieldValue()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. +Next, let's populate our fields with data. There are two ways to get data into fields: use the `Set()` PQL function to set fields individually, or use the `pilosa import` command to import many values at once. First, let's set some field data using PQL. The following queries set the age, weight, and t-cell count for the patient with ID `1` in our system: ``` request diff --git a/executor.go b/executor.go index b0083a514..4ae5ac81e 100644 --- a/executor.go +++ b/executor.go @@ -734,8 +734,7 @@ func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Ca rowID, rowOK, rowErr := c.UintArg(fieldName) if rowErr != nil { return nil, fmt.Errorf("Row() error with arg for row: %v", rowErr) - } - if !rowOK { + } else if !rowOK { return nil, fmt.Errorf("Row() must specify %v", rowLabel) } @@ -1140,7 +1139,6 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { - // Read colID. colID, ok, err := c.UintArg("_" + columnLabel) if err != nil { @@ -1172,8 +1170,9 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } } + // Int field. if f.Type() == FieldTypeInt { - // Read remaining fields using labels. + // Read row value. rowVal, ok, err := c.IntArg(fieldName) if err != nil { return false, fmt.Errorf("reading Set() row: %v", err) @@ -1182,27 +1181,27 @@ func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, op } return e.executeSetValueField(ctx, index, c, f, colID, rowVal, opt) - } else { - // Read remaining fields using labels. - rowID, ok, err := c.UintArg(fieldName) - if err != nil { - return false, fmt.Errorf("reading Set() row: %v", err) - } else if !ok { - return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) - } - - var timestamp *time.Time - sTimestamp, ok := c.Args["_timestamp"].(string) - if ok { - t, err := time.Parse(TimeFormat, sTimestamp) - if err != nil { - return false, fmt.Errorf("invalid date: %s", sTimestamp) - } - timestamp = &t - } - - return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } + + // Read row ID. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading Set() row: %v", err) + } else if !ok { + return false, fmt.Errorf("Set() row argument '%v' required", rowLabel) + } + + var timestamp *time.Time + sTimestamp, ok := c.Args["_timestamp"].(string) + if ok { + t, err := time.Parse(TimeFormat, sTimestamp) + if err != nil { + return false, fmt.Errorf("invalid date: %s", sTimestamp) + } + timestamp = &t + } + + return e.executeSetBitField(ctx, index, c, f, colID, rowID, timestamp, opt) } // executeSetBitField executes a Set() call for a specific field. @@ -1675,7 +1674,21 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { // will raise an error downstream when it's used. return nil } - if field.keys() { + + // Bool field keys do not use the translater because there + // are only two possible values. Instead, they are handled + // directly. + if field.Type() == FieldTypeBool { + boolVal, err := callArgBool(c, rowKey) + if err != nil { + return errors.Wrap(err, "getting bool key") + } + rowID := falseRowID + if boolVal { + rowID = trueRowID + } + c.Args[rowKey] = rowID + } else if field.keys() { if c.Args[rowKey] != nil && !isString(c.Args[rowKey]) { return errors.New("row value must be a string when field 'keys' option enabled") } @@ -1832,6 +1845,18 @@ func (vc *ValCount) larger(other ValCount) ValCount { } } +func callArgBool(call *pql.Call, key string) (bool, error) { + value, ok := call.Args[key] + if !ok { + return false, errors.New("missing bool argument") + } + b, ok := value.(bool) + if !ok { + return false, fmt.Errorf("invalid bool argument type: %T", value) + } + return b, nil +} + func callArgString(call *pql.Call, key string) string { value, ok := call.Args[key] if !ok { diff --git a/executor_test.go b/executor_test.go index ee919f001..828a40b09 100644 --- a/executor_test.go +++ b/executor_test.go @@ -376,6 +376,78 @@ func TestExecutor_Execute_SetBit(t *testing.T) { }) } +// Ensure a set query can be executed on a bool field. +func TestExecutor_Execute_SetBool(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set a true bit. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Set the same bit to true again verify nothing changed. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + t.Fatal(err) + } else if res.Results[0].(bool) { + t.Fatalf("expected column to be unchanged") + } + + // Set the same bit to false. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil { + t.Fatal(err) + } else if !res.Results[0].(bool) { + t.Fatalf("expected column changed") + } + + // Ensure that the false row is set. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) { + t.Fatalf("unexpected colums: %+v", columns) + } + + // Ensure that the true row is empty. + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil { + t.Fatal(err) + } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected colums: %+v", columns) + } + }) + t.Run("Error", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create fields. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeBool()); err != nil { + t.Fatal(err) + } + + // Set bool using a string value. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + // Set bool using an integer. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil { + t.Fatalf("expected invalid bool type error") + } + + }) +} + // Ensure old PQL syntax doesn't break anything too badly. func TestExecutor_Execute_OldPQL(t *testing.T) { c := test.MustRunCluster(t, 1) @@ -397,7 +469,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - // Create felds. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(0, 50)); err != nil { t.Fatal(err) diff --git a/field.go b/field.go index d038b606c..822522bc4 100644 --- a/field.go +++ b/field.go @@ -52,6 +52,7 @@ const ( FieldTypeInt = "int" FieldTypeTime = "time" FieldTypeMutex = "mutex" + FieldTypeBool = "bool" ) // Field represents a container for views. @@ -155,6 +156,16 @@ func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption { } } +func OptFieldTypeBool() FieldOption { + return func(fo *FieldOptions) error { + if fo.Type != "" { + return errors.Errorf("field type is already set to: %s", fo.Type) + } + fo.Type = FieldTypeBool + return nil + } +} + // NewField returns a new instance of field. func NewField(path, index, name string, opts FieldOption) (*Field, error) { err := validateName(name) @@ -441,6 +452,14 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Max = 0 f.options.TimeQuantum = "" f.options.Keys = opt.Keys + case FieldTypeBool: + f.options.Type = FieldTypeBool + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + f.options.Keys = false default: return errors.New("invalid field type") } @@ -681,6 +700,10 @@ func (f *Field) deleteView(name string) error { } // Row returns a row of the standard view. +// It seems this method is only being used by the test +// package, and the fact that it's only allowed on +// `set` fields is odd. This may be considered for +// deprecation in a future version. func (f *Field) Row(rowID uint64) (*Row, error) { if f.Type() != FieldTypeSet { return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) @@ -954,10 +977,18 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro return errors.New("time quantum not set in field") } + fieldType := f.Type() + // Split import data by fragment. dataByFragment := make(map[importKey]importData) for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] + + // Bool-specific data validation. + if fieldType == FieldTypeBool && rowID > 1 { + return errors.New("bool field imports only support values 0 and 1") + } + var timestamp *time.Time if len(timestamps) > i { timestamp = timestamps[i] @@ -1191,6 +1222,12 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.CacheSize, o.Keys, }) + case FieldTypeBool: + return json.Marshal(struct { + Type string `json:"type"` + }{ + o.Type, + }) } return nil, errors.New("invalid field type") } diff --git a/fragment.go b/fragment.go index 7e3cd26d9..312b4af10 100644 --- a/fragment.go +++ b/fragment.go @@ -72,6 +72,10 @@ const ( // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. defaultFragmentMaxOpN = 2000 + + // Row ids used for boolean fields. + falseRowID = uint64(0) + trueRowID = uint64(1) ) // fragment represents the intersection of a field and shard in an index. @@ -2206,3 +2210,33 @@ func (v *rowsVector) Get(colID uint64) (uint64, bool) { // Set is not used for rowsVector. func (v *rowsVector) Set(colID, rowID uint64) {} + +// boolVector implements the vector interface by looking +// at data in rows 0 and 1. +type boolVector struct { + f *fragment +} + +// newBoolVector returns a boolVector for a given fragment. +func newBoolVector(f *fragment) *boolVector { + return &boolVector{ + f: f, + } +} + +// Get returns the rowID associated to the given colID. +// Additionally, it returns true if a value was found, +// otherwise it returns false. +func (v *boolVector) Get(colID uint64) (uint64, bool) { + rows := v.f.rowsForColumn(colID) + if len(rows) == 1 { + switch rows[0] { + case falseRowID, trueRowID: + return rows[0], true + } + } + return 0, false +} + +// Set is not used for boolVector. +func (v *boolVector) Set(colID, rowID uint64) {} diff --git a/fragment_internal_test.go b/fragment_internal_test.go index e0865c606..87f4e6137 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1248,6 +1248,76 @@ func TestFragment_ImportMutex(t *testing.T) { } } +// Ensure a fragment can import bool values. +func TestFragment_ImportBool(t *testing.T) { + tests := []struct { + rowIDs []uint64 + colIDs []uint64 + exp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1, 1, 1, 1}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 0: {}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 0: {0, 2, 3}, + 1: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 0: {8}, + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 1, 2}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 0: {}, + 1: {}, // This isn't {8} because fragment doesn't validate bool values. + 2: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + err := f.bulkImport(test.rowIDs, test.colIDs) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + // Check for expected results. + for k, v := range test.exp { + cols := f.row(k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func BenchmarkFragment_Snapshot(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -1372,6 +1442,13 @@ func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType st return frag } +// mustOpenBoolFragment returns a new instance of Fragment for a bool field. +func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) *fragment { + frag := mustOpenFragment(index, field, view, shard, cacheType) + frag.mutexVector = newBoolVector(frag) + return frag +} + // Reopen closes the fragment and reopens it as a new instance. func (f *fragment) reopen() error { if err := f.Close(); err != nil { diff --git a/http/handler.go b/http/handler.go index bc108568a..820bc6521 100644 --- a/http/handler.go +++ b/http/handler.go @@ -687,6 +687,8 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { fos = append(fos, pilosa.OptFieldTypeTime(*req.Options.TimeQuantum)) case pilosa.FieldTypeMutex: fos = append(fos, pilosa.OptFieldTypeMutex(*req.Options.CacheType, *req.Options.CacheSize)) + case pilosa.FieldTypeBool: + fos = append(fos, pilosa.OptFieldTypeBool()) } if req.Options.Keys != nil { if *req.Options.Keys { @@ -778,6 +780,20 @@ func (o *fieldOptions) validate() error { } else if o.TimeQuantum != nil { return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) } + case pilosa.FieldTypeBool: + if o.CacheType != nil { + return pilosa.NewBadRequestError(errors.New("cacheType does not apply to field type bool")) + } else if o.CacheSize != nil { + return pilosa.NewBadRequestError(errors.New("cacheSize does not apply to field type bool")) + } else if o.Min != nil { + return pilosa.NewBadRequestError(errors.New("min does not apply to field type bool")) + } else if o.Max != nil { + return pilosa.NewBadRequestError(errors.New("max does not apply to field type bool")) + } else if o.TimeQuantum != nil { + return pilosa.NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) + } else if o.Keys != nil { + return pilosa.NewBadRequestError(errors.New("keys does not apply to field type bool")) + } default: return errors.Errorf("invalid field type: %s", o.Type) } diff --git a/pql/ast.go b/pql/ast.go index cd483e030..0e2ff6aef 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -248,6 +248,23 @@ func (c *Call) FieldArg() (string, error) { return "", fmt.Errorf("No field argument specified") } +// BoolArg is for reading the value at key from call.Args as a bool. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a bool. An error is +// returned if the value is not a bool. +func (c *Call) BoolArg(key string) (bool, bool, error) { + val, ok := c.Args[key] + if !ok { + return false, false, nil + } + switch tval := val.(type) { + case bool: + return tval, true, nil + default: + return false, true, fmt.Errorf("could not convert %v of type %T to bool in Call.BoolArg", tval, tval) + } +} + // UintArg is for reading the value at key from call.Args as a uint64. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a uint64 or an int64 and diff --git a/view.go b/view.go index 2feac8b44..5fc0f5eb8 100644 --- a/view.go +++ b/view.go @@ -244,6 +244,8 @@ func (v *view) newFragment(path string, shard uint64) *fragment { frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard)) if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) + } else if v.fieldType == FieldTypeBool { + frag.mutexVector = newBoolVector(frag) } return frag } From 4c1709b8bd36acfb1110b8fc7aff6ea3251c560a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 21 Sep 2018 09:24:48 -0500 Subject: [PATCH 16/45] fix spelling mistake --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 4ae5ac81e..ff3db909d 100644 --- a/executor.go +++ b/executor.go @@ -1675,7 +1675,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { return nil } - // Bool field keys do not use the translater because there + // Bool field keys do not use the translator because there // are only two possible values. Instead, they are handled // directly. if field.Type() == FieldTypeBool { From 10393906868d50a137817955779e913bfdb07a0b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 21 Sep 2018 11:40:53 -0500 Subject: [PATCH 17/45] fixes pass-by-value issue in proto decode --- api.go | 2 ++ encoding/proto/proto.go | 28 ++++++++++++++++++++++------ server.go | 2 +- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/api.go b/api.go index a8783f502..44ea0a7b0 100644 --- a/api.go +++ b/api.go @@ -801,6 +801,8 @@ func importExistenceColumns(index *Index, columnIDs []uint64) error { } // MaxShards returns the maximum shard number for each index in a map. +// TODO (2.0): This method has been deprecated. Instead, use +// AvailableShardsByIndex. func (api *API) MaxShards(_ context.Context) map[string]uint64 { m := make(map[string]uint64) for k, v := range api.holder.availableShardsByIndex() { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 0fcd9df3a..b49b97454 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package proto import ( @@ -803,30 +817,32 @@ func decodeNodeEventMessage(pb *internal.NodeEventMessage, m *pilosa.NodeEvent) func decodeNodeStatus(pb *internal.NodeStatus, m *pilosa.NodeStatus) { m.Node = &pilosa.Node{} - decodeIndexStatuses(pb.Indexes, m.Indexes) + m.Indexes = decodeIndexStatuses(pb.Indexes) m.Schema = &pilosa.Schema{} decodeSchema(pb.Schema, m.Schema) } -func decodeIndexStatuses(a []*internal.IndexStatus, m []*pilosa.IndexStatus) { - m = m[:0] +func decodeIndexStatuses(a []*internal.IndexStatus) []*pilosa.IndexStatus { + m := make([]*pilosa.IndexStatus, 0) for i := range a { m = append(m, &pilosa.IndexStatus{}) decodeIndexStatus(a[i], m[i]) } + return m } func decodeIndexStatus(pb *internal.IndexStatus, m *pilosa.IndexStatus) { m.Name = pb.Name - decodeFieldStatuses(pb.Fields, m.Fields) + m.Fields = decodeFieldStatuses(pb.Fields) } -func decodeFieldStatuses(a []*internal.FieldStatus, m []*pilosa.FieldStatus) { - m = m[:0] +func decodeFieldStatuses(a []*internal.FieldStatus) []*pilosa.FieldStatus { + m := make([]*pilosa.FieldStatus, 0) for i := range a { m = append(m, &pilosa.FieldStatus{}) decodeFieldStatus(a[i], m[i]) } + return m } func decodeFieldStatus(pb *internal.FieldStatus, m *pilosa.FieldStatus) { diff --git a/server.go b/server.go index ce6319f45..81fc434f9 100644 --- a/server.go +++ b/server.go @@ -640,7 +640,7 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { for _, fs := range is.Fields { f := s.holder.Field(is.Name, fs.Name) - // if we don't know about an field locally, log a error because + // if we don't know about a field locally, log an error because // fields should be created and synced prior to shard creation if f == nil { s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name) From cdfcd6db5ec822080ca7efbf0d84da5daf028f9e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 24 Sep 2018 14:56:29 -0500 Subject: [PATCH 18/45] store remoteAvailableShards to file --- field.go | 61 ++++++++++++++++++++++++++++++++++-- field_internal_test.go | 20 ++++++++++++ server.go | 8 +++-- server/server_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/field.go b/field.go index 822522bc4..6d90a57ce 100644 --- a/field.go +++ b/field.go @@ -15,6 +15,7 @@ package pilosa import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -230,8 +231,17 @@ func (f *Field) AvailableShards() *roaring.Bitmap { return b } -// addRemoteAvailableShards merges the set of available shards into the current known set. -func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) { +// addRemoteAvailableShards merges the set of available shards into the current known set +// and saves the set to a file. +func (f *Field) addRemoteAvailableShards(b *roaring.Bitmap) error { + f.mergeRemoteAvailableShards(b) + + // Save the updated bitmap to the data store. + return f.saveAvailableShards() +} + +// mergeRemoteAvailableShards merges the set of available shards into the current known set. +func (f *Field) mergeRemoteAvailableShards(b *roaring.Bitmap) { f.mu.Lock() defer f.mu.Unlock() f.remoteAvailableShards = f.remoteAvailableShards.Union(b) @@ -291,6 +301,10 @@ func (f *Field) Open() error { return errors.Wrap(err, "loading meta") } + if err := f.loadAvailableShards(); err != nil { + return errors.Wrap(err, "loading available shards") + } + // Apply the field options loaded from meta. if err := f.applyOptions(f.options); err != nil { return errors.Wrap(err, "applying options") @@ -467,6 +481,49 @@ func (f *Field) applyOptions(opt FieldOptions) error { return nil } +// loadAvailableShards reads remoteAvailableShards data for the field, if any. +func (f *Field) loadAvailableShards() error { + bm := roaring.NewBitmap() + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".available.shards")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "reading available shards") + } else { + if err := bm.UnmarshalBinary(buf); err != nil { + return errors.Wrap(err, "unmarshaling") + } + } + + // Merge bitmap from file into field. + f.mergeRemoteAvailableShards(bm) + + return nil +} + +// saveAvailableShards writes remoteAvailableShards data for the field. +func (f *Field) saveAvailableShards() error { + f.mu.RLock() + defer f.mu.RUnlock() + + // Write available shards to buffer. + var buf bytes.Buffer + if n, err := f.remoteAvailableShards.WriteTo(&buf); err != nil { + return errors.Wrap(err, "writing bitmap to buffer") + } else if n != int64(buf.Len()) { + return fmt.Errorf("buffer size mismatch: %d != %d", n, buf.Len()) + } + + // Write buffer to file. + if err := ioutil.WriteFile(filepath.Join(f.path, ".available.shards"), buf.Bytes(), 0666); err != nil { + return errors.Wrap(err, "writing available shards") + } + + return nil +} + // Close closes the field and its views. func (f *Field) Close() error { f.mu.Lock() diff --git a/field_internal_test.go b/field_internal_test.go index 2d37ea25c..d72a3fd6a 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/roaring" ) // Ensure a bsiGroup can adjust to its baseValue. @@ -341,3 +342,22 @@ func TestField_RowTime(t *testing.T) { } } + +func TestField_PersistAvailableShards(t *testing.T) { + f := MustOpenField(OptFieldTypeDefault()) + + // bm represents remote available shards. + bm := roaring.NewBitmap(1, 2, 3) + + if err := f.addRemoteAvailableShards(bm); err != nil { + t.Fatal(err) + } + + // Reload field and verify that shard data is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(f.remoteAvailableShards.Slice(), bm.Slice()) { + t.Fatalf("unexpected available shards (reopen). expected: %v, but got: %v", bm.Slice(), f.remoteAvailableShards.Slice()) + } + +} diff --git a/server.go b/server.go index 81fc434f9..6cd313dc0 100644 --- a/server.go +++ b/server.go @@ -481,7 +481,9 @@ func (s *Server) receiveMessage(m Message) error { if f == nil { return fmt.Errorf("Local field not found: %s/%s", obj.Index, obj.Field) } - f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)) + if err := f.addRemoteAvailableShards(roaring.NewBitmap(obj.Shard)); err != nil { + return errors.Wrap(err, "adding remote available shards") + } case *CreateIndexMessage: opt := obj.Meta _, err := s.holder.CreateIndex(obj.Index, *opt) @@ -646,7 +648,9 @@ func (s *Server) mergeRemoteStatus(ns *NodeStatus) error { s.logger.Printf("Local Field not found: %s/%s", is.Name, fs.Name) continue } - f.addRemoteAvailableShards(fs.AvailableShards) + if err := f.addRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } } } diff --git a/server/server_test.go b/server/server_test.go index ed73975d7..aa202a54c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -621,4 +621,75 @@ func TestMain_ImportTimestamp(t *testing.T) { } } +func TestClusterQueriesAfterRestart(t *testing.T) { + cluster := test.MustRunCluster(t, 3) + defer cluster.Close() + cmd1 := cluster[1] + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + // build a query to set the first bit in 100 shards + query := strings.Builder{} + for i := 0; i < 100; i++ { + query.WriteString(fmt.Sprintf("Set(%d, testfield=0)", i*pilosa.ShardWidth)) + } + _, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: query.String(), + }) + if err != nil { + t.Fatalf("setting 100 bits in 100 shards: %v", err) + } + + results, err := cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: "Count(Row(testfield=0))", + }) + if err != nil { + t.Fatalf("counting row: %v", err) + } + if results.Results[0].(uint64) != 100 { + t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0]) + } + + err = cmd1.Command.Close() + if err != nil { + t.Fatalf("closing node0: %v", err) + } + + // confirm that cluster stops accepting queries after one node closes + if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { + t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) + } + + // Create new main with the same config. + config := cmd1.Command.Config + config.Bind = cmd1.API.Node().URI.HostPort() + + // this isn't necessary, but makes the test run way faster + config.Gossip.Port = strconv.Itoa(int(cmd1.Command.GossipTransport().URI.Port)) + cmd1.Command = server.NewCommand(cmd1.Stdin, cmd1.Stdout, cmd1.Stderr) + cmd1.Command.Config = config + err = cmd1.Start() + if err != nil { + t.Fatalf("reopening node 0: %v", err) + } + + for cmd1.API.State() != pilosa.ClusterStateNormal { + time.Sleep(time.Millisecond) + } + + results, err = cmd1.API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: "Count(Row(testfield=0))", + }) + if err != nil { + t.Fatalf("counting row: %v", err) + } + if results.Results[0].(uint64) != 100 { + t.Fatalf("Count should be 100, but got %v of type %[1]T", results.Results[0]) + } +} + // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address. From b6e99734f06155bc12ed6f4b2b4a7ddfaefbf608 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 14 Sep 2018 16:47:32 -0500 Subject: [PATCH 19/45] implement ClearRow() query --- docs/query-language.md | 30 + executor.go | 74 +- executor_test.go | 186 +++ fragment.go | 45 + fragment_internal_test.go | 27 + pql/pql.peg | 1 + pql/pql.peg.go | 2500 +++++++++++++++++++------------------ 7 files changed, 1643 insertions(+), 1220 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 876cf69c6..c2319cabd 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -236,6 +236,36 @@ Clear(10, stargazer=1) This represents removing the relationship between the user with id=1 and the repository with id=10. +#### ClearRow + +**Spec:** + +``` +ClearRow(=) +``` + +**Description:** + +`ClearRow` sets all bits to 0 in a given row of the binary matrix, thus disassociating the given row in the given field from all columns. + +**Result Type:** boolean + +A return value of `true` indicates that at least one column was toggled from 1 to 0. + +A return value of `false` indicates that all bits in the row were already 0 and nothing changed. + +**Examples:** + +Clear all bit in row 1 in the stargazer field: +```request +ClearRow(stargazer=1) +``` +```response +{"results":[true]} +``` + +This represents removing the relationship between the user with id=1 and all repositories. + ### Read Operations #### Row diff --git a/executor.go b/executor.go index ff3db909d..60c057ce1 100644 --- a/executor.go +++ b/executor.go @@ -182,6 +182,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeMax(ctx, index, c, shards, opt) case "Clear": return e.executeClearBit(ctx, index, c, opt) + case "ClearRow": + return e.executeClearRow(ctx, index, c, shards, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) @@ -1107,7 +1109,7 @@ func (e *executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return e.executeClearBitField(ctx, index, c, f, colID, rowID, opt) } -// executeClearBitField executes a Clear() call for a single view. +// executeClearBitField executes a Clear() call for a field. func (e *executor) executeClearBitField(ctx context.Context, index string, c *pql.Call, f *Field, colID, rowID uint64, opt *execOptions) (bool, error) { shard := colID / ShardWidth ret := false @@ -1137,6 +1139,76 @@ func (e *executor) executeClearBitField(ctx context.Context, index string, c *pq return ret, nil } +// executeClearRow executes a ClearRow() call. +func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + // Ensure the field type supports ClearRow(). + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("ClearRow() argument required: field") + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + if field.Type() == FieldTypeInt { + return false, errors.New("ClearRow() is not supported on `int` fields") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeClearRowShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + val := v.(bool) + if prev == nil { + return val + } + return val || prev.(bool) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + return result.(bool), err +} + +// executeClearRowShard executes a ClearRow() call for a single shard. +func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("ClearRow() argument required: field") + } + + // Read fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading ClearRow() row: %v", err) + } else if !ok { + return false, fmt.Errorf("ClearRow() row argument '%v' required", rowLabel) + } + + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + // Remove the row from all views. + changed := false + for _, view := range field.views() { + fragment := e.Holder.fragment(index, fieldName, view.name, shard) + if fragment == nil { + continue + } + cleared, err := fragment.clearRow(rowID) + if err != nil { + return false, errors.Wrapf(err, "clearing row %d on view %s shard %d", rowID, view.name, shard) + } + changed = changed || cleared + } + + return changed, nil +} + // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { // Read colID. diff --git a/executor_test.go b/executor_test.go index 828a40b09..dd8be0db8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1657,6 +1657,192 @@ func TestExecutor_Execute_Not(t *testing.T) { } } +// Ensure a row can be cleared. +func TestExecutor_Execute_ClearRow(t *testing.T) { + t.Run("Set", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Clear the row again and ensure we get a `false` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Mutex", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeMutex("none", 0)) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Clear the row again and ensure we get a `false` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Time", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + if err != nil { + t.Fatal(err) + } + + // Set columns. + cc := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00) + ` + rangeCheckQuery1 := `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)` + rangeCheckQuery10 := `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)` + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure the row is empty. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery1}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + // Ensure other rows were not affected. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: rangeCheckQuery10}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Int", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeInt(0, 100)) + if err != nil { + t.Fatal(err) + } + + // Ensure that clearing a row raises an error. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil { + t.Fatal("expected clear row to return an error") + } + }) +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/fragment.go b/fragment.go index 312b4af10..095308d09 100644 --- a/fragment.go +++ b/fragment.go @@ -490,6 +490,45 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return changed, nil } +// ClearRow clears a row for a given rowID within the fragment. +// This updates both the on-disk storage and the in-cache bitmap. +func (f *fragment) clearRow(rowID uint64) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.unprotectedClearRow(rowID) +} + +func (f *fragment) unprotectedClearRow(rowID uint64) (changed bool, err error) { + changed = false + + // First container of the row in storage. + headContainerKey := rowID << shardVsContainerExponent + + // Remove every container in the row. + for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { + k := headContainerKey + i + // Technically we could bypass the Get() call and only + // call Remove(), but the Get() gives us the ability + // to return true if any existing data was removed. + if cont := f.storage.Containers.Get(k); cont != nil { + f.storage.Containers.Remove(k) + changed = true + } + } + + // Clear the row in cache. + f.cache.Add(rowID, 0) + + // Snapshot storage. + if err := f.snapshot(); err != nil { + return false, errors.Wrap(err, "snapshotting") + } + + f.stats.Count("clearRow", 1, 1.0) + + return changed, nil +} + func (f *fragment) bit(rowID, columnID uint64) (bool, error) { pos, err := f.pos(rowID, columnID) if err != nil { @@ -2182,6 +2221,12 @@ func pos(rowID, columnID uint64) uint64 { type vector interface { Get(colID uint64) (uint64, bool) Set(colID, rowID uint64) + // TODO: Clear(colID, rowID uint64) + // Set() and Clear() are required for implementations + // where the vector data is maintained independently + // from the fragment data. In those cases, the Set() + // and Clear() would also need to address the ranked + // cache as well as the effects of ClearRow(). } // rowsVector implements the vector interface by looking diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 87f4e6137..81ff36859 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -95,6 +95,33 @@ func TestFragment_ClearBit(t *testing.T) { } } +// Ensure a fragment can clear a row. +func TestFragment_ClearRow(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, "") + defer f.Close() + + // Set and then clear bits on the fragment. + if _, err := f.setBit(1000, 1); err != nil { + t.Fatal(err) + } else if _, err := f.setBit(1000, 65536); err != nil { + t.Fatal(err) + } else if _, err := f.unprotectedClearRow(1000); err != nil { + t.Fatal(err) + } + + // Verify count on row. + if n := f.row(1000).Count(); n != 0 { + t.Fatalf("unexpected count: %d", n) + } + + // Close and reopen the fragment & verify the data. + if err := f.reopen(); err != nil { + t.Fatal(err) + } else if n := f.row(1000).Count(); n != 0 { + t.Fatalf("unexpected count (reopen): %d", n) + } +} + // Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { diff --git a/pql/pql.peg b/pql/pql.peg index 33896f8ad..c38046f67 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -10,6 +10,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'SetRowAttrs' {p.startCall("SetRowAttrs")} open posfield comma row comma args close {p.endCall()} / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} + / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 21166d0ca..5adc244f8 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -62,9 +62,9 @@ const ( ruleAction9 ruleAction10 ruleAction11 - rulePegText ruleAction12 ruleAction13 + rulePegText ruleAction14 ruleAction15 ruleAction16 @@ -99,6 +99,8 @@ const ( ruleAction45 ruleAction46 ruleAction47 + ruleAction48 + ruleAction49 ) var rul3s = [...]string{ @@ -149,9 +151,9 @@ var rul3s = [...]string{ "Action9", "Action10", "Action11", - "PegText", "Action12", "Action13", + "PegText", "Action14", "Action15", "Action16", @@ -186,6 +188,8 @@ var rul3s = [...]string{ "Action45", "Action46", "Action47", + "Action48", + "Action49", } type token32 struct { @@ -302,7 +306,7 @@ type PQL struct { Buffer string buffer []rune - rules [84]func() bool + rules [86]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -411,84 +415,88 @@ func (p *PQL) Execute() { case ruleAction7: p.endCall() case ruleAction8: - p.startCall("TopN") + p.startCall("ClearRow") case ruleAction9: p.endCall() case ruleAction10: - p.startCall("Range") + p.startCall("TopN") case ruleAction11: p.endCall() case ruleAction12: - p.startCall(buffer[begin:end]) + p.startCall("Range") case ruleAction13: p.endCall() case ruleAction14: - p.addBTWN() + p.startCall(buffer[begin:end]) case ruleAction15: - p.addLTE() + p.endCall() case ruleAction16: - p.addGTE() + p.addBTWN() case ruleAction17: - p.addEQ() + p.addLTE() case ruleAction18: - p.addNEQ() + p.addGTE() case ruleAction19: - p.addLT() + p.addEQ() case ruleAction20: - p.addGT() + p.addNEQ() case ruleAction21: - p.startConditional() + p.addLT() case ruleAction22: - p.endConditional() + p.addGT() case ruleAction23: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction24: - p.condAdd(buffer[begin:end]) + p.endConditional() case ruleAction25: p.condAdd(buffer[begin:end]) case ruleAction26: - p.addPosStr("_start", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction27: - p.addPosStr("_end", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction28: - p.startList() + p.addPosStr("_start", buffer[begin:end]) case ruleAction29: - p.endList() + p.addPosStr("_end", buffer[begin:end]) case ruleAction30: - p.addVal(nil) + p.startList() case ruleAction31: - p.addVal(true) + p.endList() case ruleAction32: - p.addVal(false) + p.addVal(nil) case ruleAction33: - p.addNumVal(buffer[begin:end]) + p.addVal(true) case ruleAction34: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction35: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction36: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction37: p.addVal(buffer[begin:end]) case ruleAction38: - p.addField(buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction39: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction40: - p.addPosNum("_row", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction41: - p.addPosNum("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction42: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction43: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction44: p.addPosNum("_row", buffer[begin:end]) + case ruleAction43: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction44: + p.addPosStr("_col", buffer[begin:end]) case ruleAction45: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction46: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction47: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction48: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction49: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -601,7 +609,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('T' 'o' 'p' 'N' Action8 open posfield (comma allargs)? close Action9) / ('R' 'a' 'n' 'g' 'e' Action10 open (timerange / conditional / arg) close Action11) / ( Action12 open allargs comma? close Action13))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('T' 'o' 'p' 'N' Action10 open posfield (comma allargs)? close Action11) / ('R' 'a' 'n' 'g' 'e' Action12 open (timerange / conditional / arg) close Action13) / ( Action14 open allargs comma? close Action15))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -650,7 +658,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction47, position) + add(ruleAction49, position) } add(ruletimestamp, position12) } @@ -736,7 +744,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction44, position) + add(ruleAction46, position) } goto l19 l20: @@ -757,7 +765,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction45, position) + add(ruleAction47, position) } goto l19 l23: @@ -778,7 +786,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction46, position) + add(ruleAction48, position) } } l19: @@ -923,7 +931,27 @@ func (p *PQL) Init() { goto l7 l32: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('T') { + if buffer[position] != rune('C') { + goto l35 + } + position++ + if buffer[position] != rune('l') { + goto l35 + } + position++ + if buffer[position] != rune('e') { + goto l35 + } + position++ + if buffer[position] != rune('a') { + goto l35 + } + position++ + if buffer[position] != rune('r') { + goto l35 + } + position++ + if buffer[position] != rune('R') { goto l35 } position++ @@ -931,11 +959,7 @@ func (p *PQL) Init() { goto l35 } position++ - if buffer[position] != rune('p') { - goto l35 - } - position++ - if buffer[position] != rune('N') { + if buffer[position] != rune('w') { goto l35 } position++ @@ -945,22 +969,9 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l35 } - if !_rules[ruleposfield]() { + if !_rules[rulearg]() { goto l35 } - { - position37, tokenIndex37 := position, tokenIndex - if !_rules[rulecomma]() { - goto l37 - } - if !_rules[ruleallargs]() { - goto l37 - } - goto l38 - l37: - position, tokenIndex = position37, tokenIndex37 - } - l38: if !_rules[ruleclose]() { goto l35 } @@ -970,193 +981,240 @@ func (p *PQL) Init() { goto l7 l35: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l40 + if buffer[position] != rune('T') { + goto l38 } position++ - if buffer[position] != rune('a') { - goto l40 + if buffer[position] != rune('o') { + goto l38 } position++ - if buffer[position] != rune('n') { - goto l40 + if buffer[position] != rune('p') { + goto l38 } position++ - if buffer[position] != rune('g') { - goto l40 - } - position++ - if buffer[position] != rune('e') { - goto l40 + if buffer[position] != rune('N') { + goto l38 } position++ { add(ruleAction10, position) } if !_rules[ruleopen]() { - goto l40 + goto l38 + } + if !_rules[ruleposfield]() { + goto l38 } { - position42, tokenIndex42 := position, tokenIndex - { - position44 := position - if !_rules[rulefield]() { - goto l43 - } - if !_rules[rulesp]() { - goto l43 - } - if buffer[position] != rune('=') { - goto l43 - } - position++ - if !_rules[rulesp]() { - goto l43 - } - if !_rules[rulevalue]() { - goto l43 - } - if !_rules[rulecomma]() { - goto l43 - } - { - position45 := position - if !_rules[ruletimestampfmt]() { - goto l43 - } - add(rulePegText, position45) - } - { - add(ruleAction26, position) - } - if !_rules[rulecomma]() { - goto l43 - } - { - position47 := position - if !_rules[ruletimestampfmt]() { - goto l43 - } - add(rulePegText, position47) - } - { - add(ruleAction27, position) - } - add(ruletimerange, position44) - } - goto l42 - l43: - position, tokenIndex = position42, tokenIndex42 - { - position50 := position - { - add(ruleAction21, position) - } - if !_rules[rulecondint]() { - goto l49 - } - if !_rules[rulecondLT]() { - goto l49 - } - { - position52 := position - { - position53 := position - if !_rules[rulefieldExpr]() { - goto l49 - } - add(rulePegText, position53) - } - if !_rules[rulesp]() { - goto l49 - } - { - add(ruleAction25, position) - } - add(rulecondfield, position52) - } - if !_rules[rulecondLT]() { - goto l49 - } - if !_rules[rulecondint]() { - goto l49 - } - { - add(ruleAction22, position) - } - add(ruleconditional, position50) - } - goto l42 - l49: - position, tokenIndex = position42, tokenIndex42 - if !_rules[rulearg]() { + position40, tokenIndex40 := position, tokenIndex + if !_rules[rulecomma]() { goto l40 } + if !_rules[ruleallargs]() { + goto l40 + } + goto l41 + l40: + position, tokenIndex = position40, tokenIndex40 } - l42: + l41: if !_rules[ruleclose]() { - goto l40 + goto l38 } { add(ruleAction11, position) } goto l7 - l40: + l38: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l43 + } + position++ + if buffer[position] != rune('a') { + goto l43 + } + position++ + if buffer[position] != rune('n') { + goto l43 + } + position++ + if buffer[position] != rune('g') { + goto l43 + } + position++ + if buffer[position] != rune('e') { + goto l43 + } + position++ + { + add(ruleAction12, position) + } + if !_rules[ruleopen]() { + goto l43 + } + { + position45, tokenIndex45 := position, tokenIndex + { + position47 := position + if !_rules[rulefield]() { + goto l46 + } + if !_rules[rulesp]() { + goto l46 + } + if buffer[position] != rune('=') { + goto l46 + } + position++ + if !_rules[rulesp]() { + goto l46 + } + if !_rules[rulevalue]() { + goto l46 + } + if !_rules[rulecomma]() { + goto l46 + } + { + position48 := position + if !_rules[ruletimestampfmt]() { + goto l46 + } + add(rulePegText, position48) + } + { + add(ruleAction28, position) + } + if !_rules[rulecomma]() { + goto l46 + } + { + position50 := position + if !_rules[ruletimestampfmt]() { + goto l46 + } + add(rulePegText, position50) + } + { + add(ruleAction29, position) + } + add(ruletimerange, position47) + } + goto l45 + l46: + position, tokenIndex = position45, tokenIndex45 + { + position53 := position + { + add(ruleAction23, position) + } + if !_rules[rulecondint]() { + goto l52 + } + if !_rules[rulecondLT]() { + goto l52 + } + { + position55 := position + { + position56 := position + if !_rules[rulefieldExpr]() { + goto l52 + } + add(rulePegText, position56) + } + if !_rules[rulesp]() { + goto l52 + } + { + add(ruleAction27, position) + } + add(rulecondfield, position55) + } + if !_rules[rulecondLT]() { + goto l52 + } + if !_rules[rulecondint]() { + goto l52 + } + { + add(ruleAction24, position) + } + add(ruleconditional, position53) + } + goto l45 + l52: + position, tokenIndex = position45, tokenIndex45 + if !_rules[rulearg]() { + goto l43 + } + } + l45: + if !_rules[ruleclose]() { + goto l43 + } + { + add(ruleAction13, position) + } + goto l7 + l43: position, tokenIndex = position7, tokenIndex7 { - position57 := position + position60 := position { - position58 := position + position61 := position { - position59, tokenIndex59 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l60 + goto l63 } position++ - goto l59 - l60: - position, tokenIndex = position59, tokenIndex59 + goto l62 + l63: + position, tokenIndex = position62, tokenIndex62 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l59: - l61: + l62: + l64: { - position62, tokenIndex62 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex { - position63, tokenIndex63 := position, tokenIndex + position66, tokenIndex66 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l64 + goto l67 } position++ - goto l63 - l64: - position, tokenIndex = position63, tokenIndex63 + goto l66 + l67: + position, tokenIndex = position66, tokenIndex66 if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l68 + } + position++ + goto l66 + l68: + position, tokenIndex = position66, tokenIndex66 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l65 } position++ - goto l63 - l65: - position, tokenIndex = position63, tokenIndex63 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l62 - } - position++ } - l63: - goto l61 - l62: - position, tokenIndex = position62, tokenIndex62 + l66: + goto l64 + l65: + position, tokenIndex = position65, tokenIndex65 } - add(ruleIDENT, position58) + add(ruleIDENT, position61) } - add(rulePegText, position57) + add(rulePegText, position60) } { - add(ruleAction12, position) + add(ruleAction14, position) } if !_rules[ruleopen]() { goto l5 @@ -1165,20 +1223,20 @@ func (p *PQL) Init() { goto l5 } { - position67, tokenIndex67 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if !_rules[rulecomma]() { - goto l67 + goto l70 } - goto l68 - l67: - position, tokenIndex = position67, tokenIndex67 + goto l71 + l70: + position, tokenIndex = position70, tokenIndex70 } - l68: + l71: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction13, position) + add(ruleAction15, position) } } l7: @@ -1191,1487 +1249,1487 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position70, tokenIndex70 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex { - position71 := position + position74 := position { - position72, tokenIndex72 := position, tokenIndex + position75, tokenIndex75 := position, tokenIndex if !_rules[ruleCall]() { - goto l73 - } - l74: - { - position75, tokenIndex75 := position, tokenIndex - if !_rules[rulecomma]() { - goto l75 - } - if !_rules[ruleCall]() { - goto l75 - } - goto l74 - l75: - position, tokenIndex = position75, tokenIndex75 - } - { - position76, tokenIndex76 := position, tokenIndex - if !_rules[rulecomma]() { - goto l76 - } - if !_rules[ruleargs]() { - goto l76 - } - goto l77 - l76: - position, tokenIndex = position76, tokenIndex76 + goto l76 } l77: - goto l72 - l73: - position, tokenIndex = position72, tokenIndex72 - if !_rules[ruleargs]() { - goto l78 + { + position78, tokenIndex78 := position, tokenIndex + if !_rules[rulecomma]() { + goto l78 + } + if !_rules[ruleCall]() { + goto l78 + } + goto l77 + l78: + position, tokenIndex = position78, tokenIndex78 } - goto l72 - l78: - position, tokenIndex = position72, tokenIndex72 + { + position79, tokenIndex79 := position, tokenIndex + if !_rules[rulecomma]() { + goto l79 + } + if !_rules[ruleargs]() { + goto l79 + } + goto l80 + l79: + position, tokenIndex = position79, tokenIndex79 + } + l80: + goto l75 + l76: + position, tokenIndex = position75, tokenIndex75 + if !_rules[ruleargs]() { + goto l81 + } + goto l75 + l81: + position, tokenIndex = position75, tokenIndex75 if !_rules[rulesp]() { - goto l70 + goto l73 } } - l72: - add(ruleallargs, position71) + l75: + add(ruleallargs, position74) } return true - l70: - position, tokenIndex = position70, tokenIndex70 + l73: + position, tokenIndex = position73, tokenIndex73 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position79, tokenIndex79 := position, tokenIndex + position82, tokenIndex82 := position, tokenIndex { - position80 := position + position83 := position if !_rules[rulearg]() { - goto l79 + goto l82 } { - position81, tokenIndex81 := position, tokenIndex + position84, tokenIndex84 := position, tokenIndex if !_rules[rulecomma]() { - goto l81 + goto l84 } if !_rules[ruleargs]() { - goto l81 + goto l84 } - goto l82 - l81: - position, tokenIndex = position81, tokenIndex81 + goto l85 + l84: + position, tokenIndex = position84, tokenIndex84 } - l82: + l85: if !_rules[rulesp]() { - goto l79 + goto l82 } - add(ruleargs, position80) + add(ruleargs, position83) } return true - l79: - position, tokenIndex = position79, tokenIndex79 + l82: + position, tokenIndex = position82, tokenIndex82 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position83, tokenIndex83 := position, tokenIndex + position86, tokenIndex86 := position, tokenIndex { - position84 := position + position87 := position { - position85, tokenIndex85 := position, tokenIndex + position88, tokenIndex88 := position, tokenIndex if !_rules[rulefield]() { - goto l86 + goto l89 } if !_rules[rulesp]() { - goto l86 + goto l89 } if buffer[position] != rune('=') { - goto l86 + goto l89 } position++ if !_rules[rulesp]() { - goto l86 + goto l89 } if !_rules[rulevalue]() { + goto l89 + } + goto l88 + l89: + position, tokenIndex = position88, tokenIndex88 + if !_rules[rulefield]() { goto l86 } - goto l85 - l86: - position, tokenIndex = position85, tokenIndex85 - if !_rules[rulefield]() { - goto l83 - } if !_rules[rulesp]() { - goto l83 + goto l86 } { - position87 := position + position90 := position { - position88, tokenIndex88 := position, tokenIndex + position91, tokenIndex91 := position, tokenIndex if buffer[position] != rune('>') { - goto l89 + goto l92 } position++ if buffer[position] != rune('<') { - goto l89 - } - position++ - { - add(ruleAction14, position) - } - goto l88 - l89: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('<') { - goto l91 - } - position++ - if buffer[position] != rune('=') { - goto l91 - } - position++ - { - add(ruleAction15, position) - } - goto l88 - l91: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('>') { - goto l93 - } - position++ - if buffer[position] != rune('=') { - goto l93 + goto l92 } position++ { add(ruleAction16, position) } - goto l88 - l93: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('=') { - goto l95 + goto l91 + l92: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('<') { + goto l94 } position++ if buffer[position] != rune('=') { - goto l95 + goto l94 } position++ { add(ruleAction17, position) } - goto l88 - l95: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('!') { - goto l97 + goto l91 + l94: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('>') { + goto l96 } position++ if buffer[position] != rune('=') { - goto l97 + goto l96 } position++ { add(ruleAction18, position) } - goto l88 - l97: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('<') { - goto l99 + goto l91 + l96: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('=') { + goto l98 + } + position++ + if buffer[position] != rune('=') { + goto l98 } position++ { add(ruleAction19, position) } - goto l88 - l99: - position, tokenIndex = position88, tokenIndex88 - if buffer[position] != rune('>') { - goto l83 + goto l91 + l98: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('!') { + goto l100 + } + position++ + if buffer[position] != rune('=') { + goto l100 } position++ { add(ruleAction20, position) } - } - l88: - add(ruleCOND, position87) - } - if !_rules[rulesp]() { - goto l83 - } - if !_rules[rulevalue]() { - goto l83 - } - } - l85: - add(rulearg, position84) - } - return true - l83: - position, tokenIndex = position83, tokenIndex83 - return false - }, - /* 5 COND <- <(('>' '<' Action14) / ('<' '=' Action15) / ('>' '=' Action16) / ('=' '=' Action17) / ('!' '=' Action18) / ('<' Action19) / ('>' Action20))> */ - nil, - /* 6 conditional <- <(Action21 condint condLT condfield condLT condint Action22)> */ - nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action23)> */ - func() bool { - position104, tokenIndex104 := position, tokenIndex - { - position105 := position - { - position106 := position - { - position107, tokenIndex107 := position, tokenIndex - { - position109, tokenIndex109 := position, tokenIndex - if buffer[position] != rune('-') { - goto l109 + goto l91 + l100: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('<') { + goto l102 } position++ - goto l110 - l109: - position, tokenIndex = position109, tokenIndex109 + { + add(ruleAction21, position) + } + goto l91 + l102: + position, tokenIndex = position91, tokenIndex91 + if buffer[position] != rune('>') { + goto l86 + } + position++ + { + add(ruleAction22, position) + } } - l110: - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l108 - } - position++ - l111: + l91: + add(ruleCOND, position90) + } + if !_rules[rulesp]() { + goto l86 + } + if !_rules[rulevalue]() { + goto l86 + } + } + l88: + add(rulearg, position87) + } + return true + l86: + position, tokenIndex = position86, tokenIndex86 + return false + }, + /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ + nil, + /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ + nil, + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ + func() bool { + position107, tokenIndex107 := position, tokenIndex + { + position108 := position + { + position109 := position + { + position110, tokenIndex110 := position, tokenIndex { position112, tokenIndex112 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { + if buffer[position] != rune('-') { goto l112 } position++ - goto l111 + goto l113 l112: position, tokenIndex = position112, tokenIndex112 } - goto l107 - l108: - position, tokenIndex = position107, tokenIndex107 + l113: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l111 + } + position++ + l114: + { + position115, tokenIndex115 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l115 + } + position++ + goto l114 + l115: + position, tokenIndex = position115, tokenIndex115 + } + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 if buffer[position] != rune('0') { - goto l104 + goto l107 } position++ } - l107: - add(rulePegText, position106) + l110: + add(rulePegText, position109) } if !_rules[rulesp]() { - goto l104 + goto l107 } { - add(ruleAction23, position) + add(ruleAction25, position) } - add(rulecondint, position105) + add(rulecondint, position108) } return true - l104: - position, tokenIndex = position104, tokenIndex104 + l107: + position, tokenIndex = position107, tokenIndex107 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action24)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ func() bool { - position114, tokenIndex114 := position, tokenIndex + position117, tokenIndex117 := position, tokenIndex { - position115 := position + position118 := position { - position116 := position + position119 := position { - position117, tokenIndex117 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex if buffer[position] != rune('<') { - goto l118 + goto l121 } position++ if buffer[position] != rune('=') { - goto l118 + goto l121 } position++ - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 + goto l120 + l121: + position, tokenIndex = position120, tokenIndex120 if buffer[position] != rune('<') { - goto l114 + goto l117 } position++ } - l117: - add(rulePegText, position116) + l120: + add(rulePegText, position119) } if !_rules[rulesp]() { - goto l114 + goto l117 } { - add(ruleAction24, position) + add(ruleAction26, position) } - add(rulecondLT, position115) + add(rulecondLT, position118) } return true - l114: - position, tokenIndex = position114, tokenIndex114 + l117: + position, tokenIndex = position117, tokenIndex117 return false }, - /* 9 condfield <- <( sp Action25)> */ + /* 9 condfield <- <( sp Action27)> */ nil, - /* 10 timerange <- <(field sp '=' sp value comma Action26 comma Action27)> */ + /* 10 timerange <- <(field sp '=' sp value comma Action28 comma Action29)> */ nil, - /* 11 value <- <(item / (lbrack Action28 list rbrack Action29))> */ + /* 11 value <- <(item / (lbrack Action30 list rbrack Action31))> */ func() bool { - position122, tokenIndex122 := position, tokenIndex + position125, tokenIndex125 := position, tokenIndex { - position123 := position + position126 := position { - position124, tokenIndex124 := position, tokenIndex + position127, tokenIndex127 := position, tokenIndex if !_rules[ruleitem]() { - goto l125 + goto l128 } - goto l124 - l125: - position, tokenIndex = position124, tokenIndex124 + goto l127 + l128: + position, tokenIndex = position127, tokenIndex127 { - position126 := position + position129 := position if buffer[position] != rune('[') { - goto l122 + goto l125 } position++ if !_rules[rulesp]() { - goto l122 + goto l125 } - add(rulelbrack, position126) - } - { - add(ruleAction28, position) - } - if !_rules[rulelist]() { - goto l122 - } - { - position128 := position - if !_rules[rulesp]() { - goto l122 - } - if buffer[position] != rune(']') { - goto l122 - } - position++ - if !_rules[rulesp]() { - goto l122 - } - add(rulerbrack, position128) - } - { - add(ruleAction29, position) - } - } - l124: - add(rulevalue, position123) - } - return true - l122: - position, tokenIndex = position122, tokenIndex122 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position130, tokenIndex130 := position, tokenIndex - { - position131 := position - if !_rules[ruleitem]() { - goto l130 - } - { - position132, tokenIndex132 := position, tokenIndex - if !_rules[rulecomma]() { - goto l132 - } - if !_rules[rulelist]() { - goto l132 - } - goto l133 - l132: - position, tokenIndex = position132, tokenIndex132 - } - l133: - add(rulelist, position131) - } - return true - l130: - position, tokenIndex = position130, tokenIndex130 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action33) / (<('-'? '.' [0-9]+)> Action34) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action35) / ('"' '"' Action36) / ('\'' '\'' Action37))> */ - func() bool { - position134, tokenIndex134 := position, tokenIndex - { - position135 := position - { - position136, tokenIndex136 := position, tokenIndex - if buffer[position] != rune('n') { - goto l137 - } - position++ - if buffer[position] != rune('u') { - goto l137 - } - position++ - if buffer[position] != rune('l') { - goto l137 - } - position++ - if buffer[position] != rune('l') { - goto l137 - } - position++ - { - position138, tokenIndex138 := position, tokenIndex - { - position139, tokenIndex139 := position, tokenIndex - if !_rules[rulecomma]() { - goto l140 - } - goto l139 - l140: - position, tokenIndex = position139, tokenIndex139 - if !_rules[rulesp]() { - goto l137 - } - if !_rules[ruleclose]() { - goto l137 - } - } - l139: - position, tokenIndex = position138, tokenIndex138 + add(rulelbrack, position129) } { add(ruleAction30, position) } - goto l136 - l137: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('t') { - goto l142 + if !_rules[rulelist]() { + goto l125 } - position++ - if buffer[position] != rune('r') { - goto l142 - } - position++ - if buffer[position] != rune('u') { - goto l142 - } - position++ - if buffer[position] != rune('e') { - goto l142 - } - position++ { - position143, tokenIndex143 := position, tokenIndex - { - position144, tokenIndex144 := position, tokenIndex - if !_rules[rulecomma]() { - goto l145 - } - goto l144 - l145: - position, tokenIndex = position144, tokenIndex144 - if !_rules[rulesp]() { - goto l142 - } - if !_rules[ruleclose]() { - goto l142 - } + position131 := position + if !_rules[rulesp]() { + goto l125 } - l144: - position, tokenIndex = position143, tokenIndex143 + if buffer[position] != rune(']') { + goto l125 + } + position++ + if !_rules[rulesp]() { + goto l125 + } + add(rulerbrack, position131) } { add(ruleAction31, position) } + } + l127: + add(rulevalue, position126) + } + return true + l125: + position, tokenIndex = position125, tokenIndex125 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position133, tokenIndex133 := position, tokenIndex + { + position134 := position + if !_rules[ruleitem]() { + goto l133 + } + { + position135, tokenIndex135 := position, tokenIndex + if !_rules[rulecomma]() { + goto l135 + } + if !_rules[rulelist]() { + goto l135 + } goto l136 - l142: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('f') { - goto l147 + l135: + position, tokenIndex = position135, tokenIndex135 + } + l136: + add(rulelist, position134) + } + return true + l133: + position, tokenIndex = position133, tokenIndex133 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action35) / (<('-'? '.' [0-9]+)> Action36) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action37) / ('"' '"' Action38) / ('\'' '\'' Action39))> */ + func() bool { + position137, tokenIndex137 := position, tokenIndex + { + position138 := position + { + position139, tokenIndex139 := position, tokenIndex + if buffer[position] != rune('n') { + goto l140 } position++ - if buffer[position] != rune('a') { - goto l147 + if buffer[position] != rune('u') { + goto l140 } position++ if buffer[position] != rune('l') { - goto l147 + goto l140 } position++ - if buffer[position] != rune('s') { - goto l147 - } - position++ - if buffer[position] != rune('e') { - goto l147 + if buffer[position] != rune('l') { + goto l140 } position++ { - position148, tokenIndex148 := position, tokenIndex + position141, tokenIndex141 := position, tokenIndex { - position149, tokenIndex149 := position, tokenIndex + position142, tokenIndex142 := position, tokenIndex if !_rules[rulecomma]() { - goto l150 + goto l143 } - goto l149 - l150: - position, tokenIndex = position149, tokenIndex149 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 if !_rules[rulesp]() { - goto l147 + goto l140 } if !_rules[ruleclose]() { - goto l147 + goto l140 } } - l149: - position, tokenIndex = position148, tokenIndex148 + l142: + position, tokenIndex = position141, tokenIndex141 } { add(ruleAction32, position) } - goto l136 - l147: - position, tokenIndex = position136, tokenIndex136 + goto l139 + l140: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('t') { + goto l145 + } + position++ + if buffer[position] != rune('r') { + goto l145 + } + position++ + if buffer[position] != rune('u') { + goto l145 + } + position++ + if buffer[position] != rune('e') { + goto l145 + } + position++ { - position153 := position + position146, tokenIndex146 := position, tokenIndex { - position154, tokenIndex154 := position, tokenIndex - if buffer[position] != rune('-') { - goto l154 + position147, tokenIndex147 := position, tokenIndex + if !_rules[rulecomma]() { + goto l148 } - position++ - goto l155 - l154: - position, tokenIndex = position154, tokenIndex154 - } - l155: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l152 - } - position++ - l156: - { - position157, tokenIndex157 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l157 + goto l147 + l148: + position, tokenIndex = position147, tokenIndex147 + if !_rules[rulesp]() { + goto l145 } - position++ - goto l156 - l157: - position, tokenIndex = position157, tokenIndex157 - } - { - position158, tokenIndex158 := position, tokenIndex - if buffer[position] != rune('.') { - goto l158 + if !_rules[ruleclose]() { + goto l145 } - position++ - l160: - { - position161, tokenIndex161 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l161 - } - position++ - goto l160 - l161: - position, tokenIndex = position161, tokenIndex161 - } - goto l159 - l158: - position, tokenIndex = position158, tokenIndex158 } - l159: - add(rulePegText, position153) + l147: + position, tokenIndex = position146, tokenIndex146 } { add(ruleAction33, position) } - goto l136 - l152: - position, tokenIndex = position136, tokenIndex136 + goto l139 + l145: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('f') { + goto l150 + } + position++ + if buffer[position] != rune('a') { + goto l150 + } + position++ + if buffer[position] != rune('l') { + goto l150 + } + position++ + if buffer[position] != rune('s') { + goto l150 + } + position++ + if buffer[position] != rune('e') { + goto l150 + } + position++ { - position164 := position + position151, tokenIndex151 := position, tokenIndex { - position165, tokenIndex165 := position, tokenIndex - if buffer[position] != rune('-') { - goto l165 + position152, tokenIndex152 := position, tokenIndex + if !_rules[rulecomma]() { + goto l153 } - position++ - goto l166 - l165: - position, tokenIndex = position165, tokenIndex165 - } - l166: - if buffer[position] != rune('.') { - goto l163 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 - } - position++ - l167: - { - position168, tokenIndex168 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l168 + goto l152 + l153: + position, tokenIndex = position152, tokenIndex152 + if !_rules[rulesp]() { + goto l150 + } + if !_rules[ruleclose]() { + goto l150 } - position++ - goto l167 - l168: - position, tokenIndex = position168, tokenIndex168 } - add(rulePegText, position164) + l152: + position, tokenIndex = position151, tokenIndex151 } { add(ruleAction34, position) } - goto l136 - l163: - position, tokenIndex = position136, tokenIndex136 + goto l139 + l150: + position, tokenIndex = position139, tokenIndex139 { - position171 := position + position156 := position { - position174, tokenIndex174 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l175 - } - position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l176 - } - position++ - goto l174 - l176: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l177 - } - position++ - goto l174 - l177: - position, tokenIndex = position174, tokenIndex174 + position157, tokenIndex157 := position, tokenIndex if buffer[position] != rune('-') { - goto l178 - } - position++ - goto l174 - l178: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('_') { - goto l179 - } - position++ - goto l174 - l179: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune(':') { - goto l170 + goto l157 } position++ + goto l158 + l157: + position, tokenIndex = position157, tokenIndex157 } - l174: - l172: + l158: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l155 + } + position++ + l159: { - position173, tokenIndex173 := position, tokenIndex - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l173 - } - position++ + position160, tokenIndex160 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l160 } - l180: - goto l172 - l173: - position, tokenIndex = position173, tokenIndex173 + position++ + goto l159 + l160: + position, tokenIndex = position160, tokenIndex160 } - add(rulePegText, position171) + { + position161, tokenIndex161 := position, tokenIndex + if buffer[position] != rune('.') { + goto l161 + } + position++ + l163: + { + position164, tokenIndex164 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l164 + } + position++ + goto l163 + l164: + position, tokenIndex = position164, tokenIndex164 + } + goto l162 + l161: + position, tokenIndex = position161, tokenIndex161 + } + l162: + add(rulePegText, position156) } { add(ruleAction35, position) } - goto l136 - l170: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('"') { - goto l187 - } - position++ + goto l139 + l155: + position, tokenIndex = position139, tokenIndex139 { - position188 := position - if !_rules[ruledoublequotedstring]() { - goto l187 + position167 := position + { + position168, tokenIndex168 := position, tokenIndex + if buffer[position] != rune('-') { + goto l168 + } + position++ + goto l169 + l168: + position, tokenIndex = position168, tokenIndex168 } - add(rulePegText, position188) + l169: + if buffer[position] != rune('.') { + goto l166 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l166 + } + position++ + l170: + { + position171, tokenIndex171 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l171 + } + position++ + goto l170 + l171: + position, tokenIndex = position171, tokenIndex171 + } + add(rulePegText, position167) } - if buffer[position] != rune('"') { - goto l187 - } - position++ { add(ruleAction36, position) } - goto l136 - l187: - position, tokenIndex = position136, tokenIndex136 - if buffer[position] != rune('\'') { - goto l134 - } - position++ + goto l139 + l166: + position, tokenIndex = position139, tokenIndex139 { - position190 := position - if !_rules[rulesinglequotedstring]() { - goto l134 + position174 := position + { + position177, tokenIndex177 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l178 + } + position++ + goto l177 + l178: + position, tokenIndex = position177, tokenIndex177 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l179 + } + position++ + goto l177 + l179: + position, tokenIndex = position177, tokenIndex177 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l180 + } + position++ + goto l177 + l180: + position, tokenIndex = position177, tokenIndex177 + if buffer[position] != rune('-') { + goto l181 + } + position++ + goto l177 + l181: + position, tokenIndex = position177, tokenIndex177 + if buffer[position] != rune('_') { + goto l182 + } + position++ + goto l177 + l182: + position, tokenIndex = position177, tokenIndex177 + if buffer[position] != rune(':') { + goto l173 + } + position++ } - add(rulePegText, position190) + l177: + l175: + { + position176, tokenIndex176 := position, tokenIndex + { + position183, tokenIndex183 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l184 + } + position++ + goto l183 + l184: + position, tokenIndex = position183, tokenIndex183 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l185 + } + position++ + goto l183 + l185: + position, tokenIndex = position183, tokenIndex183 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l186 + } + position++ + goto l183 + l186: + position, tokenIndex = position183, tokenIndex183 + if buffer[position] != rune('-') { + goto l187 + } + position++ + goto l183 + l187: + position, tokenIndex = position183, tokenIndex183 + if buffer[position] != rune('_') { + goto l188 + } + position++ + goto l183 + l188: + position, tokenIndex = position183, tokenIndex183 + if buffer[position] != rune(':') { + goto l176 + } + position++ + } + l183: + goto l175 + l176: + position, tokenIndex = position176, tokenIndex176 + } + add(rulePegText, position174) } - if buffer[position] != rune('\'') { - goto l134 - } - position++ { add(ruleAction37, position) } + goto l139 + l173: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('"') { + goto l190 + } + position++ + { + position191 := position + if !_rules[ruledoublequotedstring]() { + goto l190 + } + add(rulePegText, position191) + } + if buffer[position] != rune('"') { + goto l190 + } + position++ + { + add(ruleAction38, position) + } + goto l139 + l190: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('\'') { + goto l137 + } + position++ + { + position193 := position + if !_rules[rulesinglequotedstring]() { + goto l137 + } + add(rulePegText, position193) + } + if buffer[position] != rune('\'') { + goto l137 + } + position++ + { + add(ruleAction39, position) + } } - l136: - add(ruleitem, position135) + l139: + add(ruleitem, position138) } return true - l134: - position, tokenIndex = position134, tokenIndex134 + l137: + position, tokenIndex = position137, tokenIndex137 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position193 := position - l194: + position196 := position + l197: { - position195, tokenIndex195 := position, tokenIndex + position198, tokenIndex198 := position, tokenIndex { - position196, tokenIndex196 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex { - position198, tokenIndex198 := position, tokenIndex + position201, tokenIndex201 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex if buffer[position] != rune('"') { - goto l200 + goto l203 } position++ - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l203: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { + goto l204 + } + position++ + goto l202 + l204: + position, tokenIndex = position202, tokenIndex202 + if buffer[position] != rune('\n') { goto l201 } position++ - goto l199 - l201: - position, tokenIndex = position199, tokenIndex199 - if buffer[position] != rune('\n') { - goto l198 - } - position++ } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l202: + goto l200 + l201: + position, tokenIndex = position201, tokenIndex201 } if !matchDot() { - goto l197 + goto l200 } - goto l196 - l197: - position, tokenIndex = position196, tokenIndex196 + goto l199 + l200: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l202 + goto l205 } position++ if buffer[position] != rune('n') { - goto l202 + goto l205 } position++ - goto l196 - l202: - position, tokenIndex = position196, tokenIndex196 + goto l199 + l205: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l203 + goto l206 } position++ if buffer[position] != rune('"') { - goto l203 + goto l206 } position++ - goto l196 - l203: - position, tokenIndex = position196, tokenIndex196 + goto l199 + l206: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l204 + goto l207 } position++ if buffer[position] != rune('\'') { - goto l204 + goto l207 } position++ - goto l196 - l204: - position, tokenIndex = position196, tokenIndex196 + goto l199 + l207: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l195 + goto l198 } position++ if buffer[position] != rune('\\') { - goto l195 + goto l198 } position++ } - l196: - goto l194 - l195: - position, tokenIndex = position195, tokenIndex195 + l199: + goto l197 + l198: + position, tokenIndex = position198, tokenIndex198 } - add(ruledoublequotedstring, position193) + add(ruledoublequotedstring, position196) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position206 := position - l207: + position209 := position + l210: { - position208, tokenIndex208 := position, tokenIndex + position211, tokenIndex211 := position, tokenIndex { - position209, tokenIndex209 := position, tokenIndex + position212, tokenIndex212 := position, tokenIndex { - position211, tokenIndex211 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position212, tokenIndex212 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex if buffer[position] != rune('\'') { - goto l213 + goto l216 } position++ - goto l212 - l213: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { + goto l217 + } + position++ + goto l215 + l217: + position, tokenIndex = position215, tokenIndex215 + if buffer[position] != rune('\n') { goto l214 } position++ - goto l212 - l214: - position, tokenIndex = position212, tokenIndex212 - if buffer[position] != rune('\n') { - goto l211 - } - position++ } - l212: - goto l210 - l211: - position, tokenIndex = position211, tokenIndex211 + l215: + goto l213 + l214: + position, tokenIndex = position214, tokenIndex214 } if !matchDot() { - goto l210 + goto l213 } - goto l209 - l210: - position, tokenIndex = position209, tokenIndex209 + goto l212 + l213: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\\') { - goto l215 + goto l218 } position++ if buffer[position] != rune('n') { - goto l215 + goto l218 } position++ - goto l209 - l215: - position, tokenIndex = position209, tokenIndex209 + goto l212 + l218: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\\') { - goto l216 + goto l219 } position++ if buffer[position] != rune('"') { - goto l216 + goto l219 } position++ - goto l209 - l216: - position, tokenIndex = position209, tokenIndex209 + goto l212 + l219: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\\') { - goto l217 + goto l220 } position++ if buffer[position] != rune('\'') { - goto l217 + goto l220 } position++ - goto l209 - l217: - position, tokenIndex = position209, tokenIndex209 + goto l212 + l220: + position, tokenIndex = position212, tokenIndex212 if buffer[position] != rune('\\') { - goto l208 + goto l211 } position++ if buffer[position] != rune('\\') { - goto l208 + goto l211 } position++ } - l209: - goto l207 - l208: - position, tokenIndex = position208, tokenIndex208 + l212: + goto l210 + l211: + position, tokenIndex = position211, tokenIndex211 } - add(rulesinglequotedstring, position206) + add(rulesinglequotedstring, position209) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position218, tokenIndex218 := position, tokenIndex + position221, tokenIndex221 := position, tokenIndex { - position219 := position + position222 := position { - position220, tokenIndex220 := position, tokenIndex + position223, tokenIndex223 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l224 + } + position++ + goto l223 + l224: + position, tokenIndex = position223, tokenIndex223 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l221 } position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l218 - } - position++ } - l220: - l222: + l223: + l225: { - position223, tokenIndex223 := position, tokenIndex + position226, tokenIndex226 := position, tokenIndex { - position224, tokenIndex224 := position, tokenIndex + position227, tokenIndex227 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l225 - } - position++ - goto l224 - l225: - position, tokenIndex = position224, tokenIndex224 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l226 - } - position++ - goto l224 - l226: - position, tokenIndex = position224, tokenIndex224 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l227 - } - position++ - goto l224 - l227: - position, tokenIndex = position224, tokenIndex224 - if buffer[position] != rune('_') { goto l228 } position++ - goto l224 + goto l227 l228: - position, tokenIndex = position224, tokenIndex224 + position, tokenIndex = position227, tokenIndex227 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l229 + } + position++ + goto l227 + l229: + position, tokenIndex = position227, tokenIndex227 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l230 + } + position++ + goto l227 + l230: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l231 + } + position++ + goto l227 + l231: + position, tokenIndex = position227, tokenIndex227 if buffer[position] != rune('-') { - goto l223 + goto l226 } position++ } - l224: - goto l222 - l223: - position, tokenIndex = position223, tokenIndex223 + l227: + goto l225 + l226: + position, tokenIndex = position226, tokenIndex226 } - add(rulefieldExpr, position219) + add(rulefieldExpr, position222) } return true - l218: - position, tokenIndex = position218, tokenIndex218 + l221: + position, tokenIndex = position221, tokenIndex221 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action38)> */ + /* 17 field <- <(<(fieldExpr / reserved)> Action40)> */ func() bool { - position229, tokenIndex229 := position, tokenIndex + position232, tokenIndex232 := position, tokenIndex { - position230 := position + position233 := position { - position231 := position + position234 := position { - position232, tokenIndex232 := position, tokenIndex + position235, tokenIndex235 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l233 + goto l236 } - goto l232 - l233: - position, tokenIndex = position232, tokenIndex232 + goto l235 + l236: + position, tokenIndex = position235, tokenIndex235 { - position234 := position + position237 := position { - position235, tokenIndex235 := position, tokenIndex + position238, tokenIndex238 := position, tokenIndex if buffer[position] != rune('_') { - goto l236 + goto l239 } position++ if buffer[position] != rune('r') { - goto l236 + goto l239 } position++ if buffer[position] != rune('o') { - goto l236 + goto l239 } position++ if buffer[position] != rune('w') { - goto l236 + goto l239 } position++ - goto l235 - l236: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l239: + position, tokenIndex = position238, tokenIndex238 if buffer[position] != rune('_') { - goto l237 + goto l240 } position++ if buffer[position] != rune('c') { - goto l237 + goto l240 } position++ if buffer[position] != rune('o') { - goto l237 + goto l240 } position++ if buffer[position] != rune('l') { - goto l237 + goto l240 } position++ - goto l235 - l237: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l240: + position, tokenIndex = position238, tokenIndex238 if buffer[position] != rune('_') { - goto l238 + goto l241 } position++ if buffer[position] != rune('s') { - goto l238 + goto l241 } position++ if buffer[position] != rune('t') { - goto l238 + goto l241 } position++ if buffer[position] != rune('a') { - goto l238 + goto l241 } position++ if buffer[position] != rune('r') { - goto l238 + goto l241 } position++ if buffer[position] != rune('t') { - goto l238 + goto l241 } position++ - goto l235 - l238: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l241: + position, tokenIndex = position238, tokenIndex238 if buffer[position] != rune('_') { - goto l239 + goto l242 } position++ if buffer[position] != rune('e') { - goto l239 + goto l242 } position++ if buffer[position] != rune('n') { - goto l239 + goto l242 } position++ if buffer[position] != rune('d') { - goto l239 + goto l242 } position++ - goto l235 - l239: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l242: + position, tokenIndex = position238, tokenIndex238 if buffer[position] != rune('_') { - goto l240 + goto l243 } position++ if buffer[position] != rune('t') { - goto l240 + goto l243 } position++ if buffer[position] != rune('i') { - goto l240 + goto l243 } position++ if buffer[position] != rune('m') { - goto l240 + goto l243 } position++ if buffer[position] != rune('e') { - goto l240 + goto l243 } position++ if buffer[position] != rune('s') { - goto l240 + goto l243 } position++ if buffer[position] != rune('t') { - goto l240 + goto l243 } position++ if buffer[position] != rune('a') { - goto l240 + goto l243 } position++ if buffer[position] != rune('m') { - goto l240 + goto l243 } position++ if buffer[position] != rune('p') { - goto l240 + goto l243 } position++ - goto l235 - l240: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l243: + position, tokenIndex = position238, tokenIndex238 if buffer[position] != rune('_') { - goto l229 + goto l232 } position++ if buffer[position] != rune('f') { - goto l229 + goto l232 } position++ if buffer[position] != rune('i') { - goto l229 + goto l232 } position++ if buffer[position] != rune('e') { - goto l229 + goto l232 } position++ if buffer[position] != rune('l') { - goto l229 + goto l232 } position++ if buffer[position] != rune('d') { - goto l229 + goto l232 } position++ } - l235: - add(rulereserved, position234) + l238: + add(rulereserved, position237) } } - l232: - add(rulePegText, position231) + l235: + add(rulePegText, position234) } { - add(ruleAction38, position) + add(ruleAction40, position) } - add(rulefield, position230) + add(rulefield, position233) } return true - l229: - position, tokenIndex = position229, tokenIndex229 + l232: + position, tokenIndex = position232, tokenIndex232 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action39)> */ + /* 19 posfield <- <( Action41)> */ func() bool { - position243, tokenIndex243 := position, tokenIndex + position246, tokenIndex246 := position, tokenIndex { - position244 := position + position247 := position { - position245 := position + position248 := position if !_rules[rulefieldExpr]() { - goto l243 + goto l246 } - add(rulePegText, position245) + add(rulePegText, position248) } { - add(ruleAction39, position) + add(ruleAction41, position) } - add(ruleposfield, position244) + add(ruleposfield, position247) } return true - l243: - position, tokenIndex = position243, tokenIndex243 + l246: + position, tokenIndex = position246, tokenIndex246 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position247, tokenIndex247 := position, tokenIndex + position250, tokenIndex250 := position, tokenIndex { - position248 := position + position251 := position { - position249, tokenIndex249 := position, tokenIndex + position252, tokenIndex252 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l253 + } + position++ + l254: + { + position255, tokenIndex255 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l255 + } + position++ + goto l254 + l255: + position, tokenIndex = position255, tokenIndex255 + } + goto l252 + l253: + position, tokenIndex = position252, tokenIndex252 + if buffer[position] != rune('0') { goto l250 } position++ - l251: - { - position252, tokenIndex252 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l252 - } - position++ - goto l251 - l252: - position, tokenIndex = position252, tokenIndex252 - } - goto l249 - l250: - position, tokenIndex = position249, tokenIndex249 - if buffer[position] != rune('0') { - goto l247 - } - position++ } - l249: - add(ruleuint, position248) + l252: + add(ruleuint, position251) } return true - l247: - position, tokenIndex = position247, tokenIndex247 + l250: + position, tokenIndex = position250, tokenIndex250 return false }, - /* 21 uintrow <- <( Action40)> */ + /* 21 uintrow <- <( Action42)> */ nil, - /* 22 col <- <(( Action41) / ('\'' '\'' Action42) / ('"' '"' Action43))> */ + /* 22 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ func() bool { - position254, tokenIndex254 := position, tokenIndex + position257, tokenIndex257 := position, tokenIndex { - position255 := position + position258 := position { - position256, tokenIndex256 := position, tokenIndex - { - position258 := position - if !_rules[ruleuint]() { - goto l257 - } - add(rulePegText, position258) - } - { - add(ruleAction41, position) - } - goto l256 - l257: - position, tokenIndex = position256, tokenIndex256 - if buffer[position] != rune('\'') { - goto l260 - } - position++ + position259, tokenIndex259 := position, tokenIndex { position261 := position - if !_rules[rulesinglequotedstring]() { + if !_rules[ruleuint]() { goto l260 } add(rulePegText, position261) } - if buffer[position] != rune('\'') { - goto l260 - } - position++ - { - add(ruleAction42, position) - } - goto l256 - l260: - position, tokenIndex = position256, tokenIndex256 - if buffer[position] != rune('"') { - goto l254 - } - position++ - { - position263 := position - if !_rules[ruledoublequotedstring]() { - goto l254 - } - add(rulePegText, position263) - } - if buffer[position] != rune('"') { - goto l254 - } - position++ { add(ruleAction43, position) } + goto l259 + l260: + position, tokenIndex = position259, tokenIndex259 + if buffer[position] != rune('\'') { + goto l263 + } + position++ + { + position264 := position + if !_rules[rulesinglequotedstring]() { + goto l263 + } + add(rulePegText, position264) + } + if buffer[position] != rune('\'') { + goto l263 + } + position++ + { + add(ruleAction44, position) + } + goto l259 + l263: + position, tokenIndex = position259, tokenIndex259 + if buffer[position] != rune('"') { + goto l257 + } + position++ + { + position266 := position + if !_rules[ruledoublequotedstring]() { + goto l257 + } + add(rulePegText, position266) + } + if buffer[position] != rune('"') { + goto l257 + } + position++ + { + add(ruleAction45, position) + } } - l256: - add(rulecol, position255) + l259: + add(rulecol, position258) } return true - l254: - position, tokenIndex = position254, tokenIndex254 + l257: + position, tokenIndex = position257, tokenIndex257 return false }, - /* 23 row <- <(( Action44) / ('\'' '\'' Action45) / ('"' '"' Action46))> */ + /* 23 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ nil, /* 24 open <- <('(' sp)> */ func() bool { - position266, tokenIndex266 := position, tokenIndex + position269, tokenIndex269 := position, tokenIndex { - position267 := position + position270 := position if buffer[position] != rune('(') { - goto l266 + goto l269 } position++ if !_rules[rulesp]() { - goto l266 + goto l269 } - add(ruleopen, position267) + add(ruleopen, position270) } return true - l266: - position, tokenIndex = position266, tokenIndex266 + l269: + position, tokenIndex = position269, tokenIndex269 return false }, /* 25 close <- <(')' sp)> */ func() bool { - position268, tokenIndex268 := position, tokenIndex + position271, tokenIndex271 := position, tokenIndex { - position269 := position + position272 := position if buffer[position] != rune(')') { - goto l268 + goto l271 } position++ if !_rules[rulesp]() { - goto l268 + goto l271 } - add(ruleclose, position269) + add(ruleclose, position272) } return true - l268: - position, tokenIndex = position268, tokenIndex268 + l271: + position, tokenIndex = position271, tokenIndex271 return false }, /* 26 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position271 := position - l272: + position274 := position + l275: { - position273, tokenIndex273 := position, tokenIndex + position276, tokenIndex276 := position, tokenIndex { - position274, tokenIndex274 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex if buffer[position] != rune(' ') { - goto l275 + goto l278 } position++ - goto l274 - l275: - position, tokenIndex = position274, tokenIndex274 + goto l277 + l278: + position, tokenIndex = position277, tokenIndex277 if buffer[position] != rune('\t') { + goto l279 + } + position++ + goto l277 + l279: + position, tokenIndex = position277, tokenIndex277 + if buffer[position] != rune('\n') { goto l276 } position++ - goto l274 - l276: - position, tokenIndex = position274, tokenIndex274 - if buffer[position] != rune('\n') { - goto l273 - } - position++ } - l274: - goto l272 - l273: - position, tokenIndex = position273, tokenIndex273 + l277: + goto l275 + l276: + position, tokenIndex = position276, tokenIndex276 } - add(rulesp, position271) + add(rulesp, position274) } return true }, /* 27 comma <- <(sp ',' sp)> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex { - position278 := position + position281 := position if !_rules[rulesp]() { - goto l277 + goto l280 } if buffer[position] != rune(',') { - goto l277 + goto l280 } position++ if !_rules[rulesp]() { - goto l277 + goto l280 } - add(rulecomma, position278) + add(rulecomma, position281) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l280: + position, tokenIndex = position280, tokenIndex280 return false }, /* 28 lbrack <- <('[' sp)> */ @@ -2682,139 +2740,139 @@ func (p *PQL) Init() { nil, /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position282, tokenIndex282 := position, tokenIndex + position285, tokenIndex285 := position, tokenIndex { - position283 := position + position286 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if buffer[position] != rune('-') { - goto l282 + goto l285 } position++ { - position284, tokenIndex284 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex if buffer[position] != rune('0') { + goto l288 + } + position++ + goto l287 + l288: + position, tokenIndex = position287, tokenIndex287 + if buffer[position] != rune('1') { goto l285 } position++ - goto l284 - l285: - position, tokenIndex = position284, tokenIndex284 - if buffer[position] != rune('1') { - goto l282 - } - position++ } - l284: + l287: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if buffer[position] != rune('-') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if buffer[position] != rune('T') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if buffer[position] != rune(':') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l282 + goto l285 } position++ - add(ruletimestampbasicfmt, position283) + add(ruletimestampbasicfmt, position286) } return true - l282: - position, tokenIndex = position282, tokenIndex282 + l285: + position, tokenIndex = position285, tokenIndex285 return false }, /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position286, tokenIndex286 := position, tokenIndex + position289, tokenIndex289 := position, tokenIndex { - position287 := position + position290 := position { - position288, tokenIndex288 := position, tokenIndex + position291, tokenIndex291 := position, tokenIndex if buffer[position] != rune('"') { - goto l289 + goto l292 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l289 + goto l292 } if buffer[position] != rune('"') { + goto l292 + } + position++ + goto l291 + l292: + position, tokenIndex = position291, tokenIndex291 + if buffer[position] != rune('\'') { + goto l293 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l293 + } + if buffer[position] != rune('\'') { + goto l293 + } + position++ + goto l291 + l293: + position, tokenIndex = position291, tokenIndex291 + if !_rules[ruletimestampbasicfmt]() { goto l289 } - position++ - goto l288 - l289: - position, tokenIndex = position288, tokenIndex288 - if buffer[position] != rune('\'') { - goto l290 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l290 - } - if buffer[position] != rune('\'') { - goto l290 - } - position++ - goto l288 - l290: - position, tokenIndex = position288, tokenIndex288 - if !_rules[ruletimestampbasicfmt]() { - goto l286 - } } - l288: - add(ruletimestampfmt, position287) + l291: + add(ruletimestampfmt, position290) } return true - l286: - position, tokenIndex = position286, tokenIndex286 + l289: + position, tokenIndex = position289, tokenIndex289 return false }, - /* 33 timestamp <- <( Action47)> */ + /* 33 timestamp <- <( Action49)> */ nil, /* 35 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2832,86 +2890,90 @@ func (p *PQL) Init() { nil, /* 42 Action7 <- <{p.endCall()}> */ nil, - /* 43 Action8 <- <{p.startCall("TopN")}> */ + /* 43 Action8 <- <{p.startCall("ClearRow")}> */ nil, /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 45 Action10 <- <{p.startCall("Range")}> */ + /* 45 Action10 <- <{p.startCall("TopN")}> */ nil, /* 46 Action11 <- <{p.endCall()}> */ nil, + /* 47 Action12 <- <{p.startCall("Range")}> */ nil, - /* 48 Action12 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action13 <- <{p.endCall()}> */ nil, - /* 49 Action13 <- <{ p.endCall() }> */ nil, - /* 50 Action14 <- <{ p.addBTWN() }> */ + /* 50 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 51 Action15 <- <{ p.addLTE() }> */ + /* 51 Action15 <- <{ p.endCall() }> */ nil, - /* 52 Action16 <- <{ p.addGTE() }> */ + /* 52 Action16 <- <{ p.addBTWN() }> */ nil, - /* 53 Action17 <- <{ p.addEQ() }> */ + /* 53 Action17 <- <{ p.addLTE() }> */ nil, - /* 54 Action18 <- <{ p.addNEQ() }> */ + /* 54 Action18 <- <{ p.addGTE() }> */ nil, - /* 55 Action19 <- <{ p.addLT() }> */ + /* 55 Action19 <- <{ p.addEQ() }> */ nil, - /* 56 Action20 <- <{ p.addGT() }> */ + /* 56 Action20 <- <{ p.addNEQ() }> */ nil, - /* 57 Action21 <- <{p.startConditional()}> */ + /* 57 Action21 <- <{ p.addLT() }> */ nil, - /* 58 Action22 <- <{p.endConditional()}> */ + /* 58 Action22 <- <{ p.addGT() }> */ nil, - /* 59 Action23 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action23 <- <{p.startConditional()}> */ nil, - /* 60 Action24 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action24 <- <{p.endConditional()}> */ nil, /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 62 Action26 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action26 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 63 Action27 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action28 <- <{ p.startList() }> */ + /* 64 Action28 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 65 Action29 <- <{ p.endList() }> */ + /* 65 Action29 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 66 Action30 <- <{ p.addVal(nil) }> */ + /* 66 Action30 <- <{ p.startList() }> */ nil, - /* 67 Action31 <- <{ p.addVal(true) }> */ + /* 67 Action31 <- <{ p.endList() }> */ nil, - /* 68 Action32 <- <{ p.addVal(false) }> */ + /* 68 Action32 <- <{ p.addVal(nil) }> */ nil, - /* 69 Action33 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action33 <- <{ p.addVal(true) }> */ nil, - /* 70 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action34 <- <{ p.addVal(false) }> */ nil, - /* 71 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 72 Action36 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 74 Action38 <- <{ p.addField(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 75 Action39 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 75 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action40 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 76 Action40 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 77 Action41 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 77 Action41 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 78 Action42 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 78 Action42 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 79 Action43 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 80 Action44 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 80 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 81 Action45 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 81 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 82 Action46 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 82 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 83 Action47 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 83 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 84 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 85 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules From 1b6d738bab76cd1ad6bc61ed610756b01750f2ad Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 17 Sep 2018 14:59:38 -0500 Subject: [PATCH 20/45] Support ranked cache value=0. Since there is no way to remove a value from the cache (because the interface doesn't support it), this PR modifies the ranked cache implemetation to allow setting the cache value to 0 for a row. Doing so effectively removes that row from the cache. Values below the threshold (other than 0) are still ignored by the ranked cache's Add() method. --- cache.go | 6 +++-- executor_test.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/cache.go b/cache.go index 8ae8caefc..df82a5802 100644 --- a/cache.go +++ b/cache.go @@ -167,8 +167,10 @@ func NewRankCache(maxEntries uint32) *rankCache { func (c *rankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the column count is below the threshold. - if n < c.thresholdValue { + // Ignore if the column count is below the threshold, + // unless the count is 0, which is effectively used + // to clear the cache value. + if n < c.thresholdValue && n > 0 { return } diff --git a/executor_test.go b/executor_test.go index dd8be0db8..d6d529efe 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1841,6 +1841,76 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { t.Fatal("expected clear row to return an error") } }) + t.Run("TopN", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + cc := ` + Set(2, f=1) + Set(3, f=1) + Set(4, f=1) + Set(5, f=1) + Set(6, f=1) + Set(7, f=1) + Set(8, f=1) + + Set(2, f=2) + Set(3, f=2) + Set(4, f=2) + Set(5, f=2) + Set(6, f=2) + Set(7, f=2) + + Set(2, f=3) + Set(3, f=3) + Set(4, f=3) + Set(5, f=3) + Set(6, f=3) + ` + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + if err := c[0].RecalculateCaches(); err != nil { + t.Fatalf("recalculating caches: %v", err) + } + + // Check the TopN results. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 2, Count: 6}, + {ID: 3, Count: 5}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + + // Clear the row and ensure we get a `true` response. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected clear row result: %+v", res) + } + + // Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache). + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 3, Count: 5}, + }}) { + t.Fatalf("topn wrong results: %v", res.Results) + } + }) } func benchmarkExistence(nn bool, b *testing.B) { From 506e6dcf51ac946f9bb8f73ff40f30025d7b5213 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 24 Sep 2018 16:03:50 -0500 Subject: [PATCH 21/45] change ClearRow validation to whitelist of field types --- executor.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 60c057ce1..8dd4db569 100644 --- a/executor.go +++ b/executor.go @@ -1150,8 +1150,12 @@ func (e *executor) executeClearRow(ctx context.Context, index string, c *pql.Cal if field == nil { return false, ErrFieldNotFound } - if field.Type() == FieldTypeInt { - return false, errors.New("ClearRow() is not supported on `int` fields") + + switch field.Type() { + case FieldTypeSet, FieldTypeTime, FieldTypeMutex, FieldTypeBool: + // These field types support ClearRow(). + default: + return false, fmt.Errorf("ClearRow() is not supported on %s field types", field.Type()) } // Execute calls in bulk on each remote node and merge. From 4a411067a560990d656f325965d5df63784acb03 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 24 Sep 2018 16:16:22 -0500 Subject: [PATCH 22/45] remove Set() (and Clear()) from vector interface, and add error to return --- fragment.go | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/fragment.go b/fragment.go index 095308d09..ee586afc6 100644 --- a/fragment.go +++ b/fragment.go @@ -391,12 +391,13 @@ func (f *fragment) setBit(rowID, columnID uint64) (changed bool, err error) { // handleMutex will clear an existing row and store the new row // in the vector. func (f *fragment) handleMutex(rowID, columnID uint64) error { - if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID { + if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil { + return errors.Wrap(err, "getting mutex vector data") + } else if found && existingRowID != rowID { if _, err := f.unprotectedClearBit(existingRowID, columnID); err != nil { return errors.Wrap(err, "clearing mutex value") } } - f.mutexVector.Set(columnID, rowID) return nil } @@ -1469,7 +1470,9 @@ func (f *fragment) bulkImportMutex(rowIDs, columnIDs []uint64) error { rowID, columnID := rowIDs[i], columnIDs[i] // Handle mutex vector (i.e. clear an existing row). - if existingRowID, found := f.mutexVector.Get(columnID); found && existingRowID != rowID { + if existingRowID, found, err := f.mutexVector.Get(columnID); err != nil { + return errors.Wrap(err, "getting mutex vector data") + } else if found && existingRowID != rowID { // Determine the position of the bit in the storage. pos, err := f.pos(existingRowID, columnID) if err != nil { @@ -2219,14 +2222,7 @@ func pos(rowID, columnID uint64) uint64 { // vector stores the mapping of colID to rowID. // It's used for a mutex field type. type vector interface { - Get(colID uint64) (uint64, bool) - Set(colID, rowID uint64) - // TODO: Clear(colID, rowID uint64) - // Set() and Clear() are required for implementations - // where the vector data is maintained independently - // from the fragment data. In those cases, the Set() - // and Clear() would also need to address the ranked - // cache as well as the effects of ClearRow(). + Get(colID uint64) (uint64, bool, error) } // rowsVector implements the vector interface by looking @@ -2245,17 +2241,16 @@ func newRowsVector(f *fragment) *rowsVector { // Get returns the rowID associated to the given colID. // Additionally, it returns true if a value was found, // otherwise it returns false. -func (v *rowsVector) Get(colID uint64) (uint64, bool) { +func (v *rowsVector) Get(colID uint64) (uint64, bool, error) { rows := v.f.rowsForColumn(colID) - if len(rows) == 1 { - return rows[0], true + if len(rows) > 1 { + return 0, false, errors.New("found multiple row values for column") + } else if len(rows) == 1 { + return rows[0], true, nil } - return 0, false + return 0, false, nil } -// Set is not used for rowsVector. -func (v *rowsVector) Set(colID, rowID uint64) {} - // boolVector implements the vector interface by looking // at data in rows 0 and 1. type boolVector struct { @@ -2272,16 +2267,17 @@ func newBoolVector(f *fragment) *boolVector { // Get returns the rowID associated to the given colID. // Additionally, it returns true if a value was found, // otherwise it returns false. -func (v *boolVector) Get(colID uint64) (uint64, bool) { +func (v *boolVector) Get(colID uint64) (uint64, bool, error) { rows := v.f.rowsForColumn(colID) - if len(rows) == 1 { + if len(rows) > 1 { + return 0, false, errors.New("found multiple row values for column") + } else if len(rows) == 1 { switch rows[0] { case falseRowID, trueRowID: - return rows[0], true + return rows[0], true, nil + default: + return 0, false, errors.New("found non-boolean value") } } - return 0, false + return 0, false, nil } - -// Set is not used for boolVector. -func (v *boolVector) Set(colID, rowID uint64) {} From ee503b6ec0e31c1f5d976a2e8b01c204e5c41c13 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 24 Sep 2018 16:49:39 -0500 Subject: [PATCH 23/45] remove unnecessary buffer on available shard write to file --- field.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/field.go b/field.go index 6d90a57ce..27de57d8f 100644 --- a/field.go +++ b/field.go @@ -15,7 +15,6 @@ package pilosa import ( - "bytes" "encoding/json" "fmt" "io/ioutil" @@ -505,20 +504,18 @@ func (f *Field) loadAvailableShards() error { // saveAvailableShards writes remoteAvailableShards data for the field. func (f *Field) saveAvailableShards() error { + // Open or create file. + file, err := os.OpenFile(filepath.Join(f.path, ".available.shards"), os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return errors.Wrap(err, "opening available shards file") + } + f.mu.RLock() defer f.mu.RUnlock() - // Write available shards to buffer. - var buf bytes.Buffer - if n, err := f.remoteAvailableShards.WriteTo(&buf); err != nil { + // Write available shards to file. + if _, err := f.remoteAvailableShards.WriteTo(file); err != nil { return errors.Wrap(err, "writing bitmap to buffer") - } else if n != int64(buf.Len()) { - return fmt.Errorf("buffer size mismatch: %d != %d", n, buf.Len()) - } - - // Write buffer to file. - if err := ioutil.WriteFile(filepath.Join(f.path, ".available.shards"), buf.Bytes(), 0666); err != nil { - return errors.Wrap(err, "writing available shards") } return nil From 64e86614f84ca0d2034ced612781764ff6b0de01 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 25 Sep 2018 09:14:19 -0500 Subject: [PATCH 24/45] shrank n(container bit count cache) to int32 --- enterprise/b/containers_btree.go | 4 +- roaring/containers.go | 2 +- roaring/roaring.go | 230 +++++++++++++++---------------- roaring/roaring_internal_test.go | 68 ++++----- 4 files changed, 152 insertions(+), 152 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index d687c88a3..789936a24 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -88,12 +88,12 @@ func (u updater) update(oldV *roaring.Container, exists bool) (*roaring.Containe // this struct is added to prevent the closure locals from being escaped out to the heap type updater struct { key uint64 - n int + n int32 containerType byte mapped bool } -func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int32, mapped bool) { a := updater{key, n, containerType, mapped} btc.tree.Put(key, a.update) } diff --git a/roaring/containers.go b/roaring/containers.go index 5275d4f04..bc458fba2 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -46,7 +46,7 @@ func (sc *sliceContainers) Put(key uint64, c *Container) { } -func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { +func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int32, mapped bool) { i := search64(sc.keys, key) if i < 0 { c := NewContainer() diff --git a/roaring/roaring.go b/roaring/roaring.go index 6eaad740d..fd9f15d92 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -73,7 +73,7 @@ type Containers interface { // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. - PutContainerValues(key uint64, containerType byte, n int, mapped bool) + PutContainerValues(key uint64, containerType byte, n int32, mapped bool) // Remove takes the container at key out. Remove(key uint64) @@ -242,7 +242,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { if found && skey == ekey { citer.Next() _, c := citer.Value() - return uint64(c.countRange(int(lowbits(start)), int(lowbits(end)))) + return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end)))) } for citer.Next() { @@ -252,7 +252,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { panic("should be impossible for k to be less than skey") } if k == skey { - n += uint64(c.countRange(int(lowbits(start)), maxContainerVal+1)) + n += uint64(c.countRange(int32(lowbits(start)), maxContainerVal+1)) continue } if k < ekey { @@ -260,7 +260,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) { continue } if k == ekey { - n += uint64(c.countRange(0, int(lowbits(end)))) + n += uint64(c.countRange(0, int32(lowbits(end)))) break } if k > ekey { @@ -644,7 +644,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { b.Containers.PutContainerValues( binary.LittleEndian.Uint64(buf[0:8]), byte(binary.LittleEndian.Uint16(buf[8:10])), - int(binary.LittleEndian.Uint16(buf[10:12]))+1, + int32(binary.LittleEndian.Uint16(buf[10:12]))+1, true) } opsOffset := headerSize + int(keyN)*12 @@ -805,7 +805,7 @@ type Iterator struct { citer ContainerIterator key uint64 c *Container - j, k int // i: container; j: array index, bit index, or run index; k: offset within the run + j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run } // Seek moves to the first value equal to or greater than `seek`. @@ -830,7 +830,7 @@ func (itr *Iterator) Seek(seek uint64) { if itr.j < 0 { itr.j = -itr.j - 1 } - if itr.j < len(itr.c.array) { + if itr.j < int32(len(itr.c.array)) { itr.j-- return } @@ -853,7 +853,7 @@ func (itr *Iterator) Seek(seek uint64) { j, contains := binSearchRuns(lb, itr.c.runs) if contains { itr.j = j - itr.k = int(lb) - int(itr.c.runs[j].start) - 1 + itr.k = int32(lb) - int32(itr.c.runs[j].start) - 1 } else { // Set iterator to next value in the Bitmap. itr.j = j @@ -864,7 +864,7 @@ func (itr *Iterator) Seek(seek uint64) { } // If it's a bitmap container then move to index before the value and call next(). - itr.j = int(lb) - 1 + itr.j = int32(lb) - 1 } // Next returns the next value in the bitmap. @@ -914,14 +914,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } r := itr.c.runs[itr.j] - runLength := int(r.last - r.start) + runLength := int32(r.last - r.start) if itr.k >= runLength { // Reached end of run, move to the next run. itr.j, itr.k = itr.j+1, -1 } - if itr.j >= len(itr.c.runs) { + if itr.j >= int32(len(itr.c.runs)) { // Reached end of runs, move to the next container. if !itr.citer.Next() { itr.c = nil @@ -942,7 +942,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) { // Find first non-zero bit in current bitmap, if possible. hb := itr.j >> 6 - if hb >= len(itr.c.bitmap) { + if hb >= int32(len(itr.c.bitmap)) { if !itr.citer.Next() { itr.c = nil return 0, true @@ -953,14 +953,14 @@ func (itr *Iterator) Next() (v uint64, eof bool) { } lb := itr.c.bitmap[hb] >> (uint(itr.j) % 64) if lb != 0 { - itr.j = itr.j + trailingZeroN(lb) + itr.j = itr.j + int32(trailingZeroN(lb)) return itr.peek(), false } // Otherwise iterate through remaining bitmaps to find next bit. - for hb++; hb < len(itr.c.bitmap); hb++ { + for hb++; hb < int32(len(itr.c.bitmap)); hb++ { if itr.c.bitmap[hb] != 0 { - itr.j = hb<<6 + trailingZeroN(itr.c.bitmap[hb]) + itr.j = hb<<6 + int32(trailingZeroN(itr.c.bitmap[hb])) return itr.peek(), false } } @@ -1008,7 +1008,7 @@ const runMaxSize = 2048 type Container struct { mapped bool // mapped directly to a byte slice when true containerType byte // array, bitmap, or run - n int // number of integers in container + n int32 // number of integers in container array []uint16 // used for array containers bitmap []uint64 // used for bitmap containers runs []interval16 // used for RLE containers @@ -1020,8 +1020,8 @@ type interval16 struct { } // runlen returns the count of integers in the interval. -func (iv interval16) runlen() int { - return 1 + int(iv.last-iv.start) +func (iv interval16) runlen() int32 { + return 1 + int32(iv.last-iv.start) } // newContainer returns a new instance of container. @@ -1035,12 +1035,12 @@ func (c *Container) Mapped() bool { } // N returns the cached bit count of the container -func (c *Container) N() int { +func (c *Container) N() int32 { return c.n } // Update updates the container -func (c *Container) Update(containerType byte, n int, mapped bool) { +func (c *Container) Update(containerType byte, n int32, mapped bool) { c.containerType = containerType c.n = n c.mapped = mapped @@ -1088,12 +1088,12 @@ func (c *Container) unmap() { } // count counts all bits in the container. -func (c *Container) count() (n int) { +func (c *Container) count() (n int32) { return c.countRange(0, maxContainerVal+1) } // countRange counts the number of bits set between [start, end). -func (c *Container) countRange(start, end int) (n int) { +func (c *Container) countRange(start, end int32) (n int32) { if c.isArray() { return c.arrayCountRange(start, end) } else if c.isRun() { @@ -1102,10 +1102,10 @@ func (c *Container) countRange(start, end int) (n int) { return c.bitmapCountRange(start, end) } -func (c *Container) arrayCountRange(start, end int) (n int) { - i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start }) - for ; i < len(c.array); i++ { - v := int(c.array[i]) +func (c *Container) arrayCountRange(start, end int32) (n int32) { + i := int32(sort.Search(int(len(c.array)), func(i int) bool { return int32(c.array[i]) >= start })) + for ; i < int32(len(c.array)); i++ { + v := int32(c.array[i]) if v >= end { break } @@ -1114,14 +1114,14 @@ func (c *Container) arrayCountRange(start, end int) (n int) { return n } -func (c *Container) bitmapCountRange(start, end int) int { +func (c *Container) bitmapCountRange(start, end int32) int32 { var n uint64 i, j := start/64, end/64 // Special case when start and end fall in the same word. if i == j { offi, offj := uint(start%64), uint(64-end%64) n += popcount((c.bitmap[i] >> offi) << (offj + offi)) - return int(n) + return int32(n) } // Count partial starting word. @@ -1136,39 +1136,39 @@ func (c *Container) bitmapCountRange(start, end int) int { } // Count partial ending word. - if j < len(c.bitmap) { + if j < int32(len(c.bitmap)) { off := 64 - (uint(end) % 64) n += popcount(c.bitmap[j] << off) } - return int(n) + return int32(n) } -func (c *Container) runCountRange(start, end int) (n int) { +func (c *Container) runCountRange(start, end int32) (n int32) { for _, iv := range c.runs { // iv is before range - if int(iv.last) < start { + if int32(iv.last) < start { continue } // iv is after range - if end < int(iv.start) { + if end < int32(iv.start) { break } // iv is superset of range - if int(iv.start) < start && int(iv.last) > end { + if int32(iv.start) < start && int32(iv.last) > end { return end - start } // iv is subset of range - if int(iv.start) >= start && int(iv.last) < end { + if int32(iv.start) >= start && int32(iv.last) < end { n += iv.runlen() } // iv overlaps beginning of range - if int(iv.start) < start && int(iv.last) < end { - n += int(iv.last) - start + 1 + if int32(iv.start) < start && int32(iv.last) < end { + n += int32(iv.last) - start + 1 } // iv overlaps end of range - if int(iv.start) > start && int(iv.last) >= end { - n += end - int(iv.start) + if int32(iv.start) > start && int32(iv.last) >= end { + n += end - int32(iv.start) } } return n @@ -1286,34 +1286,34 @@ func (c *Container) Contains(v uint16) bool { } } -func (c *Container) bitmapCountRuns() (r int) { +func (c *Container) bitmapCountRuns() (r int32) { for i := 0; i < 1023; i++ { v, v1 := c.bitmap[i], c.bitmap[i+1] - r = r + int(popcount((v<<1)&^v)+((v>>63)&^v1)) + r = r + int32(popcount((v<<1)&^v)+((v>>63)&^v1)) } vl := c.bitmap[len(c.bitmap)-1] - r = r + int(popcount((vl<<1)&^vl)+vl>>63) + r = r + int32(popcount((vl<<1)&^vl)+vl>>63) return r } -func (c *Container) arrayCountRuns() (r int) { - prev := -2 +func (c *Container) arrayCountRuns() (r int32) { + prev := int32(-2) for _, v := range c.array { - if prev+1 != int(v) { + if prev+1 != int32(v) { r++ } - prev = int(v) + prev = int32(v) } return r } -func (c *Container) countRuns() (r int) { +func (c *Container) countRuns() (r int32) { if c.isArray() { return c.arrayCountRuns() } else if c.isBitmap() { return c.bitmapCountRuns() } else if c.isRun() { - return len(c.runs) + return int32(len(c.runs)) } // sure hope this never happens @@ -1369,10 +1369,10 @@ func (c *Container) bitmapContains(v uint16) bool { // binSearchRuns returns the index of the run containing v, and true, when v is contained; // or the index of the next run starting after v, and false, when v is not contained. -func binSearchRuns(v uint16, a []interval16) (int, bool) { - i := sort.Search(len(a), - func(i int) bool { return a[i].last >= v }) - if i < len(a) { +func binSearchRuns(v uint16, a []interval16) (int32, bool) { + i := int32(sort.Search(len(a), + func(i int) bool { return a[i].last >= v })) + if i < int32(len(a)) { return i, (v >= a[i].start) && (v <= a[i].last) } @@ -1767,7 +1767,7 @@ func (c *Container) check() error { var a ErrorList if c.isArray() { - if len(c.array) != c.n { + if int32(len(c.array)) != c.n { a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n)) } } else if c.isRun() { @@ -1796,7 +1796,7 @@ func (c *Container) check() error { type containerInfo struct { Key uint64 // container key Type string // container type (array, bitmap, or run) - N int // number of bits + N int32 // number of bits Alloc int // memory used Pointer unsafe.Pointer // offset within the mmap } @@ -1838,7 +1838,7 @@ func flipRun(b *Container) *Container { return flipBitmap(x) } -func intersectionCount(a, b *Container) int { +func intersectionCount(a, b *Container) int32 { if a.isArray() { if b.isArray() { return intersectionCountArrayArray(a, b) @@ -1866,7 +1866,7 @@ func intersectionCount(a, b *Container) int { } } -func intersectionCountArrayArray(a, b *Container) (n int) { +func intersectionCountArrayArray(a, b *Container) (n int32) { na, nb := len(a.array), len(b.array) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.array[j] @@ -1882,7 +1882,7 @@ func intersectionCountArrayArray(a, b *Container) (n int) { return n } -func intersectionCountArrayRun(a, b *Container) (n int) { +func intersectionCountArrayRun(a, b *Container) (n int32) { na, nb := len(a.array), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.array[i], b.runs[j] @@ -1898,7 +1898,7 @@ func intersectionCountArrayRun(a, b *Container) (n int) { return n } -func intersectionCountRunRun(a, b *Container) (n int) { +func intersectionCountRunRun(a, b *Container) (n int32) { na, nb := len(a.runs), len(b.runs) for i, j := 0, 0; i < na && j < nb; { va, vb := a.runs[i], b.runs[j] @@ -1910,33 +1910,33 @@ func intersectionCountRunRun(a, b *Container) (n int) { j++ } else if va.last > vb.last && va.start >= vb.start { // |--vb-|-|-va--| - n += 1 + int(vb.last-va.start) + n += 1 + int32(vb.last-va.start) j++ } else if va.last > vb.last && va.start < vb.start { // |--va|--vb--|--| - n += 1 + int(vb.last-vb.start) + n += 1 + int32(vb.last-vb.start) j++ } else if va.last <= vb.last && va.start >= vb.start { // |--vb|--va--|--| - n += 1 + int(va.last-va.start) + n += 1 + int32(va.last-va.start) i++ } else if va.last <= vb.last && va.start < vb.start { // |--va-|-|-vb--| - n += 1 + int(va.last-vb.start) + n += 1 + int32(va.last-vb.start) i++ } } return n } -func intersectionCountBitmapRun(a, b *Container) (n int) { +func intersectionCountBitmapRun(a, b *Container) (n int32) { for _, iv := range b.runs { - n += a.bitmapCountRange(int(iv.start), int(iv.last)+1) + n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1) } return n } -func intersectionCountArrayBitmap(a, b *Container) (n int) { +func intersectionCountArrayBitmap(a, b *Container) (n int32) { ln := len(b.bitmap) for _, val := range a.array { i := int(val >> 6) @@ -1944,13 +1944,13 @@ func intersectionCountArrayBitmap(a, b *Container) (n int) { break } off := val % 64 - n += int(b.bitmap[i]>>off) & 1 + n += int32(b.bitmap[i]>>off) & 1 } return n } -func intersectionCountBitmapBitmap(a, b *Container) (n int) { - return int(popcountAndSlice(a.bitmap, b.bitmap)) +func intersectionCountBitmapBitmap(a, b *Container) (n int32) { + return int32(popcountAndSlice(a.bitmap, b.bitmap)) } func intersect(a, b *Container) *Container { @@ -1995,7 +1995,7 @@ func intersectArrayArray(a, b *Container) *Container { i, j = i+1, j+1 } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2016,7 +2016,7 @@ func intersectArrayRun(a, b *Container) *Container { i++ } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2050,7 +2050,7 @@ func intersectRunRun(a, b *Container) *Container { i++ } } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -2076,7 +2076,7 @@ func intersectBitmapRun(a, b *Container) *Container { } } } - output.n = len(output.array) + output.n = int32(len(output.array)) } else { // right now this iterates through the runs and sets integers in the // bitmap that are in the runs. alternately, we could zero out ranges in @@ -2093,22 +2093,22 @@ func intersectBitmapRun(a, b *Container) *Container { for valast >= vb.start && vastart <= vb.last && i < bitmapN { if vastart >= vb.start && valast <= vb.last { // a within b output.bitmap[i] = a.bitmap[i] - output.n += int(popcount(a.bitmap[i])) + output.n += int32(popcount(a.bitmap[i])) } else if vb.start >= vastart && vb.last <= valast { // b within a var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart) bits := a.bitmap[i] & mask output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } else if vastart < vb.start { // a overlaps front of b offset := 64 - (1 + valast - vb.start) bits := (a.bitmap[i] >> offset) << offset output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } else if vb.start < vastart { // b overlaps front of a offset := 64 - (1 + vb.last - vastart) bits := (a.bitmap[i] << offset) >> offset output.bitmap[i] |= bits - output.n += int(popcount(bits)) + output.n += int32(popcount(bits)) } // update loop vars i++ @@ -2134,7 +2134,7 @@ func intersectArrayBitmap(a, b *Container) *Container { output.array = append(output.array, va) } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2146,11 +2146,11 @@ func intersectBitmapBitmap(a, b *Container) *Container { bb = b.bitmap[:bitmapN] buf = make([]uint64, bitmapN) ob = buf[:bitmapN] - n int + n int32 ) for i := 0; i < bitmapN; i++ { ob[i] = ab[i] & bb[i] - n += int(popcount(ob[i])) + n += int32(popcount(ob[i])) } output := &Container{ @@ -2259,10 +2259,10 @@ func unionArrayRun(a, b *Container) *Container { // interval is earlier than the start of the last interval in the list of runs. // Its return value is the amount by which the cardinality of the container was // increased. -func (c *Container) runAppendInterval(v interval16) int { +func (c *Container) runAppendInterval(v interval16) int32 { if len(c.runs) == 0 { c.runs = append(c.runs, v) - return int(v.last-v.start) + 1 + return int32(v.last-v.start) + 1 } last := c.runs[len(c.runs)-1] @@ -2271,10 +2271,10 @@ func (c *Container) runAppendInterval(v interval16) int { } if last.last+1 >= v.start && v.last > last.last { c.runs[len(c.runs)-1].last = v.last - return int(v.last - last.last) + return int32(v.last - last.last) } else if last.last+1 < v.start { c.runs = append(c.runs, v) - return int(v.last-v.start) + 1 + return int32(v.last-v.start) + 1 } return 0 } @@ -2338,16 +2338,16 @@ func (c *Container) bitmapSetRange(i, j uint64) { xcnt := popcount(X) ycnt := popcount(Y) if x == y { - c.n += int((j - i) - popcount(c.bitmap[x]&(X&Y))) + c.n += int32((j - i) - popcount(c.bitmap[x]&(X&Y))) c.bitmap[x] |= (X & Y) } else { - c.n += int(xcnt - popcount(c.bitmap[x]&X)) + c.n += int32(xcnt - popcount(c.bitmap[x]&X)) c.bitmap[x] |= X for i := x + 1; i < y; i++ { - c.n += int(64 - popcount(c.bitmap[i])) + c.n += int32(64 - popcount(c.bitmap[i])) c.bitmap[i] = maxBitmap } - c.n += int(ycnt - popcount(c.bitmap[y]&Y)) + c.n += int32(ycnt - popcount(c.bitmap[y]&Y)) c.bitmap[y] |= Y } } @@ -2361,19 +2361,19 @@ func (c *Container) bitmapXorRange(i, j uint64) { if x == y { cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= (X & Y) //// flip - c.n += int(popcount(c.bitmap[x]) - cnt) + c.n += int32(popcount(c.bitmap[x]) - cnt) } else { cnt := popcount(c.bitmap[x]) c.bitmap[x] ^= X - c.n += int(popcount(c.bitmap[x]) - cnt) + c.n += int32(popcount(c.bitmap[x]) - cnt) for i := x + 1; i < y; i++ { cnt = popcount(c.bitmap[i]) c.bitmap[i] ^= maxBitmap - c.n += int(popcount(c.bitmap[i]) - cnt) + c.n += int32(popcount(c.bitmap[i]) - cnt) } cnt = popcount(c.bitmap[y]) c.bitmap[y] ^= Y - c.n += int(popcount(c.bitmap[y]) - cnt) + c.n += int32(popcount(c.bitmap[y]) - cnt) } } @@ -2384,16 +2384,16 @@ func (c *Container) bitmapZeroRange(i, j uint64) { var X uint64 = maxBitmap << (i % 64) var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) if x == y { - c.n -= int(popcount(c.bitmap[x] & (X & Y))) + c.n -= int32(popcount(c.bitmap[x] & (X & Y))) c.bitmap[x] &= ^(X & Y) } else { - c.n -= int(popcount(c.bitmap[x] & X)) + c.n -= int32(popcount(c.bitmap[x] & X)) c.bitmap[x] &= ^X for i := x + 1; i < y; i++ { - c.n -= int(popcount(c.bitmap[i])) + c.n -= int32(popcount(c.bitmap[i])) c.bitmap[i] = 0 } - c.n -= int(popcount(c.bitmap[y] & Y)) + c.n -= int32(popcount(c.bitmap[y] & Y)) c.bitmap[y] &= ^Y } } @@ -2456,12 +2456,12 @@ func unionBitmapBitmap(a, b *Container) *Container { buf = make([]uint64, bitmapN) ob = buf[:bitmapN] - n int + n int32 ) for i := 0; i < bitmapN; i++ { ob[i] = ab[i] | bb[i] - n += int(popcount(ob[i])) + n += int32(popcount(ob[i])) } output := &Container{ @@ -2540,7 +2540,7 @@ func differenceArrayRun(a, b *Container) *Container { j := 0 // run index // handle overlap - for i < a.n { + for i < int(a.n) { // keep all array elements before beginning of runs if a.array[i] < b.runs[j].start { @@ -2572,7 +2572,7 @@ func differenceArrayRun(a, b *Container) *Container { output.array = append(output.array, a.array[i:]...) // TODO: consider handling container.n mutations in one place // like we do with container.add(). - output.n += len(a.array[i:]) + output.n += int32(len(a.array[i:])) } else { for _, v := range a.array[i:] { output.add(v) @@ -2630,7 +2630,7 @@ RUNLOOP: continue } output.runs = append(output.runs, interval16{start: start, last: vb - 1}) - output.n += int(vb - start) + output.n += int32(vb - start) if vb == 65535 { // overflow break RUNLOOP } @@ -2644,7 +2644,7 @@ RUNLOOP: if start <= run.last { output.runs = append(output.runs, interval16{start: start, last: run.last}) - output.n += int(run.last - start + 1) + output.n += int32(run.last - start + 1) } } output.optimize() @@ -2700,7 +2700,7 @@ func differenceRunBitmap(a, b *Container) *Container { } } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -2784,7 +2784,7 @@ func differenceArrayBitmap(a, b *Container) *Container { output.array = append(output.array, va) } } - output.n = len(output.array) + output.n = int32(len(output.array)) return output } @@ -2813,12 +2813,12 @@ func differenceBitmapBitmap(a, b *Container) *Container { buf = make([]uint64, bitmapN) ob = buf[:bitmapN] - n int + n int32 ) for i := 0; i < bitmapN; i++ { ob[i] = ab[i] & (^bb[i]) - n += int(popcount(ob[i])) + n += int32(popcount(ob[i])) } output := &Container{ @@ -2918,12 +2918,12 @@ func xorBitmapBitmap(a, b *Container) *Container { buf = make([]uint64, bitmapN) ob = buf[:bitmapN] - n int + n int32 ) for i := 0; i < bitmapN; i++ { ob[i] = ab[i] ^ bb[i] - n += int(popcount(ob[i])) + n += int32(popcount(ob[i])) } output := &Container{ @@ -3009,9 +3009,9 @@ func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } // search32 returns the index of value in a. If value is not found, it works the // same way as search64. -func search32(a []uint16, value uint16) int { +func search32(a []uint16, value uint16) int32 { // Optimize for elements and the last element. - n := len(a) + n := int32(len(a)) if n == 0 { return -1 } else if a[n-1] == value { @@ -3019,9 +3019,9 @@ func search32(a []uint16, value uint16) int { } // Otherwise perform binary search for exact match. - lo, hi := 0, n-1 + lo, hi := int32(0), n-1 for lo+16 <= hi { - i := int(uint((lo + hi)) >> 1) + i := int32(uint((lo + hi)) >> 1) v := a[i] if v < value { @@ -3327,7 +3327,7 @@ func xorRunRun(a, b *Container) *Container { } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -3342,7 +3342,7 @@ func xorBitmapRun(a, b *Container) *Container { output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } - if output.n < ArrayMaxSize && len(output.runs) > output.n/2 { + if output.n < ArrayMaxSize && int32(len(output.runs)) > output.n/2 { output.runToArray() } else if len(output.runs) > runMaxSize { output.runToBitmap() @@ -3481,7 +3481,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { b.Containers.PutContainerValues( uint64(binary.LittleEndian.Uint16(buf[0:2])), containerTyper(i, card), /// container type voodo with isRunBitmap - card, + int32(card), true) } @@ -3546,7 +3546,7 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { c.runs = nil c.bitmap = nil c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[pos]))[:c.n] - pos += c.n * 2 + pos += int(c.n * 2) case containerBitmap: c.array = nil c.runs = nil diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index c77ef4ee1..a4c162629 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -39,7 +39,7 @@ func TestRunAppendInterval(t *testing.T) { tests := []struct { base []interval16 app interval16 - exp int + exp int32 }{ { base: []interval16{}, @@ -207,10 +207,10 @@ func TestRunContains(t *testing.T) { func TestBitmapCountRange(t *testing.T) { c := Container{containerType: containerBitmap} tests := []struct { - start int - end int + start int32 + end int32 bitmap []uint64 - exp int + exp int32 }{ {start: 0, end: 1, bitmap: []uint64{1}, exp: 1}, {start: 2, end: 7, bitmap: []uint64{0xFFFFFFFFFFFFFF18}, exp: 2}, @@ -251,7 +251,7 @@ func TestIntersectionCountArrayBitmap3(t *testing.T) { b.bitmapToRun() res = intersectRunRun(a, b) n := intersectionCountRunRun(a, b) - if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) { + if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int32(n) { t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1) } } @@ -261,7 +261,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) { tests := []struct { array []uint16 bitmap []uint64 - exp int + exp int32 }{ { array: []uint16{0}, @@ -384,7 +384,7 @@ func TestIntersectionCountRunRun(t *testing.T) { tests := []struct { aruns []interval16 bruns []interval16 - exp int + exp int32 }{ { aruns: []interval16{}, @@ -478,7 +478,7 @@ func TestIntersectRunRun(t *testing.T) { aruns []interval16 bruns []interval16 exp []interval16 - expN int + expN int32 }{ { aruns: []interval16{}, @@ -540,7 +540,7 @@ func TestIntersectBitmapRunBitmap(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{1}, @@ -602,7 +602,7 @@ func TestIntersectBitmapRunArray(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint16 - expN int + expN int32 }{ { bitmap: []uint64{1}, @@ -761,11 +761,11 @@ func TestDifferenceMixed(t *testing.T) { a.containerType = containerRun b.array = []uint16{0, 2, 4, 6, 8, 10, 12} - b.n = len(b.array) + b.n = int32(len(b.array)) b.containerType = containerArray d.array = []uint16{1, 3, 5, 7, 9, 11, 12} - d.n = len(d.array) + d.n = int32(len(d.array)) d.containerType = containerArray res := difference(a, b) @@ -939,7 +939,7 @@ func TestBitmapSetRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000FFF900}, @@ -991,7 +991,7 @@ func TestArrayToBitmap(t *testing.T) { copy(exp, test.exp) a.array = test.array - a.n = len(test.array) + a.n = int32(len(test.array)) a.arrayToBitmap() if !reflect.DeepEqual(a.bitmap, exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) @@ -1016,10 +1016,10 @@ func TestBitmapToArray(t *testing.T) { } for i, test := range tests { a.bitmap = make([]uint64, bitmapN) - n := 0 + n := int32(0) for i, v := range test.bitmap { a.bitmap[i] = v - n += int(popcount(v)) + n += int32(popcount(v)) } a.n = n @@ -1067,7 +1067,7 @@ func TestRunToBitmap(t *testing.T) { } a.runs = test.runs - a.n = n + a.n = int32(n) a.runToBitmap() if !reflect.DeepEqual(a.bitmap, exp) { t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap) @@ -1149,7 +1149,7 @@ func TestBitmapToRun(t *testing.T) { a.bitmap[i] = v n += int(popcount(v)) } - a.n = n + a.n = int32(n) x := a.bitmap a.bitmapToRun() if !reflect.DeepEqual(a.runs, test.exp) { @@ -1188,7 +1188,7 @@ func TestArrayToRun(t *testing.T) { for i, test := range tests { a.array = test.array - a.n = int(len(test.array)) + a.n = int32(len(test.array)) a.arrayToRun() if !reflect.DeepEqual(a.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs) @@ -1222,7 +1222,7 @@ func TestRunToArray(t *testing.T) { for i, test := range tests { a.runs = test.runs - a.n = len(test.exp) + a.n = int32(len(test.exp)) a.runToArray() if !reflect.DeepEqual(a.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array) @@ -1237,7 +1237,7 @@ func TestBitmapZeroRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000FFFF00}, @@ -1279,7 +1279,7 @@ func TestUnionBitmapRun(t *testing.T) { bitmap []uint64 runs []interval16 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{2}, @@ -1313,7 +1313,7 @@ func TestBitmapCountRuns(t *testing.T) { c := &Container{containerType: containerBitmap, bitmap: make([]uint64, bitmapN)} tests := []struct { bitmap []uint64 - exp int + exp int32 }{ { bitmap: []uint64{0xFF00FF00}, @@ -1361,7 +1361,7 @@ func TestArrayCountRuns(t *testing.T) { c := &Container{containerType: containerArray} tests := []struct { array []uint16 - exp int + exp int32 }{ { array: []uint16{}, @@ -1414,7 +1414,7 @@ func TestDifferenceArrayRun(t *testing.T) { } for i, test := range tests { a.array = test.array - a.n = len(a.array) + a.n = int32(len(a.array)) b.runs = test.runs b.n = b.runCountRange(0, 100) ret := differenceArrayRun(a, b) @@ -1482,7 +1482,7 @@ func TestDifferenceRunArray(t *testing.T) { a.runs = test.runs a.n = a.runCountRange(0, 100) b.array = test.array - b.n = len(b.array) + b.n = int32(len(b.array)) ret := differenceRunArray(a, b) if !reflect.DeepEqual(ret.runs, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs) @@ -1730,7 +1730,7 @@ func TestDifferenceRunRun(t *testing.T) { aruns []interval16 bruns []interval16 exp []interval16 - expn int + expn int32 }{ { // this tests all six overlap combinations @@ -2009,7 +2009,7 @@ func TestBitmapXorRange(t *testing.T) { start uint64 last uint64 exp []uint64 - expN int + expN int32 }{ { bitmap: []uint64{0x0000000000000000}, @@ -2302,7 +2302,7 @@ func TestRunBinSearchContains(t *testing.T) { runs []interval16 index uint16 exp struct { - index int + index int32 found bool } }{ @@ -2310,7 +2310,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}}, index: uint16(3), exp: struct { - index int + index int32 found bool }{index: 0, found: true}, }, @@ -2318,7 +2318,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}}, index: uint16(13), exp: struct { - index int + index int32 found bool }{index: 0, found: false}, }, @@ -2326,7 +2326,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, index: uint16(13), exp: struct { - index int + index int32 found bool }{index: 0, found: false}, }, @@ -2334,7 +2334,7 @@ func TestRunBinSearchContains(t *testing.T) { runs: []interval16{{start: 0, last: 10}, {start: 20, last: 30}}, index: uint16(36), exp: struct { - index int + index int32 found bool }{index: 1, found: false}, }, @@ -2355,7 +2355,7 @@ func TestRunBinSearch(t *testing.T) { runs []interval16 search uint16 exp bool - expi int + expi int32 }{ { runs: []interval16{{2, 10}, {50, 60}, {80, 90}}, From be3bc105cdffe5b90d92ad6579cd6b0f7540caf2 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 25 Sep 2018 10:40:04 -0500 Subject: [PATCH 25/45] fixed upconvert;reverted to released interface --- enterprise/b/containers_btree.go | 4 ++-- roaring/containers.go | 6 +++--- roaring/roaring.go | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index 789936a24..7487eea70 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -93,8 +93,8 @@ type updater struct { mapped bool } -func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int32, mapped bool) { - a := updater{key, n, containerType, mapped} +func (btc *bTreeContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { + a := updater{key, int32(n), containerType, mapped} btc.tree.Put(key, a.update) } diff --git a/roaring/containers.go b/roaring/containers.go index bc458fba2..ed745a915 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -46,18 +46,18 @@ func (sc *sliceContainers) Put(key uint64, c *Container) { } -func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int32, mapped bool) { +func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) { i := search64(sc.keys, key) if i < 0 { c := NewContainer() c.containerType = containerType - c.n = n + c.n = int32(n) c.mapped = mapped sc.insertAt(key, c, -i-1) } else { c := sc.containers[i] c.containerType = containerType - c.n = n + c.n = int32(n) c.mapped = mapped } diff --git a/roaring/roaring.go b/roaring/roaring.go index fd9f15d92..8ad4c783d 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -73,7 +73,7 @@ type Containers interface { // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. - PutContainerValues(key uint64, containerType byte, n int32, mapped bool) + PutContainerValues(key uint64, containerType byte, n int, mapped bool) // Remove takes the container at key out. Remove(key uint64) @@ -644,7 +644,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { b.Containers.PutContainerValues( binary.LittleEndian.Uint64(buf[0:8]), byte(binary.LittleEndian.Uint16(buf[8:10])), - int32(binary.LittleEndian.Uint16(buf[10:12]))+1, + int(binary.LittleEndian.Uint16(buf[10:12]))+1, true) } opsOffset := headerSize + int(keyN)*12 @@ -1103,7 +1103,7 @@ func (c *Container) countRange(start, end int32) (n int32) { } func (c *Container) arrayCountRange(start, end int32) (n int32) { - i := int32(sort.Search(int(len(c.array)), func(i int) bool { return int32(c.array[i]) >= start })) + i := int32(sort.Search(len(c.array), func(i int) bool { return int32(c.array[i]) >= start })) for ; i < int32(len(c.array)); i++ { v := int32(c.array[i]) if v >= end { @@ -3481,7 +3481,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { b.Containers.PutContainerValues( uint64(binary.LittleEndian.Uint16(buf[0:2])), containerTyper(i, card), /// container type voodo with isRunBitmap - int32(card), + card, true) } From 6aad568f28aecf794caa2916a299d9f8bf9d0c8c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 25 Sep 2018 11:07:01 -0500 Subject: [PATCH 26/45] added TODO(2.0) comment --- roaring/roaring.go | 1 + 1 file changed, 1 insertion(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8ad4c783d..ed7be6be1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -73,6 +73,7 @@ type Containers interface { // PutContainerValues updates an existing container at key. // If a container does not exist for key, a new one is allocated. + // TODO(2.0) make n int32 PutContainerValues(key uint64, containerType byte, n int, mapped bool) // Remove takes the container at key out. From 2521922d7b2da1e305d7183b266b5f460d7533e6 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Sep 2018 11:35:58 -0500 Subject: [PATCH 27/45] Properly wrap translation store errors, decrease test map size to prevent failure on 32-bit --- server.go | 8 ++++---- test/pilosa.go | 6 +----- translate.go | 8 ++++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/server.go b/server.go index 6cd313dc0..61e4838a1 100644 --- a/server.go +++ b/server.go @@ -337,20 +337,20 @@ func (s *Server) Open() error { // Initialize id-key storage. if err := s.holder.translateFile.Open(); err != nil { - return err + return errors.Wrap(err, "opening TranslateFile") } // Open Cluster management. if err := s.cluster.waitForStarted(); err != nil { - return fmt.Errorf("opening Cluster: %v", err) + return errors.Wrap(err, "opening Cluster") } // Open holder. if err := s.holder.Open(); err != nil { - return fmt.Errorf("opening Holder: %v", err) + return errors.Wrap(err, "opening Holder") } if err := s.cluster.setNodeState(nodeStateReady); err != nil { - return fmt.Errorf("setting nodeState: %v", err) + return errors.Wrap(err, "setting nodeState") } // Listen for joining nodes. diff --git a/test/pilosa.go b/test/pilosa.go index f8f60eebe..27331d53f 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -63,17 +63,13 @@ func newCommand(opts ...server.CommandOption) *Command { // does not fail on 32-bit systems. opts = append([]server.CommandOption{ server.OptCommandCloseTimeout(time.Millisecond * 2), - server.OptCommandServerOptions( - pilosa.OptServerTranslateFileMapSize( - 2 << 25, - ), - ), }, opts...) m := &Command{commandOptions: opts} m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true + m.Config.Translation.MapSize = 100000 if testing.Verbose() { m.Command.Stdout = os.Stdout diff --git a/translate.go b/translate.go index 2ef289f45..bdf13d82a 100644 --- a/translate.go +++ b/translate.go @@ -131,20 +131,20 @@ func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { func (s *TranslateFile) Open() (err error) { // Open writer & buffered writer. if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { - return err + return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { - return err + return errors.Wrapf(err, "open file %s", s.Path) } s.w = bufio.NewWriter(s.file) // Memory map data file. if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { - return err + return errors.Wrapf(err, "creating Mmap (size: %d)", s.mapSize) } // Replay the log. if err := s.replayEntries(); err != nil { - return err + return errors.Wrap(err, "replay log") } // Listen to primaryStoreEvents channel. From 81dbe38081d871f366ac5165774e87c79b6bb66a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Sep 2018 11:59:49 -0500 Subject: [PATCH 28/45] Change wording for consistency --- translate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translate.go b/translate.go index bdf13d82a..cb16de929 100644 --- a/translate.go +++ b/translate.go @@ -144,7 +144,7 @@ func (s *TranslateFile) Open() (err error) { // Replay the log. if err := s.replayEntries(); err != nil { - return errors.Wrap(err, "replay log") + return errors.Wrap(err, "replaying log entries") } // Listen to primaryStoreEvents channel. From 066c3a48531a538655ae74489f4a6c7e91b67afd Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 1 Oct 2018 14:15:51 +0300 Subject: [PATCH 29/45] Fixes #1660 --- api.go | 14 ++++++++++---- http/handler.go | 7 ++++++- server/handler_test.go | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 44ea0a7b0..f2ba86279 100644 --- a/api.go +++ b/api.go @@ -321,13 +321,19 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, nodes := api.cluster.shardNodes(indexName, shard) var eg errgroup.Group + field := api.holder.Field(indexName, fieldName) + if field == nil { + return newNotFoundError(ErrFieldNotFound) + } + + // only set fields are supported + if field.Type() != "set" { + return NewBadRequestError(errors.New("roaring import is only supported for set fields")) + } + for _, node := range nodes { node := node if node.ID == api.server.nodeID { - field := api.holder.Field(indexName, fieldName) - if field == nil { - return newNotFoundError(ErrFieldNotFound) - } // must make a copy of data to operate on locally. field.importRoaring changes data d2 := make([]byte, len(data)) copy(d2, data) diff --git a/http/handler.go b/http/handler.go index 820bc6521..4bbaf3a69 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1473,11 +1473,16 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } + resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body) - resp := &pilosa.ImportResponse{} if err != nil { resp.Err = err.Error() + if _, ok := err.(pilosa.BadRequestError); ok { + w.WriteHeader(http.StatusBadRequest) + } else { + w.WriteHeader(http.StatusInternalServerError) + } } // Marshal response object. buf, err := h.api.Serializer.Marshal(resp) diff --git a/server/handler_test.go b/server/handler_test.go index 2c1aa9868..9e76d77b9 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -104,6 +104,22 @@ func TestHandler_Endpoints(t *testing.T) { }) + t.Run("ImportRoaringFieldTypeFail", func(t *testing.T) { + // Roaring import into a non-set field should fail. + if _, err := i0.CreateFieldIfNotExists("int-field", pilosa.OptFieldTypeInt(0, 1)); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") + req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData)) + req.Header.Set("Content-Type", "application/x-binary") + h.ServeHTTP(w, req) + if w.Code != gohttp.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } + + }) + t.Run("Status", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) From 2830fb46e603823628b8fd404dbde7895979cdd1 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 1 Oct 2018 15:31:13 +0300 Subject: [PATCH 30/45] Adds diagnostics CPUArch field --- diagnostics.go | 11 +++++++++++ gopsutil/systeminfo.go | 7 +++++++ gopsutil/systeminfo_test.go | 16 +++++----------- server.go | 1 + 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index cad7725d7..673fb0c7f 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -174,6 +174,11 @@ func (d *diagnosticsCollector) logErr(err error) bool { return false } +// EnrichWithCPUInfo adds CPU information to the diagnostics payload. +func (d *diagnosticsCollector) EnrichWithCPUInfo() { + d.Set("CPUArch", d.server.systemInfo.CPUArch()) +} + // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *diagnosticsCollector) EnrichWithOSInfo() { uptime, err := d.server.systemInfo.Uptime() @@ -265,6 +270,7 @@ type SystemInfo interface { MemFree() (uint64, error) MemTotal() (uint64, error) MemUsed() (uint64, error) + CPUArch() string } // newNopSystemInfo creates a no-op implementation of SystemInfo. @@ -315,3 +321,8 @@ func (n *nopSystemInfo) MemTotal() (uint64, error) { func (n *nopSystemInfo) MemUsed() (uint64, error) { return 0, nil } + +// CPUArch returns the CPU architecture, such as amd64 +func (n *nopSystemInfo) CPUArch() string { + return "" +} diff --git a/gopsutil/systeminfo.go b/gopsutil/systeminfo.go index 8cdd77778..290e1278c 100644 --- a/gopsutil/systeminfo.go +++ b/gopsutil/systeminfo.go @@ -15,6 +15,8 @@ package gopsutil import ( + "runtime" + "github.com/pilosa/pilosa" "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/mem" @@ -109,6 +111,11 @@ func (s *systemInfo) KernelVersion() (string, error) { return host.KernelVersion() } +// CPUArch returns the CPU architecture, such as amd64 +func (s *systemInfo) CPUArch() string { + return runtime.GOARCH +} + // NewSystemInfo is a constructor for the gopsutil implementation of SystemInfo. func NewSystemInfo() *systemInfo { return &systemInfo{} diff --git a/gopsutil/systeminfo_test.go b/gopsutil/systeminfo_test.go index 0f76b62da..28d09e66a 100644 --- a/gopsutil/systeminfo_test.go +++ b/gopsutil/systeminfo_test.go @@ -15,7 +15,6 @@ package gopsutil_test import ( - "log" "testing" "github.com/pilosa/pilosa" @@ -25,15 +24,6 @@ import ( func TestSystemInfo(t *testing.T) { var systemInfo pilosa.SystemInfo = gopsutil.NewSystemInfo() - // Uptime()(uint64, error) - // Platform()(string, error) - // Family()(string, error) - // OSVersion()(string, error) - // KernelVersion()(string, error) - // MemFree()(uint64, error) - // MemTotal()(uint64, error) - // MemUsed()(uint64, error) - // uptime, err := systemInfo.Uptime() if err != nil || uptime == 0 { t.Fatalf("Error collecting uptime (error: %v)", err) @@ -70,8 +60,12 @@ func TestSystemInfo(t *testing.T) { } memtotal, err := systemInfo.MemTotal() - log.Println(memtotal) if err != nil { t.Fatalf("Error getting memtotal. (memtotal: %v, error: %v)", memtotal, err) } + + cpuArch := systemInfo.CPUArch() + if cpuArch == "" { + t.Fatalf("Error getting CPU arch.") + } } diff --git a/server.go b/server.go index 61e4838a1..3bb3ce1b2 100644 --- a/server.go +++ b/server.go @@ -675,6 +675,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.nodeID) s.diagnostics.Set("ClusterID", s.cluster.id) + s.diagnostics.EnrichWithCPUInfo() s.diagnostics.EnrichWithOSInfo() // Flush the diagnostics metrics at startup, then on each tick interval From f7f6ff743ad8d297fa3b207e72e153b0833a6a82 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 1 Oct 2018 17:45:21 +0300 Subject: [PATCH 31/45] index trackExistence is true by default --- http/handler.go | 14 ++++++++++++-- http/handler_internal_test.go | 7 ++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/http/handler.go b/http/handler.go index 820bc6521..c5e8d5b18 100644 --- a/http/handler.go +++ b/http/handler.go @@ -521,7 +521,12 @@ func (p *postIndexRequest) UnmarshalJSON(b []byte) error { return err } // Unmarshal expected values. - var _p _postIndexRequest + _p := _postIndexRequest{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }, + } if err := json.Unmarshal(b, &_p); err != nil { return errors.Wrap(err, "unmarshalling expected values") } @@ -597,7 +602,12 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp := successResponse{} // Decode request. - var req postIndexRequest + req := postIndexRequest{ + Options: pilosa.IndexOptions{ + Keys: false, + TrackExistence: true, + }, + } err := json.NewDecoder(r.Body).Decode(&req) if err != nil && err != io.EOF { resp.write(w, err) diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index b0852a79d..7ca9c0401 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -31,8 +31,9 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, - {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: true}}}, + {json: `{"options": {"trackExistence": false}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{TrackExistence: false}}}, + {json: `{"options": {"keys": true}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{Keys: true, TrackExistence: true}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, @@ -53,7 +54,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { if test.err == "" { if !reflect.DeepEqual(*actual, test.expected) { - t.Errorf("expected: %v, but got: %v", test.expected, *actual) + t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json) } } From fc6fc82f6774bb5ba83da64d5909606d6c87e7b3 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 1 Oct 2018 17:55:56 +0300 Subject: [PATCH 32/45] use FieldTypeSet --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index f2ba86279..910bc582f 100644 --- a/api.go +++ b/api.go @@ -327,7 +327,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } // only set fields are supported - if field.Type() != "set" { + if field.Type() != FieldTypeSet { return NewBadRequestError(errors.New("roaring import is only supported for set fields")) } From 040cce162bc6ac18b073c2efc00c6c40dc7db8ce Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 1 Oct 2018 10:45:41 -0500 Subject: [PATCH 33/45] Add CircleCI step to generate Docker image and push to Docker hub --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b42b2aa64..bd1ca86cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,6 +72,16 @@ jobs: - *fast-checkout - run: sudo pip install awscli - run: make prerelease-upload + dockerhub-upload: + <<: *defaults + steps: + - run: '[[ -v CIRCLE_PR_NUMBER ]] && circleci step halt || true' # Skip job if this is a PR + - *fast-checkout + - setup_remote_docker + - run: make docker + - run: docker tag pilosa:$(git describe --tags) pilosa/pilosa:master + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker push pilosa/pilosa:master workflows: version: 2 test: @@ -105,3 +115,7 @@ workflows: - prerelease-upload: requires: - prerelease + - dockerhub-upload: + requires: + - linter + - test-golang-1.10 From 3539b54835ebfc74f92a02a0fce46077ed67903b Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 2 Oct 2018 13:58:01 +0300 Subject: [PATCH 34/45] Synced query validation for handlers --- http/handler.go | 77 +++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/http/handler.go b/http/handler.go index 820bc6521..47f5cf374 100644 --- a/http/handler.go +++ b/http/handler.go @@ -170,13 +170,34 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") - h.validators["GetShardMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["Home"] = queryValidationSpecRequired() + h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired() + h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired() + h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired() h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "shard") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "shard") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "shard") + h.validators["GetIndexes"] = queryValidationSpecRequired() + h.validators["GetIndex"] = queryValidationSpecRequired() + h.validators["PostIndex"] = queryValidationSpecRequired() + h.validators["DeleteIndex"] = queryValidationSpecRequired() + h.validators["PostField"] = queryValidationSpecRequired() + h.validators["DeleteField"] = queryValidationSpecRequired() + h.validators["PostImport"] = queryValidationSpecRequired() + h.validators["PostImportRoaring"] = queryValidationSpecRequired().Optional("remote") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("shards", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetInfo"] = queryValidationSpecRequired() + h.validators["RecalculateCaches"] = queryValidationSpecRequired() + h.validators["GetSchema"] = queryValidationSpecRequired() + h.validators["GetStatus"] = queryValidationSpecRequired() + h.validators["GetVersion"] = queryValidationSpecRequired() + h.validators["PostClusterMessage"] = queryValidationSpecRequired() + h.validators["GetFragmentBlockData"] = queryValidationSpecRequired() h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") + h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired() + h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired() + h.validators["GetNodes"] = queryValidationSpecRequired() + h.validators["GetShardMax"] = queryValidationSpecRequired() + h.validators["GetTranslateData"] = queryValidationSpecRequired("offset") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -203,40 +224,40 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET") - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home") + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET").Name("GetIndexes") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET").Name("GetIndex") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST").Name("PostIndex") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE").Name("DeleteIndex") //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") - router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE").Name("DeleteField") + router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST").Name("PostImport") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostImportRoaring).Methods("POST").Name("PostImportRoaring") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET").Name("GetInfo") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST").Name("RecalculateCaches") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") // /internal endpoints are for internal use only; they may change at any time. // DO NOT rely on these for external applications! - router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage") + router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData") router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff") + router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") - router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET") + router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: From efed5ea3fce3b20f795d2dbdf9c649b2d7aed175 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 3 Oct 2018 13:43:15 +0300 Subject: [PATCH 35/45] Fixes #1632 --- api.go | 64 +++------------------------------------------ executor.go | 75 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 69 deletions(-) diff --git a/api.go b/api.go index 44ea0a7b0..d910fbbe8 100644 --- a/api.go +++ b/api.go @@ -102,11 +102,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return QueryResponse{}, errors.Wrap(err, "validating api method") } - resp := QueryResponse{} - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - return resp, errors.Wrap(err, "parsing") + return QueryResponse{}, errors.Wrap(err, "parsing") } execOpts := &execOptions{ Remote: req.Remote, @@ -114,70 +112,14 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. } - results, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) + resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { - return resp, errors.Wrap(err, "executing") + return QueryResponse{}, errors.Wrap(err, "executing") } - resp.Results = results - // Fill column attributes if requested. - // execOpts.ColumnAttrs may be set by the Execute method if any of the Calls use Options(columnAttrs=true) - if execOpts.ColumnAttrs { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Row) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := api.readColumnAttrSets(api.holder.Index(req.Index), columnIDs) - if err != nil { - return resp, errors.Wrap(err, "reading column attrs") - } - - // Translate column attributes, if necessary. - if api.holder.translateFile != nil { - for _, col := range resp.ColumnAttrSets { - v, err := api.holder.translateFile.TranslateColumnToString(req.Index, col.ID) - if err != nil { - return resp, err - } - col.Key, col.ID = v, 0 - } - } - - resp.ColumnAttrSets = columnAttrSets - } return resp, nil } -// readColumnAttrSets returns a list of column attribute objects by id. -func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { - if index == nil { - return nil, nil - } - - ax := make([]*ColumnAttrSet, 0, len(ids)) - for _, id := range ids { - // Read attributes for column. Skip column if empty. - attrs, err := index.ColumnAttrStore().Attrs(id) - if err != nil { - return nil, errors.Wrap(err, "getting attrs") - } else if len(attrs) == 0 { - continue - } - - // Append column with attributes. - ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) - } - - return ax, nil -} - // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(_ context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { diff --git a/executor.go b/executor.go index 8dd4db569..29da40b27 100644 --- a/executor.go +++ b/executor.go @@ -79,20 +79,21 @@ func newExecutor(opts ...executorOption) *executor { } // Execute executes a PQL query. -func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { +func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { + resp := QueryResponse{} // Verify that an index is set. if index == "" { - return nil, ErrIndexRequired + return resp, ErrIndexRequired } idx := e.Holder.Index(index) if idx == nil { - return nil, ErrIndexNotFound + return resp, ErrIndexNotFound } // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { - return nil, ErrTooManyWrites + return resp, ErrTooManyWrites } // Default options. @@ -105,27 +106,85 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { for i := range q.Calls { if err := e.translateCall(index, idx, q.Calls[i]); err != nil { - return nil, err + return resp, err } } } results, err := e.execute(ctx, index, q, shards, opt) if err != nil { - return nil, err + return resp, err } + resp.Results = results + // Translate response objects from ids to keys, if necessary. // No need to translate a remote call. if !opt.Remote { for i := range results { results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) if err != nil { - return nil, err + return resp, err } } } - return results, nil + + // Fill column attributes if requested. + if opt.ColumnAttrs { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Row) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := e.readColumnAttrSets(e.Holder.Index(index), columnIDs) + if err != nil { + return resp, errors.Wrap(err, "reading column attrs") + } + + // Translate column attributes, if necessary. + if e.Holder.translateFile != nil { + for _, col := range resp.ColumnAttrSets { + v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + + resp.ColumnAttrSets = columnAttrSets + } + + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, errors.Wrap(err, "getting attrs") + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil } func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) ([]interface{}, error) { From 432c2b5506d572ea2aedd7ca2a6c20ecbabc9bd0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 3 Oct 2018 17:32:22 +0300 Subject: [PATCH 36/45] set trackExistence to true in NewIndex --- index.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/index.go b/index.go index 0992ab213..297c02f20 100644 --- a/index.go +++ b/index.go @@ -69,9 +69,10 @@ func NewIndex(path, name string) (*Index, error) { newAttrStore: newNopAttrStore, columnAttrs: nopStore, - broadcaster: NopBroadcaster, - Stats: NopStatsClient, - logger: NopLogger, + broadcaster: NopBroadcaster, + Stats: NopStatsClient, + logger: NopLogger, + trackExistence: true, }, nil } From bd48db1435456dd6eb0c283eb7022199803985b4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 5 Oct 2018 16:29:29 +0300 Subject: [PATCH 37/45] updated executor.Execute logic for columnAttrs with keys; added columnAttrs with keys test --- executor.go | 35 +++++++++++++++++++++-------------- executor_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/executor.go b/executor.go index 29da40b27..235000938 100644 --- a/executor.go +++ b/executor.go @@ -118,17 +118,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar resp.Results = results - // Translate response objects from ids to keys, if necessary. - // No need to translate a remote call. - if !opt.Remote { - for i := range results { - results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) - if err != nil { - return resp, err - } - } - } - // Fill column attributes if requested. if opt.ColumnAttrs { // Consolidate all column ids across all calls. @@ -148,8 +137,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } // Translate column attributes, if necessary. - if e.Holder.translateFile != nil { - for _, col := range resp.ColumnAttrSets { + if idx.Keys() { + for _, col := range columnAttrSets { v, err := e.Holder.translateFile.TranslateColumnToString(index, col.ID) if err != nil { return resp, err @@ -161,6 +150,17 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar resp.ColumnAttrSets = columnAttrSets } + // Translate response objects from ids to keys, if necessary. + // No need to translate a remote call. + if !opt.Remote { + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return resp, err + } + } + } + return resp, nil } @@ -1765,9 +1765,16 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } } +var translateCallCol = map[string]struct{}{ + "Set": struct{}{}, + "Clear": struct{}{}, + "Row": struct{}{}, + "SetColumnAttrs": struct{}{}, +} + func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { var colKey, rowKey, fieldName string - if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + if _, ok := translateCallCol[c.Name]; ok { // Positional args in new PQL syntax require special handling here. colKey = "_" + columnLabel fieldName, _ = c.FieldArg() diff --git a/executor_test.go b/executor_test.go index d6d529efe..5a5f68ff9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1509,6 +1509,36 @@ func TestExecutor_QueryCall(t *testing.T) { } }) + t.Run("columnAttrsWithKeys", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Set columns for rows 0, 10, & 20 across two shards. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set("one-hundred", f="ten") + SetColumnAttrs("one-hundred", foo="bar") + `}); err != nil { + t.Fatal(err) + } + + targetColAttrSets := []*pilosa.ColumnAttrSet{ + {Key: "one-hundred", Attrs: map[string]interface{}{"foo": "bar"}}, + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Options(Row(f="ten"), columnAttrs=true)`}); err != nil { + t.Fatal(err) + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"one-hundred"}) { + t.Fatalf("unexpected keys: %+v", keys) + } else if attrs := res.ColumnAttrSets; !reflect.DeepEqual(attrs, targetColAttrSets) { + t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) + } + }) + t.Run("shards", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() From 619bc1bcd9501157d22c913d71a1d13786a26d25 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 25 Sep 2018 14:57:12 -0500 Subject: [PATCH 38/45] implements fragment.setRow(row, rowID) --- fragment.go | 54 +++++++++++++++++++++++++++++++++++++-- fragment_internal_test.go | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/fragment.go b/fragment.go index ee586afc6..63c9cce37 100644 --- a/fragment.go +++ b/fragment.go @@ -352,11 +352,11 @@ func (f *fragment) unprotectedRow(rowID uint64) *Row { } // Only use a subset of the containers. - // NOTE: The start & end ranges must be divisible by + // NOTE: The start & end ranges must be divisible by container width. data := f.storage.OffsetRange(f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // Reference bitmap subrange in storage. - // We Clone() data because otherwise row will contains pointers to containers in storage. + // We Clone() data because otherwise row will contain pointers to containers in storage. // This causes unexpected results when we cache the row and try to use it later. row := &Row{ segments: []rowSegment{{ @@ -491,6 +491,56 @@ func (f *fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, er return changed, nil } +// setRow replaces an existing row (specified by rowID) with the given +// Row. This updates both the on-disk storage and the in-cache bitmap. +func (f *fragment) setRow(row *Row, rowID uint64) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.unprotectedSetRow(row, rowID) +} + +func (f *fragment) unprotectedSetRow(row *Row, rowID uint64) (changed bool, err error) { + // TODO: In order to return `changed`, we need to first compare + // the existing row with the given row. Determine if the overhead + // of this is worth having `changed`. + // For now we will assume changed is always true. + changed = true + + // First container of the row in storage. + headContainerKey := rowID << shardVsContainerExponent + + // Remove every existing container in the row. + for i := uint64(0); i < (1 << shardVsContainerExponent); i++ { + f.storage.Containers.Remove(headContainerKey + i) + } + + // From the given row, get the rowSegment for this shard. + seg := row.segment(f.shard) + if seg == nil { + return changed, nil + } + + // Put each container from rowSegment to fragment storage. + citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) + for citer.Next() { + k, c := citer.Value() + f.storage.Containers.Put(headContainerKey+(k%(1< Date: Tue, 25 Sep 2018 17:02:49 -0500 Subject: [PATCH 39/45] implement Store() in the executor (i.e. setRow()) --- executor.go | 94 ++ executor_test.go | 139 +++ pql/pql.peg | 1 + pql/pql.peg.go | 2486 ++++++++++++++++++++++++---------------------- 4 files changed, 1505 insertions(+), 1215 deletions(-) diff --git a/executor.go b/executor.go index 8dd4db569..53a39d04e 100644 --- a/executor.go +++ b/executor.go @@ -184,6 +184,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeClearBit(ctx, index, c, opt) case "ClearRow": return e.executeClearRow(ctx, index, c, shards, opt) + case "Store": + return e.executeSetRow(ctx, index, c, shards, opt) case "Count": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeCount(ctx, index, c, shards, opt) @@ -1213,6 +1215,98 @@ func (e *executor) executeClearRowShard(_ context.Context, index string, c *pql. return changed, nil } +// executeSetRow executes a SetRow() call. +func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (bool, error) { + // Ensure the field type supports SetRow(). + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + switch field.Type() { + case FieldTypeSet: + // These field types support SetRow(). + default: + return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeSetRowShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + val := v.(bool) + if prev == nil { + return val + } + return val || prev.(bool) + } + + result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + return result.(bool), err +} + +// executeSetRowShard executes a SetRow() call for a single shard. +func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (bool, error) { + fieldName, err := c.FieldArg() + if err != nil { + return false, errors.New("SetRow() argument required: field") + } + + // Read fields using labels. + rowID, ok, err := c.UintArg(fieldName) + if err != nil { + return false, fmt.Errorf("reading SetRow() row: %v", err) + } else if !ok { + return false, fmt.Errorf("SetRow() row argument '%v' required", rowLabel) + } + + field := e.Holder.Field(index, fieldName) + if field == nil { + return false, ErrFieldNotFound + } + + // Retrieve source row. + var src *Row + if len(c.Children) == 1 { + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return false, errors.Wrap(err, "getting source row") + } + src = row + } else { + return false, errors.New("SetRow() requires a source row") + } + + // Set the row on the standard view. + changed := false + fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) + if fragment == nil { + // Since the destination fragment doesn't exist, create one. + view, err := field.createViewIfNotExists(viewStandard) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + fragment, err = view.createFragmentIfNotExists(shard) + if err != nil { + return false, errors.Wrapf(err, "creating fragment: %d", shard) + } + } + cleared, err := fragment.setRow(src, rowID) + if err != nil { + return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) + } + changed = changed || cleared + + return changed, nil +} + // executeSet executes a Set() call. func (e *executor) executeSet(ctx context.Context, index string, c *pql.Call, opt *execOptions) (bool, error) { // Read colID. diff --git a/executor_test.go b/executor_test.go index d6d529efe..8d3786cd5 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1913,6 +1913,145 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { }) } +// Ensure a row can be set. +func TestExecutor_Execute_SetRow(t *testing.T) { + t.Run("Set_NewRow", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + if _, err := index.CreateField("tmp", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_NoSource", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a different row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 9 (which doesn't exist) into a row that does exist. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) + t.Run("Set_ExistingDestination", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Set bits. + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + + fmt.Sprintf("Set(%d, f=%d)\n", 1, 20) + + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 20), + }); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + + // Store row 10 into an existing row. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil { + t.Fatal(err) + } else if res := res.Results[0].(bool); !res { + t.Fatalf("unexpected set row result: %+v", res) + } + + // Ensure the row was populated. + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + t.Fatal(err) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) + } + }) +} + func benchmarkExistence(nn bool, b *testing.B) { c := test.MustRunCluster(b, 1) defer c.Close() diff --git a/pql/pql.peg b/pql/pql.peg index c38046f67..9e0087046 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -11,6 +11,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'SetColumnAttrs' {p.startCall("SetColumnAttrs")} open col comma args close {p.endCall()} / 'Clear' {p.startCall("Clear")} open col comma args close {p.endCall()} / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} + / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 5adc244f8..2d5877fbb 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -64,9 +64,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 - rulePegText ruleAction14 ruleAction15 + rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -101,6 +101,8 @@ const ( ruleAction47 ruleAction48 ruleAction49 + ruleAction50 + ruleAction51 ) var rul3s = [...]string{ @@ -153,9 +155,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", - "PegText", "Action14", "Action15", + "PegText", "Action16", "Action17", "Action18", @@ -190,6 +192,8 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", + "Action50", + "Action51", } type token32 struct { @@ -306,7 +310,7 @@ type PQL struct { Buffer string buffer []rune - rules [86]func() bool + rules [88]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -419,84 +423,88 @@ func (p *PQL) Execute() { case ruleAction9: p.endCall() case ruleAction10: - p.startCall("TopN") + p.startCall("Store") case ruleAction11: p.endCall() case ruleAction12: - p.startCall("Range") + p.startCall("TopN") case ruleAction13: p.endCall() case ruleAction14: - p.startCall(buffer[begin:end]) + p.startCall("Range") case ruleAction15: p.endCall() case ruleAction16: - p.addBTWN() + p.startCall(buffer[begin:end]) case ruleAction17: - p.addLTE() + p.endCall() case ruleAction18: - p.addGTE() + p.addBTWN() case ruleAction19: - p.addEQ() + p.addLTE() case ruleAction20: - p.addNEQ() + p.addGTE() case ruleAction21: - p.addLT() + p.addEQ() case ruleAction22: - p.addGT() + p.addNEQ() case ruleAction23: - p.startConditional() + p.addLT() case ruleAction24: - p.endConditional() + p.addGT() case ruleAction25: - p.condAdd(buffer[begin:end]) + p.startConditional() case ruleAction26: - p.condAdd(buffer[begin:end]) + p.endConditional() case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.addPosStr("_start", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction29: - p.addPosStr("_end", buffer[begin:end]) + p.condAdd(buffer[begin:end]) case ruleAction30: - p.startList() + p.addPosStr("_start", buffer[begin:end]) case ruleAction31: - p.endList() + p.addPosStr("_end", buffer[begin:end]) case ruleAction32: - p.addVal(nil) + p.startList() case ruleAction33: - p.addVal(true) + p.endList() case ruleAction34: - p.addVal(false) + p.addVal(nil) case ruleAction35: - p.addNumVal(buffer[begin:end]) + p.addVal(true) case ruleAction36: - p.addNumVal(buffer[begin:end]) + p.addVal(false) case ruleAction37: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction38: - p.addVal(buffer[begin:end]) + p.addNumVal(buffer[begin:end]) case ruleAction39: p.addVal(buffer[begin:end]) case ruleAction40: - p.addField(buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction41: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction42: - p.addPosNum("_row", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction43: - p.addPosNum("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction44: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction45: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction46: p.addPosNum("_row", buffer[begin:end]) + case ruleAction45: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction46: + p.addPosStr("_col", buffer[begin:end]) case ruleAction47: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction49: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -609,7 +617,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('T' 'o' 'p' 'N' Action10 open posfield (comma allargs)? close Action11) / ('R' 'a' 'n' 'g' 'e' Action12 open (timerange / conditional / arg) close Action13) / ( Action14 open allargs comma? close Action15))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'a' 'n' 'g' 'e' Action14 open (timerange / conditional / arg) close Action15) / ( Action16 open allargs comma? close Action17))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -658,7 +666,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction49, position) + add(ruleAction51, position) } add(ruletimestamp, position12) } @@ -744,7 +752,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction46, position) + add(ruleAction48, position) } goto l19 l20: @@ -765,7 +773,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction47, position) + add(ruleAction49, position) } goto l19 l23: @@ -786,7 +794,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction50, position) } } l19: @@ -981,7 +989,11 @@ func (p *PQL) Init() { goto l7 l35: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('T') { + if buffer[position] != rune('S') { + goto l38 + } + position++ + if buffer[position] != rune('t') { goto l38 } position++ @@ -989,11 +1001,11 @@ func (p *PQL) Init() { goto l38 } position++ - if buffer[position] != rune('p') { + if buffer[position] != rune('r') { goto l38 } position++ - if buffer[position] != rune('N') { + if buffer[position] != rune('e') { goto l38 } position++ @@ -1003,22 +1015,15 @@ func (p *PQL) Init() { if !_rules[ruleopen]() { goto l38 } - if !_rules[ruleposfield]() { + if !_rules[ruleCall]() { goto l38 } - { - position40, tokenIndex40 := position, tokenIndex - if !_rules[rulecomma]() { - goto l40 - } - if !_rules[ruleallargs]() { - goto l40 - } - goto l41 - l40: - position, tokenIndex = position40, tokenIndex40 + if !_rules[rulecomma]() { + goto l38 + } + if !_rules[rulearg]() { + goto l38 } - l41: if !_rules[ruleclose]() { goto l38 } @@ -1028,193 +1033,240 @@ func (p *PQL) Init() { goto l7 l38: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l43 + if buffer[position] != rune('T') { + goto l41 } position++ - if buffer[position] != rune('a') { - goto l43 + if buffer[position] != rune('o') { + goto l41 } position++ - if buffer[position] != rune('n') { - goto l43 + if buffer[position] != rune('p') { + goto l41 } position++ - if buffer[position] != rune('g') { - goto l43 - } - position++ - if buffer[position] != rune('e') { - goto l43 + if buffer[position] != rune('N') { + goto l41 } position++ { add(ruleAction12, position) } if !_rules[ruleopen]() { - goto l43 + goto l41 + } + if !_rules[ruleposfield]() { + goto l41 } { - position45, tokenIndex45 := position, tokenIndex - { - position47 := position - if !_rules[rulefield]() { - goto l46 - } - if !_rules[rulesp]() { - goto l46 - } - if buffer[position] != rune('=') { - goto l46 - } - position++ - if !_rules[rulesp]() { - goto l46 - } - if !_rules[rulevalue]() { - goto l46 - } - if !_rules[rulecomma]() { - goto l46 - } - { - position48 := position - if !_rules[ruletimestampfmt]() { - goto l46 - } - add(rulePegText, position48) - } - { - add(ruleAction28, position) - } - if !_rules[rulecomma]() { - goto l46 - } - { - position50 := position - if !_rules[ruletimestampfmt]() { - goto l46 - } - add(rulePegText, position50) - } - { - add(ruleAction29, position) - } - add(ruletimerange, position47) - } - goto l45 - l46: - position, tokenIndex = position45, tokenIndex45 - { - position53 := position - { - add(ruleAction23, position) - } - if !_rules[rulecondint]() { - goto l52 - } - if !_rules[rulecondLT]() { - goto l52 - } - { - position55 := position - { - position56 := position - if !_rules[rulefieldExpr]() { - goto l52 - } - add(rulePegText, position56) - } - if !_rules[rulesp]() { - goto l52 - } - { - add(ruleAction27, position) - } - add(rulecondfield, position55) - } - if !_rules[rulecondLT]() { - goto l52 - } - if !_rules[rulecondint]() { - goto l52 - } - { - add(ruleAction24, position) - } - add(ruleconditional, position53) - } - goto l45 - l52: - position, tokenIndex = position45, tokenIndex45 - if !_rules[rulearg]() { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulecomma]() { goto l43 } + if !_rules[ruleallargs]() { + goto l43 + } + goto l44 + l43: + position, tokenIndex = position43, tokenIndex43 } - l45: + l44: if !_rules[ruleclose]() { - goto l43 + goto l41 } { add(ruleAction13, position) } goto l7 - l43: + l41: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l46 + } + position++ + if buffer[position] != rune('a') { + goto l46 + } + position++ + if buffer[position] != rune('n') { + goto l46 + } + position++ + if buffer[position] != rune('g') { + goto l46 + } + position++ + if buffer[position] != rune('e') { + goto l46 + } + position++ + { + add(ruleAction14, position) + } + if !_rules[ruleopen]() { + goto l46 + } + { + position48, tokenIndex48 := position, tokenIndex + { + position50 := position + if !_rules[rulefield]() { + goto l49 + } + if !_rules[rulesp]() { + goto l49 + } + if buffer[position] != rune('=') { + goto l49 + } + position++ + if !_rules[rulesp]() { + goto l49 + } + if !_rules[rulevalue]() { + goto l49 + } + if !_rules[rulecomma]() { + goto l49 + } + { + position51 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position51) + } + { + add(ruleAction30, position) + } + if !_rules[rulecomma]() { + goto l49 + } + { + position53 := position + if !_rules[ruletimestampfmt]() { + goto l49 + } + add(rulePegText, position53) + } + { + add(ruleAction31, position) + } + add(ruletimerange, position50) + } + goto l48 + l49: + position, tokenIndex = position48, tokenIndex48 + { + position56 := position + { + add(ruleAction25, position) + } + if !_rules[rulecondint]() { + goto l55 + } + if !_rules[rulecondLT]() { + goto l55 + } + { + position58 := position + { + position59 := position + if !_rules[rulefieldExpr]() { + goto l55 + } + add(rulePegText, position59) + } + if !_rules[rulesp]() { + goto l55 + } + { + add(ruleAction29, position) + } + add(rulecondfield, position58) + } + if !_rules[rulecondLT]() { + goto l55 + } + if !_rules[rulecondint]() { + goto l55 + } + { + add(ruleAction26, position) + } + add(ruleconditional, position56) + } + goto l48 + l55: + position, tokenIndex = position48, tokenIndex48 + if !_rules[rulearg]() { + goto l46 + } + } + l48: + if !_rules[ruleclose]() { + goto l46 + } + { + add(ruleAction15, position) + } + goto l7 + l46: position, tokenIndex = position7, tokenIndex7 { - position60 := position + position63 := position { - position61 := position + position64 := position { - position62, tokenIndex62 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l63 + goto l66 } position++ - goto l62 - l63: - position, tokenIndex = position62, tokenIndex62 + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l5 } position++ } - l62: - l64: + l65: + l67: { - position65, tokenIndex65 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex { - position66, tokenIndex66 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l67 + goto l70 } position++ - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 + goto l69 + l70: + position, tokenIndex = position69, tokenIndex69 if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l71 + } + position++ + goto l69 + l71: + position, tokenIndex = position69, tokenIndex69 + if c := buffer[position]; c < rune('0') || c > rune('9') { goto l68 } position++ - goto l66 - l68: - position, tokenIndex = position66, tokenIndex66 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l65 - } - position++ } - l66: - goto l64 - l65: - position, tokenIndex = position65, tokenIndex65 + l69: + goto l67 + l68: + position, tokenIndex = position68, tokenIndex68 } - add(ruleIDENT, position61) + add(ruleIDENT, position64) } - add(rulePegText, position60) + add(rulePegText, position63) } { - add(ruleAction14, position) + add(ruleAction16, position) } if !_rules[ruleopen]() { goto l5 @@ -1223,20 +1275,20 @@ func (p *PQL) Init() { goto l5 } { - position70, tokenIndex70 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if !_rules[rulecomma]() { - goto l70 + goto l73 } - goto l71 - l70: - position, tokenIndex = position70, tokenIndex70 + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 } - l71: + l74: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction15, position) + add(ruleAction17, position) } } l7: @@ -1249,1487 +1301,1487 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position73, tokenIndex73 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position74 := position + position77 := position { - position75, tokenIndex75 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex if !_rules[ruleCall]() { - goto l76 - } - l77: - { - position78, tokenIndex78 := position, tokenIndex - if !_rules[rulecomma]() { - goto l78 - } - if !_rules[ruleCall]() { - goto l78 - } - goto l77 - l78: - position, tokenIndex = position78, tokenIndex78 - } - { - position79, tokenIndex79 := position, tokenIndex - if !_rules[rulecomma]() { - goto l79 - } - if !_rules[ruleargs]() { - goto l79 - } - goto l80 - l79: - position, tokenIndex = position79, tokenIndex79 + goto l79 } l80: - goto l75 - l76: - position, tokenIndex = position75, tokenIndex75 - if !_rules[ruleargs]() { - goto l81 + { + position81, tokenIndex81 := position, tokenIndex + if !_rules[rulecomma]() { + goto l81 + } + if !_rules[ruleCall]() { + goto l81 + } + goto l80 + l81: + position, tokenIndex = position81, tokenIndex81 } - goto l75 - l81: - position, tokenIndex = position75, tokenIndex75 + { + position82, tokenIndex82 := position, tokenIndex + if !_rules[rulecomma]() { + goto l82 + } + if !_rules[ruleargs]() { + goto l82 + } + goto l83 + l82: + position, tokenIndex = position82, tokenIndex82 + } + l83: + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 + if !_rules[ruleargs]() { + goto l84 + } + goto l78 + l84: + position, tokenIndex = position78, tokenIndex78 if !_rules[rulesp]() { - goto l73 + goto l76 } } - l75: - add(ruleallargs, position74) + l78: + add(ruleallargs, position77) } return true - l73: - position, tokenIndex = position73, tokenIndex73 + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position82, tokenIndex82 := position, tokenIndex + position85, tokenIndex85 := position, tokenIndex { - position83 := position + position86 := position if !_rules[rulearg]() { - goto l82 + goto l85 } { - position84, tokenIndex84 := position, tokenIndex + position87, tokenIndex87 := position, tokenIndex if !_rules[rulecomma]() { - goto l84 + goto l87 } if !_rules[ruleargs]() { - goto l84 + goto l87 } - goto l85 - l84: - position, tokenIndex = position84, tokenIndex84 + goto l88 + l87: + position, tokenIndex = position87, tokenIndex87 } - l85: + l88: if !_rules[rulesp]() { - goto l82 + goto l85 } - add(ruleargs, position83) + add(ruleargs, position86) } return true - l82: - position, tokenIndex = position82, tokenIndex82 + l85: + position, tokenIndex = position85, tokenIndex85 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position86, tokenIndex86 := position, tokenIndex + position89, tokenIndex89 := position, tokenIndex { - position87 := position + position90 := position { - position88, tokenIndex88 := position, tokenIndex + position91, tokenIndex91 := position, tokenIndex if !_rules[rulefield]() { - goto l89 + goto l92 } if !_rules[rulesp]() { - goto l89 + goto l92 } if buffer[position] != rune('=') { - goto l89 + goto l92 } position++ if !_rules[rulesp]() { - goto l89 + goto l92 } if !_rules[rulevalue]() { + goto l92 + } + goto l91 + l92: + position, tokenIndex = position91, tokenIndex91 + if !_rules[rulefield]() { goto l89 } - goto l88 - l89: - position, tokenIndex = position88, tokenIndex88 - if !_rules[rulefield]() { - goto l86 - } if !_rules[rulesp]() { - goto l86 + goto l89 } { - position90 := position + position93 := position { - position91, tokenIndex91 := position, tokenIndex + position94, tokenIndex94 := position, tokenIndex if buffer[position] != rune('>') { - goto l92 + goto l95 } position++ if buffer[position] != rune('<') { - goto l92 - } - position++ - { - add(ruleAction16, position) - } - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('<') { - goto l94 - } - position++ - if buffer[position] != rune('=') { - goto l94 - } - position++ - { - add(ruleAction17, position) - } - goto l91 - l94: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('>') { - goto l96 - } - position++ - if buffer[position] != rune('=') { - goto l96 + goto l95 } position++ { add(ruleAction18, position) } - goto l91 - l96: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('=') { - goto l98 + goto l94 + l95: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('<') { + goto l97 } position++ if buffer[position] != rune('=') { - goto l98 + goto l97 } position++ { add(ruleAction19, position) } - goto l91 - l98: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('!') { - goto l100 + goto l94 + l97: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('>') { + goto l99 } position++ if buffer[position] != rune('=') { - goto l100 + goto l99 } position++ { add(ruleAction20, position) } - goto l91 - l100: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('<') { - goto l102 + goto l94 + l99: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('=') { + goto l101 + } + position++ + if buffer[position] != rune('=') { + goto l101 } position++ { add(ruleAction21, position) } - goto l91 - l102: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('>') { - goto l86 + goto l94 + l101: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('!') { + goto l103 + } + position++ + if buffer[position] != rune('=') { + goto l103 } position++ { add(ruleAction22, position) } - } - l91: - add(ruleCOND, position90) - } - if !_rules[rulesp]() { - goto l86 - } - if !_rules[rulevalue]() { - goto l86 - } - } - l88: - add(rulearg, position87) - } - return true - l86: - position, tokenIndex = position86, tokenIndex86 - return false - }, - /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ - nil, - /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ - nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ - func() bool { - position107, tokenIndex107 := position, tokenIndex - { - position108 := position - { - position109 := position - { - position110, tokenIndex110 := position, tokenIndex - { - position112, tokenIndex112 := position, tokenIndex - if buffer[position] != rune('-') { - goto l112 + goto l94 + l103: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('<') { + goto l105 } position++ - goto l113 - l112: - position, tokenIndex = position112, tokenIndex112 + { + add(ruleAction23, position) + } + goto l94 + l105: + position, tokenIndex = position94, tokenIndex94 + if buffer[position] != rune('>') { + goto l89 + } + position++ + { + add(ruleAction24, position) + } } - l113: - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l111 - } - position++ - l114: + l94: + add(ruleCOND, position93) + } + if !_rules[rulesp]() { + goto l89 + } + if !_rules[rulevalue]() { + goto l89 + } + } + l91: + add(rulearg, position90) + } + return true + l89: + position, tokenIndex = position89, tokenIndex89 + return false + }, + /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ + nil, + /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ + nil, + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ + func() bool { + position110, tokenIndex110 := position, tokenIndex + { + position111 := position + { + position112 := position + { + position113, tokenIndex113 := position, tokenIndex { position115, tokenIndex115 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { + if buffer[position] != rune('-') { goto l115 } position++ - goto l114 + goto l116 l115: position, tokenIndex = position115, tokenIndex115 } - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 + l116: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l114 + } + position++ + l117: + { + position118, tokenIndex118 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l118 + } + position++ + goto l117 + l118: + position, tokenIndex = position118, tokenIndex118 + } + goto l113 + l114: + position, tokenIndex = position113, tokenIndex113 if buffer[position] != rune('0') { - goto l107 + goto l110 } position++ } - l110: - add(rulePegText, position109) + l113: + add(rulePegText, position112) } if !_rules[rulesp]() { - goto l107 + goto l110 } { - add(ruleAction25, position) + add(ruleAction27, position) } - add(rulecondint, position108) + add(rulecondint, position111) } return true - l107: - position, tokenIndex = position107, tokenIndex107 + l110: + position, tokenIndex = position110, tokenIndex110 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position117, tokenIndex117 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex { - position118 := position + position121 := position { - position119 := position + position122 := position { - position120, tokenIndex120 := position, tokenIndex + position123, tokenIndex123 := position, tokenIndex if buffer[position] != rune('<') { - goto l121 + goto l124 } position++ if buffer[position] != rune('=') { - goto l121 + goto l124 } position++ - goto l120 - l121: - position, tokenIndex = position120, tokenIndex120 + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 if buffer[position] != rune('<') { - goto l117 + goto l120 } position++ } - l120: - add(rulePegText, position119) + l123: + add(rulePegText, position122) } if !_rules[rulesp]() { - goto l117 + goto l120 } { - add(ruleAction26, position) + add(ruleAction28, position) } - add(rulecondLT, position118) + add(rulecondLT, position121) } return true - l117: - position, tokenIndex = position117, tokenIndex117 + l120: + position, tokenIndex = position120, tokenIndex120 return false }, - /* 9 condfield <- <( sp Action27)> */ + /* 9 condfield <- <( sp Action29)> */ nil, - /* 10 timerange <- <(field sp '=' sp value comma Action28 comma Action29)> */ + /* 10 timerange <- <(field sp '=' sp value comma Action30 comma Action31)> */ nil, - /* 11 value <- <(item / (lbrack Action30 list rbrack Action31))> */ + /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ func() bool { - position125, tokenIndex125 := position, tokenIndex + position128, tokenIndex128 := position, tokenIndex { - position126 := position + position129 := position { - position127, tokenIndex127 := position, tokenIndex + position130, tokenIndex130 := position, tokenIndex if !_rules[ruleitem]() { - goto l128 + goto l131 } - goto l127 - l128: - position, tokenIndex = position127, tokenIndex127 + goto l130 + l131: + position, tokenIndex = position130, tokenIndex130 { - position129 := position + position132 := position if buffer[position] != rune('[') { - goto l125 + goto l128 } position++ if !_rules[rulesp]() { - goto l125 + goto l128 } - add(rulelbrack, position129) - } - { - add(ruleAction30, position) - } - if !_rules[rulelist]() { - goto l125 - } - { - position131 := position - if !_rules[rulesp]() { - goto l125 - } - if buffer[position] != rune(']') { - goto l125 - } - position++ - if !_rules[rulesp]() { - goto l125 - } - add(rulerbrack, position131) - } - { - add(ruleAction31, position) - } - } - l127: - add(rulevalue, position126) - } - return true - l125: - position, tokenIndex = position125, tokenIndex125 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position133, tokenIndex133 := position, tokenIndex - { - position134 := position - if !_rules[ruleitem]() { - goto l133 - } - { - position135, tokenIndex135 := position, tokenIndex - if !_rules[rulecomma]() { - goto l135 - } - if !_rules[rulelist]() { - goto l135 - } - goto l136 - l135: - position, tokenIndex = position135, tokenIndex135 - } - l136: - add(rulelist, position134) - } - return true - l133: - position, tokenIndex = position133, tokenIndex133 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action35) / (<('-'? '.' [0-9]+)> Action36) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action37) / ('"' '"' Action38) / ('\'' '\'' Action39))> */ - func() bool { - position137, tokenIndex137 := position, tokenIndex - { - position138 := position - { - position139, tokenIndex139 := position, tokenIndex - if buffer[position] != rune('n') { - goto l140 - } - position++ - if buffer[position] != rune('u') { - goto l140 - } - position++ - if buffer[position] != rune('l') { - goto l140 - } - position++ - if buffer[position] != rune('l') { - goto l140 - } - position++ - { - position141, tokenIndex141 := position, tokenIndex - { - position142, tokenIndex142 := position, tokenIndex - if !_rules[rulecomma]() { - goto l143 - } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 - if !_rules[rulesp]() { - goto l140 - } - if !_rules[ruleclose]() { - goto l140 - } - } - l142: - position, tokenIndex = position141, tokenIndex141 + add(rulelbrack, position132) } { add(ruleAction32, position) } - goto l139 - l140: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('t') { - goto l145 + if !_rules[rulelist]() { + goto l128 } - position++ - if buffer[position] != rune('r') { - goto l145 - } - position++ - if buffer[position] != rune('u') { - goto l145 - } - position++ - if buffer[position] != rune('e') { - goto l145 - } - position++ { - position146, tokenIndex146 := position, tokenIndex - { - position147, tokenIndex147 := position, tokenIndex - if !_rules[rulecomma]() { - goto l148 - } - goto l147 - l148: - position, tokenIndex = position147, tokenIndex147 - if !_rules[rulesp]() { - goto l145 - } - if !_rules[ruleclose]() { - goto l145 - } + position134 := position + if !_rules[rulesp]() { + goto l128 } - l147: - position, tokenIndex = position146, tokenIndex146 + if buffer[position] != rune(']') { + goto l128 + } + position++ + if !_rules[rulesp]() { + goto l128 + } + add(rulerbrack, position134) } { add(ruleAction33, position) } + } + l130: + add(rulevalue, position129) + } + return true + l128: + position, tokenIndex = position128, tokenIndex128 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position136, tokenIndex136 := position, tokenIndex + { + position137 := position + if !_rules[ruleitem]() { + goto l136 + } + { + position138, tokenIndex138 := position, tokenIndex + if !_rules[rulecomma]() { + goto l138 + } + if !_rules[rulelist]() { + goto l138 + } goto l139 - l145: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('f') { - goto l150 + l138: + position, tokenIndex = position138, tokenIndex138 + } + l139: + add(rulelist, position137) + } + return true + l136: + position, tokenIndex = position136, tokenIndex136 + return false + }, + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action39) / ('"' '"' Action40) / ('\'' '\'' Action41))> */ + func() bool { + position140, tokenIndex140 := position, tokenIndex + { + position141 := position + { + position142, tokenIndex142 := position, tokenIndex + if buffer[position] != rune('n') { + goto l143 } position++ - if buffer[position] != rune('a') { - goto l150 + if buffer[position] != rune('u') { + goto l143 } position++ if buffer[position] != rune('l') { - goto l150 + goto l143 } position++ - if buffer[position] != rune('s') { - goto l150 - } - position++ - if buffer[position] != rune('e') { - goto l150 + if buffer[position] != rune('l') { + goto l143 } position++ { - position151, tokenIndex151 := position, tokenIndex + position144, tokenIndex144 := position, tokenIndex { - position152, tokenIndex152 := position, tokenIndex + position145, tokenIndex145 := position, tokenIndex if !_rules[rulecomma]() { - goto l153 + goto l146 } - goto l152 - l153: - position, tokenIndex = position152, tokenIndex152 + goto l145 + l146: + position, tokenIndex = position145, tokenIndex145 if !_rules[rulesp]() { - goto l150 + goto l143 } if !_rules[ruleclose]() { - goto l150 + goto l143 } } - l152: - position, tokenIndex = position151, tokenIndex151 + l145: + position, tokenIndex = position144, tokenIndex144 } { add(ruleAction34, position) } - goto l139 - l150: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('t') { + goto l148 + } + position++ + if buffer[position] != rune('r') { + goto l148 + } + position++ + if buffer[position] != rune('u') { + goto l148 + } + position++ + if buffer[position] != rune('e') { + goto l148 + } + position++ { - position156 := position + position149, tokenIndex149 := position, tokenIndex { - position157, tokenIndex157 := position, tokenIndex - if buffer[position] != rune('-') { - goto l157 + position150, tokenIndex150 := position, tokenIndex + if !_rules[rulecomma]() { + goto l151 } - position++ - goto l158 - l157: - position, tokenIndex = position157, tokenIndex157 - } - l158: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l155 - } - position++ - l159: - { - position160, tokenIndex160 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l160 + goto l150 + l151: + position, tokenIndex = position150, tokenIndex150 + if !_rules[rulesp]() { + goto l148 } - position++ - goto l159 - l160: - position, tokenIndex = position160, tokenIndex160 - } - { - position161, tokenIndex161 := position, tokenIndex - if buffer[position] != rune('.') { - goto l161 + if !_rules[ruleclose]() { + goto l148 } - position++ - l163: - { - position164, tokenIndex164 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l164 - } - position++ - goto l163 - l164: - position, tokenIndex = position164, tokenIndex164 - } - goto l162 - l161: - position, tokenIndex = position161, tokenIndex161 } - l162: - add(rulePegText, position156) + l150: + position, tokenIndex = position149, tokenIndex149 } { add(ruleAction35, position) } - goto l139 - l155: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l148: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('f') { + goto l153 + } + position++ + if buffer[position] != rune('a') { + goto l153 + } + position++ + if buffer[position] != rune('l') { + goto l153 + } + position++ + if buffer[position] != rune('s') { + goto l153 + } + position++ + if buffer[position] != rune('e') { + goto l153 + } + position++ { - position167 := position + position154, tokenIndex154 := position, tokenIndex { - position168, tokenIndex168 := position, tokenIndex - if buffer[position] != rune('-') { - goto l168 + position155, tokenIndex155 := position, tokenIndex + if !_rules[rulecomma]() { + goto l156 } - position++ - goto l169 - l168: - position, tokenIndex = position168, tokenIndex168 - } - l169: - if buffer[position] != rune('.') { - goto l166 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l166 - } - position++ - l170: - { - position171, tokenIndex171 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l171 + goto l155 + l156: + position, tokenIndex = position155, tokenIndex155 + if !_rules[rulesp]() { + goto l153 + } + if !_rules[ruleclose]() { + goto l153 } - position++ - goto l170 - l171: - position, tokenIndex = position171, tokenIndex171 } - add(rulePegText, position167) + l155: + position, tokenIndex = position154, tokenIndex154 } { add(ruleAction36, position) } - goto l139 - l166: - position, tokenIndex = position139, tokenIndex139 + goto l142 + l153: + position, tokenIndex = position142, tokenIndex142 { - position174 := position + position159 := position { - position177, tokenIndex177 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l178 - } - position++ - goto l177 - l178: - position, tokenIndex = position177, tokenIndex177 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l179 - } - position++ - goto l177 - l179: - position, tokenIndex = position177, tokenIndex177 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l180 - } - position++ - goto l177 - l180: - position, tokenIndex = position177, tokenIndex177 + position160, tokenIndex160 := position, tokenIndex if buffer[position] != rune('-') { - goto l181 - } - position++ - goto l177 - l181: - position, tokenIndex = position177, tokenIndex177 - if buffer[position] != rune('_') { - goto l182 - } - position++ - goto l177 - l182: - position, tokenIndex = position177, tokenIndex177 - if buffer[position] != rune(':') { - goto l173 + goto l160 } position++ + goto l161 + l160: + position, tokenIndex = position160, tokenIndex160 } - l177: - l175: + l161: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l158 + } + position++ + l162: { - position176, tokenIndex176 := position, tokenIndex - { - position183, tokenIndex183 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l184 - } - position++ - goto l183 - l184: - position, tokenIndex = position183, tokenIndex183 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l185 - } - position++ - goto l183 - l185: - position, tokenIndex = position183, tokenIndex183 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l186 - } - position++ - goto l183 - l186: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune('-') { - goto l187 - } - position++ - goto l183 - l187: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune('_') { - goto l188 - } - position++ - goto l183 - l188: - position, tokenIndex = position183, tokenIndex183 - if buffer[position] != rune(':') { - goto l176 - } - position++ + position163, tokenIndex163 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l163 } - l183: - goto l175 - l176: - position, tokenIndex = position176, tokenIndex176 + position++ + goto l162 + l163: + position, tokenIndex = position163, tokenIndex163 } - add(rulePegText, position174) + { + position164, tokenIndex164 := position, tokenIndex + if buffer[position] != rune('.') { + goto l164 + } + position++ + l166: + { + position167, tokenIndex167 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l167 + } + position++ + goto l166 + l167: + position, tokenIndex = position167, tokenIndex167 + } + goto l165 + l164: + position, tokenIndex = position164, tokenIndex164 + } + l165: + add(rulePegText, position159) } { add(ruleAction37, position) } - goto l139 - l173: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('"') { - goto l190 - } - position++ + goto l142 + l158: + position, tokenIndex = position142, tokenIndex142 { - position191 := position - if !_rules[ruledoublequotedstring]() { - goto l190 + position170 := position + { + position171, tokenIndex171 := position, tokenIndex + if buffer[position] != rune('-') { + goto l171 + } + position++ + goto l172 + l171: + position, tokenIndex = position171, tokenIndex171 } - add(rulePegText, position191) + l172: + if buffer[position] != rune('.') { + goto l169 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l169 + } + position++ + l173: + { + position174, tokenIndex174 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l174 + } + position++ + goto l173 + l174: + position, tokenIndex = position174, tokenIndex174 + } + add(rulePegText, position170) } - if buffer[position] != rune('"') { - goto l190 - } - position++ { add(ruleAction38, position) } - goto l139 - l190: - position, tokenIndex = position139, tokenIndex139 - if buffer[position] != rune('\'') { - goto l137 - } - position++ + goto l142 + l169: + position, tokenIndex = position142, tokenIndex142 { - position193 := position - if !_rules[rulesinglequotedstring]() { - goto l137 + position177 := position + { + position180, tokenIndex180 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l181 + } + position++ + goto l180 + l181: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l182 + } + position++ + goto l180 + l182: + position, tokenIndex = position180, tokenIndex180 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l183 + } + position++ + goto l180 + l183: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('-') { + goto l184 + } + position++ + goto l180 + l184: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune('_') { + goto l185 + } + position++ + goto l180 + l185: + position, tokenIndex = position180, tokenIndex180 + if buffer[position] != rune(':') { + goto l176 + } + position++ } - add(rulePegText, position193) + l180: + l178: + { + position179, tokenIndex179 := position, tokenIndex + { + position186, tokenIndex186 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l187 + } + position++ + goto l186 + l187: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l188 + } + position++ + goto l186 + l188: + position, tokenIndex = position186, tokenIndex186 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l189 + } + position++ + goto l186 + l189: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('-') { + goto l190 + } + position++ + goto l186 + l190: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune('_') { + goto l191 + } + position++ + goto l186 + l191: + position, tokenIndex = position186, tokenIndex186 + if buffer[position] != rune(':') { + goto l179 + } + position++ + } + l186: + goto l178 + l179: + position, tokenIndex = position179, tokenIndex179 + } + add(rulePegText, position177) } - if buffer[position] != rune('\'') { - goto l137 - } - position++ { add(ruleAction39, position) } + goto l142 + l176: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + position194 := position + if !_rules[ruledoublequotedstring]() { + goto l193 + } + add(rulePegText, position194) + } + if buffer[position] != rune('"') { + goto l193 + } + position++ + { + add(ruleAction40, position) + } + goto l142 + l193: + position, tokenIndex = position142, tokenIndex142 + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + position196 := position + if !_rules[rulesinglequotedstring]() { + goto l140 + } + add(rulePegText, position196) + } + if buffer[position] != rune('\'') { + goto l140 + } + position++ + { + add(ruleAction41, position) + } } - l139: - add(ruleitem, position138) + l142: + add(ruleitem, position141) } return true - l137: - position, tokenIndex = position137, tokenIndex137 + l140: + position, tokenIndex = position140, tokenIndex140 return false }, /* 14 doublequotedstring <- <((!('"' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position196 := position - l197: + position199 := position + l200: { - position198, tokenIndex198 := position, tokenIndex + position201, tokenIndex201 := position, tokenIndex { - position199, tokenIndex199 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex { - position201, tokenIndex201 := position, tokenIndex + position204, tokenIndex204 := position, tokenIndex { - position202, tokenIndex202 := position, tokenIndex + position205, tokenIndex205 := position, tokenIndex if buffer[position] != rune('"') { - goto l203 + goto l206 } position++ - goto l202 - l203: - position, tokenIndex = position202, tokenIndex202 + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 if buffer[position] != rune('\\') { + goto l207 + } + position++ + goto l205 + l207: + position, tokenIndex = position205, tokenIndex205 + if buffer[position] != rune('\n') { goto l204 } position++ - goto l202 - l204: - position, tokenIndex = position202, tokenIndex202 - if buffer[position] != rune('\n') { - goto l201 - } - position++ } - l202: - goto l200 - l201: - position, tokenIndex = position201, tokenIndex201 + l205: + goto l203 + l204: + position, tokenIndex = position204, tokenIndex204 } if !matchDot() { - goto l200 + goto l203 } - goto l199 - l200: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l203: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l205 + goto l208 } position++ if buffer[position] != rune('n') { - goto l205 + goto l208 } position++ - goto l199 - l205: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l208: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l206 + goto l209 } position++ if buffer[position] != rune('"') { - goto l206 + goto l209 } position++ - goto l199 - l206: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l209: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l207 + goto l210 } position++ if buffer[position] != rune('\'') { - goto l207 + goto l210 } position++ - goto l199 - l207: - position, tokenIndex = position199, tokenIndex199 + goto l202 + l210: + position, tokenIndex = position202, tokenIndex202 if buffer[position] != rune('\\') { - goto l198 + goto l201 } position++ if buffer[position] != rune('\\') { - goto l198 + goto l201 } position++ } - l199: - goto l197 - l198: - position, tokenIndex = position198, tokenIndex198 + l202: + goto l200 + l201: + position, tokenIndex = position201, tokenIndex201 } - add(ruledoublequotedstring, position196) + add(ruledoublequotedstring, position199) } return true }, /* 15 singlequotedstring <- <((!('\'' / '\\' / '\n') .) / ('\\' 'n') / ('\\' '"') / ('\\' '\'') / ('\\' '\\'))*> */ func() bool { { - position209 := position - l210: + position212 := position + l213: { - position211, tokenIndex211 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position212, tokenIndex212 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex { - position214, tokenIndex214 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex { - position215, tokenIndex215 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex if buffer[position] != rune('\'') { - goto l216 + goto l219 } position++ - goto l215 - l216: - position, tokenIndex = position215, tokenIndex215 + goto l218 + l219: + position, tokenIndex = position218, tokenIndex218 if buffer[position] != rune('\\') { + goto l220 + } + position++ + goto l218 + l220: + position, tokenIndex = position218, tokenIndex218 + if buffer[position] != rune('\n') { goto l217 } position++ - goto l215 - l217: - position, tokenIndex = position215, tokenIndex215 - if buffer[position] != rune('\n') { - goto l214 - } - position++ } - l215: - goto l213 - l214: - position, tokenIndex = position214, tokenIndex214 + l218: + goto l216 + l217: + position, tokenIndex = position217, tokenIndex217 } if !matchDot() { - goto l213 + goto l216 } - goto l212 - l213: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l218 + goto l221 } position++ if buffer[position] != rune('n') { - goto l218 + goto l221 } position++ - goto l212 - l218: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l221: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l219 + goto l222 } position++ if buffer[position] != rune('"') { - goto l219 + goto l222 } position++ - goto l212 - l219: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l222: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l220 + goto l223 } position++ if buffer[position] != rune('\'') { - goto l220 + goto l223 } position++ - goto l212 - l220: - position, tokenIndex = position212, tokenIndex212 + goto l215 + l223: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l211 + goto l214 } position++ if buffer[position] != rune('\\') { - goto l211 + goto l214 } position++ } - l212: - goto l210 - l211: - position, tokenIndex = position211, tokenIndex211 + l215: + goto l213 + l214: + position, tokenIndex = position214, tokenIndex214 } - add(rulesinglequotedstring, position209) + add(rulesinglequotedstring, position212) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position221, tokenIndex221 := position, tokenIndex + position224, tokenIndex224 := position, tokenIndex { - position222 := position + position225 := position { - position223, tokenIndex223 := position, tokenIndex + position226, tokenIndex226 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l227 + } + position++ + goto l226 + l227: + position, tokenIndex = position226, tokenIndex226 + if c := buffer[position]; c < rune('A') || c > rune('Z') { goto l224 } position++ - goto l223 - l224: - position, tokenIndex = position223, tokenIndex223 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l221 - } - position++ } - l223: - l225: + l226: + l228: { - position226, tokenIndex226 := position, tokenIndex + position229, tokenIndex229 := position, tokenIndex { - position227, tokenIndex227 := position, tokenIndex + position230, tokenIndex230 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l228 - } - position++ - goto l227 - l228: - position, tokenIndex = position227, tokenIndex227 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l229 - } - position++ - goto l227 - l229: - position, tokenIndex = position227, tokenIndex227 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l230 - } - position++ - goto l227 - l230: - position, tokenIndex = position227, tokenIndex227 - if buffer[position] != rune('_') { goto l231 } position++ - goto l227 + goto l230 l231: - position, tokenIndex = position227, tokenIndex227 + position, tokenIndex = position230, tokenIndex230 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l232 + } + position++ + goto l230 + l232: + position, tokenIndex = position230, tokenIndex230 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l233 + } + position++ + goto l230 + l233: + position, tokenIndex = position230, tokenIndex230 + if buffer[position] != rune('_') { + goto l234 + } + position++ + goto l230 + l234: + position, tokenIndex = position230, tokenIndex230 if buffer[position] != rune('-') { - goto l226 + goto l229 } position++ } - l227: - goto l225 - l226: - position, tokenIndex = position226, tokenIndex226 + l230: + goto l228 + l229: + position, tokenIndex = position229, tokenIndex229 } - add(rulefieldExpr, position222) + add(rulefieldExpr, position225) } return true - l221: - position, tokenIndex = position221, tokenIndex221 + l224: + position, tokenIndex = position224, tokenIndex224 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action40)> */ + /* 17 field <- <(<(fieldExpr / reserved)> Action42)> */ func() bool { - position232, tokenIndex232 := position, tokenIndex + position235, tokenIndex235 := position, tokenIndex { - position233 := position + position236 := position { - position234 := position + position237 := position { - position235, tokenIndex235 := position, tokenIndex + position238, tokenIndex238 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l236 + goto l239 } - goto l235 - l236: - position, tokenIndex = position235, tokenIndex235 + goto l238 + l239: + position, tokenIndex = position238, tokenIndex238 { - position237 := position + position240 := position { - position238, tokenIndex238 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex if buffer[position] != rune('_') { - goto l239 + goto l242 } position++ if buffer[position] != rune('r') { - goto l239 + goto l242 } position++ if buffer[position] != rune('o') { - goto l239 + goto l242 } position++ if buffer[position] != rune('w') { - goto l239 + goto l242 } position++ - goto l238 - l239: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l242: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l240 + goto l243 } position++ if buffer[position] != rune('c') { - goto l240 + goto l243 } position++ if buffer[position] != rune('o') { - goto l240 + goto l243 } position++ if buffer[position] != rune('l') { - goto l240 + goto l243 } position++ - goto l238 - l240: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l243: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l241 + goto l244 } position++ if buffer[position] != rune('s') { - goto l241 + goto l244 } position++ if buffer[position] != rune('t') { - goto l241 + goto l244 } position++ if buffer[position] != rune('a') { - goto l241 + goto l244 } position++ if buffer[position] != rune('r') { - goto l241 + goto l244 } position++ if buffer[position] != rune('t') { - goto l241 + goto l244 } position++ - goto l238 - l241: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l244: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l242 + goto l245 } position++ if buffer[position] != rune('e') { - goto l242 + goto l245 } position++ if buffer[position] != rune('n') { - goto l242 + goto l245 } position++ if buffer[position] != rune('d') { - goto l242 + goto l245 } position++ - goto l238 - l242: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l245: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l243 + goto l246 } position++ if buffer[position] != rune('t') { - goto l243 + goto l246 } position++ if buffer[position] != rune('i') { - goto l243 + goto l246 } position++ if buffer[position] != rune('m') { - goto l243 + goto l246 } position++ if buffer[position] != rune('e') { - goto l243 + goto l246 } position++ if buffer[position] != rune('s') { - goto l243 + goto l246 } position++ if buffer[position] != rune('t') { - goto l243 + goto l246 } position++ if buffer[position] != rune('a') { - goto l243 + goto l246 } position++ if buffer[position] != rune('m') { - goto l243 + goto l246 } position++ if buffer[position] != rune('p') { - goto l243 + goto l246 } position++ - goto l238 - l243: - position, tokenIndex = position238, tokenIndex238 + goto l241 + l246: + position, tokenIndex = position241, tokenIndex241 if buffer[position] != rune('_') { - goto l232 + goto l235 } position++ if buffer[position] != rune('f') { - goto l232 + goto l235 } position++ if buffer[position] != rune('i') { - goto l232 + goto l235 } position++ if buffer[position] != rune('e') { - goto l232 + goto l235 } position++ if buffer[position] != rune('l') { - goto l232 + goto l235 } position++ if buffer[position] != rune('d') { - goto l232 + goto l235 } position++ } - l238: - add(rulereserved, position237) + l241: + add(rulereserved, position240) } } - l235: - add(rulePegText, position234) + l238: + add(rulePegText, position237) } { - add(ruleAction40, position) + add(ruleAction42, position) } - add(rulefield, position233) + add(rulefield, position236) } return true - l232: - position, tokenIndex = position232, tokenIndex232 + l235: + position, tokenIndex = position235, tokenIndex235 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action41)> */ + /* 19 posfield <- <( Action43)> */ func() bool { - position246, tokenIndex246 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position247 := position + position250 := position { - position248 := position + position251 := position if !_rules[rulefieldExpr]() { - goto l246 + goto l249 } - add(rulePegText, position248) + add(rulePegText, position251) } { - add(ruleAction41, position) + add(ruleAction43, position) } - add(ruleposfield, position247) + add(ruleposfield, position250) } return true - l246: - position, tokenIndex = position246, tokenIndex246 + l249: + position, tokenIndex = position249, tokenIndex249 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position250, tokenIndex250 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position251 := position + position254 := position { - position252, tokenIndex252 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l256 + } + position++ + l257: + { + position258, tokenIndex258 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l258 + } + position++ + goto l257 + l258: + position, tokenIndex = position258, tokenIndex258 + } + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 + if buffer[position] != rune('0') { goto l253 } position++ - l254: - { - position255, tokenIndex255 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l255 - } - position++ - goto l254 - l255: - position, tokenIndex = position255, tokenIndex255 - } - goto l252 - l253: - position, tokenIndex = position252, tokenIndex252 - if buffer[position] != rune('0') { - goto l250 - } - position++ } - l252: - add(ruleuint, position251) + l255: + add(ruleuint, position254) } return true - l250: - position, tokenIndex = position250, tokenIndex250 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, - /* 21 uintrow <- <( Action42)> */ + /* 21 uintrow <- <( Action44)> */ nil, - /* 22 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ + /* 22 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ func() bool { - position257, tokenIndex257 := position, tokenIndex + position260, tokenIndex260 := position, tokenIndex { - position258 := position + position261 := position { - position259, tokenIndex259 := position, tokenIndex - { - position261 := position - if !_rules[ruleuint]() { - goto l260 - } - add(rulePegText, position261) - } - { - add(ruleAction43, position) - } - goto l259 - l260: - position, tokenIndex = position259, tokenIndex259 - if buffer[position] != rune('\'') { - goto l263 - } - position++ + position262, tokenIndex262 := position, tokenIndex { position264 := position - if !_rules[rulesinglequotedstring]() { + if !_rules[ruleuint]() { goto l263 } add(rulePegText, position264) } - if buffer[position] != rune('\'') { - goto l263 - } - position++ - { - add(ruleAction44, position) - } - goto l259 - l263: - position, tokenIndex = position259, tokenIndex259 - if buffer[position] != rune('"') { - goto l257 - } - position++ - { - position266 := position - if !_rules[ruledoublequotedstring]() { - goto l257 - } - add(rulePegText, position266) - } - if buffer[position] != rune('"') { - goto l257 - } - position++ { add(ruleAction45, position) } + goto l262 + l263: + position, tokenIndex = position262, tokenIndex262 + if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + position267 := position + if !_rules[rulesinglequotedstring]() { + goto l266 + } + add(rulePegText, position267) + } + if buffer[position] != rune('\'') { + goto l266 + } + position++ + { + add(ruleAction46, position) + } + goto l262 + l266: + position, tokenIndex = position262, tokenIndex262 + if buffer[position] != rune('"') { + goto l260 + } + position++ + { + position269 := position + if !_rules[ruledoublequotedstring]() { + goto l260 + } + add(rulePegText, position269) + } + if buffer[position] != rune('"') { + goto l260 + } + position++ + { + add(ruleAction47, position) + } } - l259: - add(rulecol, position258) + l262: + add(rulecol, position261) } return true - l257: - position, tokenIndex = position257, tokenIndex257 + l260: + position, tokenIndex = position260, tokenIndex260 return false }, - /* 23 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ + /* 23 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ nil, /* 24 open <- <('(' sp)> */ func() bool { - position269, tokenIndex269 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex { - position270 := position + position273 := position if buffer[position] != rune('(') { - goto l269 + goto l272 } position++ if !_rules[rulesp]() { - goto l269 + goto l272 } - add(ruleopen, position270) + add(ruleopen, position273) } return true - l269: - position, tokenIndex = position269, tokenIndex269 + l272: + position, tokenIndex = position272, tokenIndex272 return false }, /* 25 close <- <(')' sp)> */ func() bool { - position271, tokenIndex271 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position272 := position + position275 := position if buffer[position] != rune(')') { - goto l271 + goto l274 } position++ if !_rules[rulesp]() { - goto l271 + goto l274 } - add(ruleclose, position272) + add(ruleclose, position275) } return true - l271: - position, tokenIndex = position271, tokenIndex271 + l274: + position, tokenIndex = position274, tokenIndex274 return false }, /* 26 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position274 := position - l275: + position277 := position + l278: { - position276, tokenIndex276 := position, tokenIndex + position279, tokenIndex279 := position, tokenIndex { - position277, tokenIndex277 := position, tokenIndex + position280, tokenIndex280 := position, tokenIndex if buffer[position] != rune(' ') { - goto l278 + goto l281 } position++ - goto l277 - l278: - position, tokenIndex = position277, tokenIndex277 + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 if buffer[position] != rune('\t') { + goto l282 + } + position++ + goto l280 + l282: + position, tokenIndex = position280, tokenIndex280 + if buffer[position] != rune('\n') { goto l279 } position++ - goto l277 - l279: - position, tokenIndex = position277, tokenIndex277 - if buffer[position] != rune('\n') { - goto l276 - } - position++ } - l277: - goto l275 - l276: - position, tokenIndex = position276, tokenIndex276 + l280: + goto l278 + l279: + position, tokenIndex = position279, tokenIndex279 } - add(rulesp, position274) + add(rulesp, position277) } return true }, /* 27 comma <- <(sp ',' sp)> */ func() bool { - position280, tokenIndex280 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position281 := position + position284 := position if !_rules[rulesp]() { - goto l280 + goto l283 } if buffer[position] != rune(',') { - goto l280 + goto l283 } position++ if !_rules[rulesp]() { - goto l280 + goto l283 } - add(rulecomma, position281) + add(rulecomma, position284) } return true - l280: - position, tokenIndex = position280, tokenIndex280 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, /* 28 lbrack <- <('[' sp)> */ @@ -2740,139 +2792,139 @@ func (p *PQL) Init() { nil, /* 31 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position285, tokenIndex285 := position, tokenIndex + position288, tokenIndex288 := position, tokenIndex { - position286 := position + position289 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('-') { - goto l285 + goto l288 } position++ { - position287, tokenIndex287 := position, tokenIndex + position290, tokenIndex290 := position, tokenIndex if buffer[position] != rune('0') { + goto l291 + } + position++ + goto l290 + l291: + position, tokenIndex = position290, tokenIndex290 + if buffer[position] != rune('1') { goto l288 } position++ - goto l287 - l288: - position, tokenIndex = position287, tokenIndex287 - if buffer[position] != rune('1') { - goto l285 - } - position++ } - l287: + l290: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('-') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune('T') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if buffer[position] != rune(':') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l285 + goto l288 } position++ - add(ruletimestampbasicfmt, position286) + add(ruletimestampbasicfmt, position289) } return true - l285: - position, tokenIndex = position285, tokenIndex285 + l288: + position, tokenIndex = position288, tokenIndex288 return false }, /* 32 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position289, tokenIndex289 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex { - position290 := position + position293 := position { - position291, tokenIndex291 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex if buffer[position] != rune('"') { - goto l292 + goto l295 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l292 + goto l295 } if buffer[position] != rune('"') { + goto l295 + } + position++ + goto l294 + l295: + position, tokenIndex = position294, tokenIndex294 + if buffer[position] != rune('\'') { + goto l296 + } + position++ + if !_rules[ruletimestampbasicfmt]() { + goto l296 + } + if buffer[position] != rune('\'') { + goto l296 + } + position++ + goto l294 + l296: + position, tokenIndex = position294, tokenIndex294 + if !_rules[ruletimestampbasicfmt]() { goto l292 } - position++ - goto l291 - l292: - position, tokenIndex = position291, tokenIndex291 - if buffer[position] != rune('\'') { - goto l293 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l293 - } - if buffer[position] != rune('\'') { - goto l293 - } - position++ - goto l291 - l293: - position, tokenIndex = position291, tokenIndex291 - if !_rules[ruletimestampbasicfmt]() { - goto l289 - } } - l291: - add(ruletimestampfmt, position290) + l294: + add(ruletimestampfmt, position293) } return true - l289: - position, tokenIndex = position289, tokenIndex289 + l292: + position, tokenIndex = position292, tokenIndex292 return false }, - /* 33 timestamp <- <( Action49)> */ + /* 33 timestamp <- <( Action51)> */ nil, /* 35 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2894,86 +2946,90 @@ func (p *PQL) Init() { nil, /* 44 Action9 <- <{p.endCall()}> */ nil, - /* 45 Action10 <- <{p.startCall("TopN")}> */ + /* 45 Action10 <- <{p.startCall("Store")}> */ nil, /* 46 Action11 <- <{p.endCall()}> */ nil, - /* 47 Action12 <- <{p.startCall("Range")}> */ + /* 47 Action12 <- <{p.startCall("TopN")}> */ nil, /* 48 Action13 <- <{p.endCall()}> */ nil, + /* 49 Action14 <- <{p.startCall("Range")}> */ nil, - /* 50 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 50 Action15 <- <{p.endCall()}> */ nil, - /* 51 Action15 <- <{ p.endCall() }> */ nil, - /* 52 Action16 <- <{ p.addBTWN() }> */ + /* 52 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 53 Action17 <- <{ p.addLTE() }> */ + /* 53 Action17 <- <{ p.endCall() }> */ nil, - /* 54 Action18 <- <{ p.addGTE() }> */ + /* 54 Action18 <- <{ p.addBTWN() }> */ nil, - /* 55 Action19 <- <{ p.addEQ() }> */ + /* 55 Action19 <- <{ p.addLTE() }> */ nil, - /* 56 Action20 <- <{ p.addNEQ() }> */ + /* 56 Action20 <- <{ p.addGTE() }> */ nil, - /* 57 Action21 <- <{ p.addLT() }> */ + /* 57 Action21 <- <{ p.addEQ() }> */ nil, - /* 58 Action22 <- <{ p.addGT() }> */ + /* 58 Action22 <- <{ p.addNEQ() }> */ nil, - /* 59 Action23 <- <{p.startConditional()}> */ + /* 59 Action23 <- <{ p.addLT() }> */ nil, - /* 60 Action24 <- <{p.endConditional()}> */ + /* 60 Action24 <- <{ p.addGT() }> */ nil, - /* 61 Action25 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action25 <- <{p.startConditional()}> */ nil, - /* 62 Action26 <- <{p.condAdd(buffer[begin:end])}> */ + /* 62 Action26 <- <{p.endConditional()}> */ nil, /* 63 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action28 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 64 Action28 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 65 Action29 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 65 Action29 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 66 Action30 <- <{ p.startList() }> */ + /* 66 Action30 <- <{p.addPosStr("_start", buffer[begin:end])}> */ nil, - /* 67 Action31 <- <{ p.endList() }> */ + /* 67 Action31 <- <{p.addPosStr("_end", buffer[begin:end])}> */ nil, - /* 68 Action32 <- <{ p.addVal(nil) }> */ + /* 68 Action32 <- <{ p.startList() }> */ nil, - /* 69 Action33 <- <{ p.addVal(true) }> */ + /* 69 Action33 <- <{ p.endList() }> */ nil, - /* 70 Action34 <- <{ p.addVal(false) }> */ + /* 70 Action34 <- <{ p.addVal(nil) }> */ nil, - /* 71 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 71 Action35 <- <{ p.addVal(true) }> */ nil, - /* 72 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 72 Action36 <- <{ p.addVal(false) }> */ nil, - /* 73 Action37 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 74 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 74 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, /* 75 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action40 <- <{ p.addField(buffer[begin:end]) }> */ + /* 76 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action41 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 77 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 78 Action42 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 78 Action42 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 79 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 79 Action43 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 80 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 80 Action44 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 81 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 81 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 82 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 82 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 83 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 84 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 84 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 85 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 85 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 86 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 87 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules From 338f69b71d2b4c9ef131199b882a64f990c056ed Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 26 Sep 2018 14:19:51 -0500 Subject: [PATCH 40/45] replace switch with simplified if statement --- executor.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 53a39d04e..9c52573b8 100644 --- a/executor.go +++ b/executor.go @@ -1226,11 +1226,7 @@ func (e *executor) executeSetRow(ctx context.Context, index string, c *pql.Call, if field == nil { return false, ErrFieldNotFound } - - switch field.Type() { - case FieldTypeSet: - // These field types support SetRow(). - default: + if field.Type() != FieldTypeSet { return false, fmt.Errorf("SetRow() is not supported on %s field types", field.Type()) } @@ -1298,11 +1294,11 @@ func (e *executor) executeSetRowShard(ctx context.Context, index string, c *pql. return false, errors.Wrapf(err, "creating fragment: %d", shard) } } - cleared, err := fragment.setRow(src, rowID) + set, err := fragment.setRow(src, rowID) if err != nil { return false, errors.Wrapf(err, "setting row %d on view %s shard %d", rowID, viewStandard, shard) } - changed = changed || cleared + changed = changed || set return changed, nil } From a6190f5cfa13e8cf901bc7dd124f5120292f72e5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 26 Sep 2018 14:30:08 -0500 Subject: [PATCH 41/45] Store() docs --- docs/query-language.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/query-language.md b/docs/query-language.md index c2319cabd..5030db3cc 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -266,6 +266,32 @@ ClearRow(stargazer=1) This represents removing the relationship between the user with id=1 and all repositories. +#### Store + +**Spec:** + +``` +Store(, =) +``` + +**Description:** + +`Store` writes the results of to the specified row. If the row already exists, it will be replaced. The destination field must be of field type `set`. + +**Result Type:** boolean + +Upon success, this method always returns `true`. A future version of Pilosa may use this boolean result to indicate whether or not the data in the destination row was changed by the `Store` call. + +**Examples:** + +Store the contents of stargazer row 1 into stargazer row 2: +```request +Store(Row(stargazer=1), stargazer=2) +``` +```response +{"results":[true]} +``` + ### Read Operations #### Row From 16811bc04ce5d251adfc59e8a649f5d036445bdd Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 4 Oct 2018 10:43:09 -0500 Subject: [PATCH 42/45] add a "store intersect" example to the Store() docs --- docs/query-language.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/query-language.md b/docs/query-language.md index 5030db3cc..9bc231e6f 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -292,6 +292,14 @@ Store(Row(stargazer=1), stargazer=2) {"results":[true]} ``` +Store the results of the intersection of stargazer rows 10 and 11 into stargazer row 20. +```request +Store(Intersect(Row(stargazer=10), Row(stargazer=11)), stargazer=20) +``` +```response +{"results":[true]} +``` + ### Read Operations #### Row From b6981930b142fd98bad4a17500709994f9d59baa Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 5 Oct 2018 16:47:53 +0300 Subject: [PATCH 43/45] ColumnAttrsSet omit empty ID --- pilosa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pilosa.go b/pilosa.go index 7378eb628..9798ff315 100644 --- a/pilosa.go +++ b/pilosa.go @@ -117,7 +117,7 @@ var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { - ID uint64 `json:"id"` + ID uint64 `json:"id,omitempty"` Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } From b617db84eb6be6fd590a8c2e57d300cbc8dd34d1 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 5 Oct 2018 17:44:02 +0300 Subject: [PATCH 44/45] gfmt'ed --- executor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 235000938..76f4d7b65 100644 --- a/executor.go +++ b/executor.go @@ -1766,10 +1766,10 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu } var translateCallCol = map[string]struct{}{ - "Set": struct{}{}, - "Clear": struct{}{}, - "Row": struct{}{}, - "SetColumnAttrs": struct{}{}, + "Set": {}, + "Clear": {}, + "Row": {}, + "SetColumnAttrs": {}, } func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { From 051b71e540f4d53e9a4b47ab8967c0f1411f458a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 4 Oct 2018 10:31:49 -0500 Subject: [PATCH 45/45] ensure a Range() query with field keys is handled correctly --- executor.go | 7 ++++--- executor_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 9c52573b8..1907147a6 100644 --- a/executor.go +++ b/executor.go @@ -1798,16 +1798,17 @@ func (e *executor) mapperLocal(ctx context.Context, shards []uint64, mapFn mapFu func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { var colKey, rowKey, fieldName string - if c.Name == "Set" || c.Name == "Clear" || c.Name == "Row" { + switch c.Name { + case "Set", "Clear", "Row", "Range": // Positional args in new PQL syntax require special handling here. colKey = "_" + columnLabel fieldName, _ = c.FieldArg() rowKey = fieldName - } else if c.Name == "SetRowAttrs" { + case "SetRowAttrs": // Positional args in new PQL syntax require special handling here. rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") - } else { + default: colKey = "col" fieldName = callArgString(c, "field") rowKey = "row" diff --git a/executor_test.go b/executor_test.go index 8d3786cd5..d5cd6b770 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1020,6 +1020,58 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } +// Ensure a range query with keys can be executed. +func TestExecutor_Execute_Range_WithKeys(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + // Create index. + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), pilosa.OptFieldKeys()); err != nil { + t.Fatal(err) + } + + // Set columns. + cc := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) + + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00) + ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + t.Fatal(err) + } + + t.Run("Standard", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Clear( 2, f="foo")`}); err != nil { + t.Fatal(err) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) +} + // Ensure a Range(bsiGroup) query can be executed. func TestExecutor_Execute_BSIGroupRange(t *testing.T) { c := test.MustRunCluster(t, 1)