From c7e0168729e741c4c39f756cf11d6b3291be5f01 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Mon, 8 May 2017 17:59:18 -0500 Subject: [PATCH 01/15] Add syntax hints tooltip to WebUI --- webui/assets/style.css | 40 +++++++++++++++++++++++++++++++++++++++- webui/index.html | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/webui/assets/style.css b/webui/assets/style.css index 515335ef9..5fec5b5c6 100644 --- a/webui/assets/style.css +++ b/webui/assets/style.css @@ -295,4 +295,42 @@ td{ .expand { text-align: center; -} \ No newline at end of file +} + +.query h2 { + display: inline-block; +} + +.query-tooltip { + position: relative; + display: inline; + color: #1db598; + margin-left: 5px; +} + +.query-tooltip-content { + visibility: hidden; + background-color: rgb(250, 250, 250); + border: solid 1.5px #e4eff4; + color: #102445; + border-radius: 2px; + padding: 15px; + margin-bottom: 15px; + + /* Position the tooltip */ + position: absolute; + /* + left: 423px; + top: 100px; + */ + z-index: 1; +} + +.query-tooltip:hover+.query-tooltip-content { + visibility: visible; +} + +.code{ + font-family: monospace; +} + diff --git a/webui/index.html b/webui/index.html index 8422d4c45..53feb0b7a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -34,10 +34,39 @@
-

Query

+

Query

? +
+
PQL
+
+ SetBit(frame=foo, row=0, column=0)
+ ClearBit(frame=foo, row=0, column=0)
+ SetRowAttrs(frame=foo, row=0, color="blue")
+ SetColumnAttrs(frame=foo, column=0, shape="circle")
+ Bitmap(frame=foo, row=0)
+ Range(frame=foo, row=0, start="2010-01-01T00:00", end="2017-03-02T03:00")
+ Count(<BITMAP_CALL>)
+ TopN([BITMAP_CALL], frame=foo, n=20)
+ Union([BITMAP_CALL, ...])
+ Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
+ Difference(<BITMAP_CALL>, <BITMAP_CALL>) +
+
+
Special commands
+
+ :create index test [columnLabel=column]
+ :use index test
+ :create frame foo [rowLabel=row]
+ :delete index test
+ :delete frame foo +
+
+ <tab>: autocomplete
+ <up>/<down>: history
+
-
- +
+
+
From 62d1c20eefe61bfc63334f16e160593c4d654a05 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 9 May 2017 17:52:29 -0500 Subject: [PATCH 02/15] Style fixes --- webui/assets/style.css | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/webui/assets/style.css b/webui/assets/style.css index 5fec5b5c6..3a5bcaf39 100644 --- a/webui/assets/style.css +++ b/webui/assets/style.css @@ -304,12 +304,15 @@ td{ .query-tooltip { position: relative; display: inline; - color: #1db598; + color: #000; margin-left: 5px; } +.query-tooltip:hover { + color: #000; +} + .query-tooltip-content { - visibility: hidden; background-color: rgb(250, 250, 250); border: solid 1.5px #e4eff4; color: #102445; @@ -317,16 +320,18 @@ td{ padding: 15px; margin-bottom: 15px; - /* Position the tooltip */ position: absolute; - /* - left: 423px; - top: 100px; - */ + left: 80px; + top: -30px; z-index: 1; } -.query-tooltip:hover+.query-tooltip-content { +.query-tooltip-container { + position: relative; + visibility: hidden; +} + +.query-tooltip:hover+.query-tooltip-container{ visibility: visible; } From 43dac449b0eae1518b6b9f6590e22fe3808998b7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 9 May 2017 17:53:25 -0500 Subject: [PATCH 03/15] Fix broken html structure --- webui/index.html | 85 +++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/webui/index.html b/webui/index.html index 53feb0b7a..fbab04911 100644 --- a/webui/index.html +++ b/webui/index.html @@ -34,58 +34,61 @@
-

Query

? -
-
PQL
-
- SetBit(frame=foo, row=0, column=0)
- ClearBit(frame=foo, row=0, column=0)
- SetRowAttrs(frame=foo, row=0, color="blue")
- SetColumnAttrs(frame=foo, column=0, shape="circle")
- Bitmap(frame=foo, row=0)
- Range(frame=foo, row=0, start="2010-01-01T00:00", end="2017-03-02T03:00")
- Count(<BITMAP_CALL>)
- TopN([BITMAP_CALL], frame=foo, n=20)
- Union([BITMAP_CALL, ...])
- Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
- Difference(<BITMAP_CALL>, <BITMAP_CALL>) +

Query

+ ? +
+
+
PQL
+
+ SetBit(frame=foo, row=0, column=0)
+ ClearBit(frame=foo, row=0, column=0)
+ SetRowAttrs(frame=foo, row=0, color="blue")
+ SetColumnAttrs(frame=foo, column=0, shape="circle")
+ Bitmap(frame=foo, row=0)
+ Range(frame=foo, row=0, start="2010-01", end="2017-03")
+ Count(<BITMAP_CALL>)
+ TopN([BITMAP_CALL], frame=foo, n=20)
+ Union([BITMAP_CALL, ...])
+ Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
+ Difference(<BITMAP_CALL>, <BITMAP_CALL>) +
+
+
Special commands
+
+ :create index test [columnLabel=column]
+ :use index test
+ :create frame foo [rowLabel=row]
+ :delete index test
+ :delete frame foo +
+
+ <tab>: autocomplete
+ <up>/<down>: history
-
-
Special commands
-
- :create index test [columnLabel=column]
- :use index test
- :create frame foo [rowLabel=row]
- :delete index test
- :delete frame foo -
-
- <tab>: autocomplete
- <up>/<down>: history
-
-
-
- -     - +
+
+ +     + +
-
-
+
-

Output

-
+

Output

+
+ + +
-
-
From 742ca5d18f2f16ebf7d3fbb08fdcb4ba39b6105d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 9 May 2017 17:53:47 -0500 Subject: [PATCH 04/15] Update autocomplete list --- webui/assets/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webui/assets/main.js b/webui/assets/main.js index e18c562fb..887181338 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -20,7 +20,8 @@ class REPL { // keyword: length of substring that comes after cursor "SetBit()": 1, "ClearBit()": 1, - "SetBitmapAttrs()": 1, + "SetRowAttrs()": 1, + "SetColumnAttrs()": 1, "Bitmap()": 1, "Union()": 1, "Intersect()": 1, From 3eff46b07abe6fd0238a5d8ed3cba2eec56b4198 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 9 May 2017 21:22:45 -0500 Subject: [PATCH 05/15] Use default row/column labels --- webui/index.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/webui/index.html b/webui/index.html index fbab04911..09e39d17a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -40,12 +40,12 @@
PQL
- SetBit(frame=foo, row=0, column=0)
- ClearBit(frame=foo, row=0, column=0)
- SetRowAttrs(frame=foo, row=0, color="blue")
- SetColumnAttrs(frame=foo, column=0, shape="circle")
- Bitmap(frame=foo, row=0)
- Range(frame=foo, row=0, start="2010-01", end="2017-03")
+ SetBit(frame=foo, rowID=0, columnID=0)
+ ClearBit(frame=foo, rowID=0, columnID=0)
+ SetRowAttrs(frame=foo, rowID=0, color="blue")
+ SetColumnAttrs(frame=foo, columnID=0, shape="circle")
+ Bitmap(frame=foo, rowID=0)
+ Range(frame=foo, rowID=0, start="2010-01", end="2017-03")
Count(<BITMAP_CALL>)
TopN([BITMAP_CALL], frame=foo, n=20)
Union([BITMAP_CALL, ...])
From 86c254bdbd47bcd4e1e898e1b87b6fdc2ab0a80c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 10 May 2017 13:57:41 -0500 Subject: [PATCH 06/15] Keep ldflags in one place --- Makefile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 64eca7400..e8ab1e90d 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa BUILD_TIME=`date -u +%FT%T%z` -LDFLAGS=-ldflags "-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) -X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" +LDFLAGS="-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) -X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" default: test pilosa @@ -31,14 +31,14 @@ test: vendor go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) pilosa: vendor - go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa crossbuild: vendor mkdir -p build/pilosa-$(IDENTIFIER) make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" install: vendor - go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa .protoc-gen-gofast: vendor ifndef PROTOC @@ -61,7 +61,5 @@ ifndef STATIK endif docker: - docker build -t "pilosa:$(VERSION)" \ - --build-arg ldflags="-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) \ - -X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" . + docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . @echo "Created image: pilosa:$(VERSION)" From 6c11746ac8391e2597ba7db186c2c528416c9d67 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 10 May 2017 13:59:34 -0500 Subject: [PATCH 07/15] Fix vendor target as to not call "glide install" on every invocation --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e8ab1e90d..b318f52ea 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,10 @@ ifndef GLIDE curl https://glide.sh/get | sh endif -vendor: glide glide.yaml +$(GLIDE): + make glide + +vendor: $(GLIDE) glide.yaml glide install glide.lock: glide glide.yaml From 3a1bcc353e214d064d8284718315474ee6ff7433 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 10 May 2017 14:04:09 -0500 Subject: [PATCH 08/15] Add release target in Makefile --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b318f52ea..f614068d4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release GLIDE := $(shell command -v glide 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -39,6 +39,13 @@ pilosa: vendor crossbuild: vendor mkdir -p build/pilosa-$(IDENTIFIER) make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" + cp {LICENSE,README.md} build/pilosa-$(IDENTIFIER) + tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ + @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + +release: + make crossbuild GOOS=linux GOARCH=amd64 + make crossbuild GOOS=darwin GOARCH=amd64 install: vendor go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From 76354c69e9ee84a9a7a78758fc28ab0614fba18a Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Fri, 12 May 2017 16:48:47 -0400 Subject: [PATCH 09/15] provide details on glide checksum mistmach before: Step 6/13 : RUN wget ${GLIDE} -O /go/glide.tar.gz -q && tar xf /go/glide.tar.gz && mv /go/linux-amd64/glide /go/bin && [ "$(sha256sum /go/bin/glide | cut -d' ' -f1)" = "$GLIDE_HASH" ] && cd /go/src/github.com/pilosa/pilosa && make vendor && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa ---> Running in e2a6fcdf5930 The command '/bin/sh -c wget ${GLIDE} -O /go/glide.tar.gz -q && tar xf /go/glide.tar.gz && mv /go/linux-amd64/glide /go/bin && [ "$(sha256sum /go/bin/glide | cut -d' ' -f1)" = "$GLIDE_HASH" ] && cd /go/src/github.com/pilosa/pilosa && make vendor && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa' returned a non-zero code: 1 make: *** [docker] Error 1 after: Step 6/13 : RUN wget ${GLIDE} -O /go/glide.tar.gz -q && tar xf /go/glide.tar.gz && mv /go/linux-amd64/glide /go/bin && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - && cd /go/src/github.com/pilosa/pilosa && make vendor && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa ---> Running in 5aaaab387641 shasum: WARNING: 1 computed checksum did NOT match /go/bin/glide: FAILED The command '/bin/sh -c wget ${GLIDE} -O /go/glide.tar.gz -q && tar xf /go/glide.tar.gz && mv /go/linux-amd64/glide /go/bin && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - && cd /go/src/github.com/pilosa/pilosa && make vendor && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa' returned a non-zero code: 1 make: *** [docker] Error 1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e32437ef7..d49baaf8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ COPY . /go/src/github.com/pilosa/pilosa RUN wget ${GLIDE} -O /go/glide.tar.gz -q \ && tar xf /go/glide.tar.gz \ && mv /go/linux-amd64/glide /go/bin \ - && [ "$(sha256sum /go/bin/glide | cut -d' ' -f1)" = "$GLIDE_HASH" ] \ + && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - \ && cd /go/src/github.com/pilosa/pilosa \ && make vendor \ && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa From e20b9075e26d675812588e2e6bb0168350b3b519 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 11:17:48 -0500 Subject: [PATCH 10/15] #413 set standard view as default for export command --- client.go | 9 ++++++--- cmd/export.go | 1 + ctl/export.go | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/client.go b/client.go index 96e725876..22ea7d59c 100644 --- a/client.go +++ b/client.go @@ -389,11 +389,13 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired + } else if view == "" { + view = ViewStandard } // Retrieve a list of nodes that own the slice. @@ -407,7 +409,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) continue } else { @@ -419,7 +421,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 } // exportNode copies a CSV export from a node to w. -func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { // Create URL. u := url.URL{ Scheme: "http", @@ -428,6 +430,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame str RawQuery: url.Values{ "index": {index}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } diff --git a/cmd/export.go b/cmd/export.go index 310e3f05c..3572ffb7c 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -53,6 +53,7 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") + flags.StringVarP(&Exporter.View, "view", "v", "", "View to export into - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") return exportCmd diff --git a/ctl/export.go b/ctl/export.go index 8f67b7896..3ad665551 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -31,7 +31,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - + View string // Filename to export to. Path string @@ -55,6 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired + } else if cmd.View == "" { + cmd.View = pilosa.ViewStandard } // Use output file, if specified. @@ -85,7 +87,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { return err } } From c8d03ff62543b4073010a287459a8d7bf659a0e1 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 15 May 2017 19:35:04 +0300 Subject: [PATCH 11/15] added goveralls --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d5c27fd2..937416af0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: go go: - - 1.7.x + - 1.7 + - 1.8 - master addons: apt: @@ -8,7 +9,10 @@ addons: - sourceline: 'ppa:masterminds/glide' packages: - glide -script: make test +before_install: + - go get github.com/mattn/goveralls +script: + - make vendor && $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" notifications: slack: secure: k0m8IqQIjb8L5b1plR9+w4hGo5iWlGtrR9nxvPsHL7gShwez74dpHCmFUCjR/uJoZa5Xq6849IWmiBB71ZHE/wNVN60HpMeui9VKkYvUDVpgBuqoIo969ySPDyQo4jNsk0wRRv/dG5/wXo7ArY8qOp5UDjdVuEf4nEbDlBtKjo0A2W8qXBTdZ5EJ3VeP5nrtrT7knuWxXgeUyVmLkpnUaLgkY7icvH6evqkftfcb56n3l5ulKtlQkI/ij2XCUK+5xtNB9m0e1QbiqIFS7sjwlSly/GCstPVnTEp4oHNHE7LtS6TJ8lgRV90Qy6LPtROjbSdfVjhJ5cYKus8DdddSfcsm2qgHCHQThyYtVAfOXaFPYPb50WXEu3HOckHk1tYChfkdxwEGdn/cN8/xxa7Khd7T/+SaS1B4CPxOFp0ie+rD4lF6IhApmpr/UwtiW6srHcPB04lF5mioYWmyS2eyGKTzSZR82nifg5KYUgBL1FjywQDYb8t9QaAn/6svu08k4H29TBnYpOQWREb38pYmEnMtRqiRmbfQL6IyV8405TJkl/PZmIElX6IQGjf53M1KoFdCUuTL1we6rh8gen5zRVUwmM6BJxkVt4NJ2wGkrXg/HD3feYCMrv+DY3lfDN7tADx90lPb+ukmanB8oSIQHLuLCJ2XcFixEC3QSvVlnH0= From 0b8d935f1cd57d1afa675528a749fb491cd363f0 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 13 May 2017 15:35:17 -0600 Subject: [PATCH 12/15] Add inverse TopN() support. The TopN() call now supports an `inverse` boolean argument to specify if the call should operate on the standard view or the inverse view. --- executor.go | 13 +++++++--- executor_test.go | 62 +++++++++++++++++++++++++++++++++--------------- pql/ast.go | 12 ++++++---- 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/executor.go b/executor.go index 86d333b16..e6669b5e1 100644 --- a/executor.go +++ b/executor.go @@ -279,7 +279,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - rowIDs, _, err := c.UintSliceArg("ids") + idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } @@ -296,7 +296,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote { + if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote { return pairs, nil } // Only the original caller should refetch the full counts. @@ -344,6 +344,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) + inverse, _ := c.Args["inverse"].(bool) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -380,7 +381,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca frame = DefaultFrame } - f := e.Holder.Fragment(index, frame, ViewStandard, slice) + // Determine view. + view := ViewStandard + if inverse { + view = ViewInverse + } + + f := e.Holder.Fragment(index, frame, view, slice) if f == nil { return nil, nil } diff --git a/executor_test.go b/executor_test.go index 8ae53b0a8..64e47c935 100644 --- a/executor_test.go +++ b/executor_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "reflect" + "strconv" "strings" "testing" @@ -274,32 +275,55 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { func TestExecutor_Execute_TopN(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() + e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0) + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=1) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, rowID=10, columnID=0) + SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, rowID=0, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - // Execute query. - e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) - } + t.Run("Standard", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("Inverse", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + {ID: SliceWidth, Count: 3}, + {ID: 0, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := MustOpenHolder() diff --git a/pql/ast.go b/pql/ast.go index 19b5381ed..cf404ad12 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -184,16 +184,20 @@ func (c *Call) String() string { // SupportsInverse indicates that the call may be on an inverse frame. func (c *Call) SupportsInverse() bool { - if c.Name == "Bitmap" { - return true - } - return false + return c.Name == "Bitmap" || c.Name == "TopN" } // IsInverse specifies if the call is for an inverse view. // Return defaults to false unless absolutely sure of inversion. func (c *Call) IsInverse(rowLabel, columnLabel string) bool { if c.SupportsInverse() { + // Top-n has an explicit inverse flag. + if c.Name == "TopN" { + inverse, _ := c.Args["inverse"].(bool) + return inverse + } + + // Bitmap calls use the row/column labels to determine whether inverse. _, rowOK, rowErr := c.UintArg(rowLabel) _, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { From b635bdf970431a0f375bbcc96a508c1fe066ebea Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 12:54:49 -0500 Subject: [PATCH 13/15] #413 add tests and remove redundant word --- client.go | 4 ++-- cmd/export.go | 6 +++--- cmd/export_test.go | 8 ++++++++ ctl/export.go | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/client.go b/client.go index 22ea7d59c..323eef9c3 100644 --- a/client.go +++ b/client.go @@ -394,8 +394,8 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if view == "" { - view = ViewStandard + } else if view != ViewInverse || view != ViewStandard { + return ErrInvalidView } // Retrieve a list of nodes that own the slice. diff --git a/cmd/export.go b/cmd/export.go index 3572ffb7c..e55f84e59 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -51,9 +51,9 @@ The file does not contain any headers. flags := exportCmd.Flags() flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") - flags.StringVarP(&Exporter.View, "view", "v", "", "View to export into - default standard") + flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") + flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") return exportCmd diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..2d01cf4d6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -44,6 +44,7 @@ frame = "f1" v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.View, "standard") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, @@ -51,3 +52,10 @@ frame = "f1" } executeDry(t, tests) } + +func TestExportInvalidView(t *testing.T) { + output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test") + if !strings.Contains(err.Error(), "invalid view") { + t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output) + } +} diff --git a/ctl/export.go b/ctl/export.go index 3ad665551..8b2d6cd35 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -55,8 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if cmd.View == "" { - cmd.View = pilosa.ViewStandard + } else if cmd.View != pilosa.ViewStandard || cmd.View != pilosa.ViewInverse { + return pilosa.ErrInvalidView } // Use output file, if specified. From 384329bc911850d156428368dc22e2edb1df773b Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 13:33:44 -0500 Subject: [PATCH 14/15] fix view always return error --- client.go | 10 +++++++++- ctl/export.go | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/client.go b/client.go index 323eef9c3..ef7a28a72 100644 --- a/client.go +++ b/client.go @@ -394,7 +394,15 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if view != ViewInverse || view != ViewStandard { + } + foundView := false + for _, v := range []string{ViewInverse, ViewStandard} { + if view == v { + foundView = true + break + } + } + if foundView == false { return ErrInvalidView } diff --git a/ctl/export.go b/ctl/export.go index 8b2d6cd35..63266a8d6 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -49,13 +49,22 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand // Run executes the export. func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) + foundView := false // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if cmd.View != pilosa.ViewStandard || cmd.View != pilosa.ViewInverse { + } + + for _, v := range []string{pilosa.ViewInverse, pilosa.ViewStandard} { + if cmd.View == v { + foundView = true + break + } + } + if foundView == false { return pilosa.ErrInvalidView } From 5de9a7d0d0264667689c34039d69ddf52204a6b8 Mon Sep 17 00:00:00 2001 From: Linh Vo Date: Mon, 15 May 2017 14:56:47 -0500 Subject: [PATCH 15/15] refactor view validator --- client.go | 10 +--------- ctl/export.go | 11 +---------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/client.go b/client.go index ef7a28a72..058d6d836 100644 --- a/client.go +++ b/client.go @@ -394,15 +394,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } - foundView := false - for _, v := range []string{ViewInverse, ViewStandard} { - if view == v { - foundView = true - break - } - } - if foundView == false { + } else if !(view == ViewStandard || view == ViewInverse) { return ErrInvalidView } diff --git a/ctl/export.go b/ctl/export.go index 63266a8d6..7da310623 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -49,22 +49,13 @@ func NewExportCommand(stdin io.Reader, stdout, stderr io.Writer) *ExportCommand // Run executes the export. func (cmd *ExportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - foundView := false // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } - - for _, v := range []string{pilosa.ViewInverse, pilosa.ViewStandard} { - if cmd.View == v { - foundView = true - break - } - } - if foundView == false { + } else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) { return pilosa.ErrInvalidView }