diff --git a/.travis.yml b/.travis.yml index 4d5c27fd2..937416af0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: go go: - - 1.7.x + - 1.7 + - 1.8 - master addons: apt: @@ -8,7 +9,10 @@ addons: - sourceline: 'ppa:masterminds/glide' packages: - glide -script: make test +before_install: + - go get github.com/mattn/goveralls +script: + - make vendor && $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" notifications: slack: secure: k0m8IqQIjb8L5b1plR9+w4hGo5iWlGtrR9nxvPsHL7gShwez74dpHCmFUCjR/uJoZa5Xq6849IWmiBB71ZHE/wNVN60HpMeui9VKkYvUDVpgBuqoIo969ySPDyQo4jNsk0wRRv/dG5/wXo7ArY8qOp5UDjdVuEf4nEbDlBtKjo0A2W8qXBTdZ5EJ3VeP5nrtrT7knuWxXgeUyVmLkpnUaLgkY7icvH6evqkftfcb56n3l5ulKtlQkI/ij2XCUK+5xtNB9m0e1QbiqIFS7sjwlSly/GCstPVnTEp4oHNHE7LtS6TJ8lgRV90Qy6LPtROjbSdfVjhJ5cYKus8DdddSfcsm2qgHCHQThyYtVAfOXaFPYPb50WXEu3HOckHk1tYChfkdxwEGdn/cN8/xxa7Khd7T/+SaS1B4CPxOFp0ie+rD4lF6IhApmpr/UwtiW6srHcPB04lF5mioYWmyS2eyGKTzSZR82nifg5KYUgBL1FjywQDYb8t9QaAn/6svu08k4H29TBnYpOQWREb38pYmEnMtRqiRmbfQL6IyV8405TJkl/PZmIElX6IQGjf53M1KoFdCUuTL1we6rh8gen5zRVUwmM6BJxkVt4NJ2wGkrXg/HD3feYCMrv+DY3lfDN7tADx90lPb+ukmanB8oSIQHLuLCJ2XcFixEC3QSvVlnH0= diff --git a/Dockerfile b/Dockerfile index e32437ef7..d49baaf8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ COPY . /go/src/github.com/pilosa/pilosa RUN wget ${GLIDE} -O /go/glide.tar.gz -q \ && tar xf /go/glide.tar.gz \ && mv /go/linux-amd64/glide /go/bin \ - && [ "$(sha256sum /go/bin/glide | cut -d' ' -f1)" = "$GLIDE_HASH" ] \ + && echo "${GLIDE_HASH} /go/bin/glide" | shasum -a 256 -c - \ && cd /go/src/github.com/pilosa/pilosa \ && make vendor \ && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa diff --git a/Makefile b/Makefile index 6b9ff1dc7..801ac52cc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: glide vendor-update docker pilosa crossbuild install generate statik +.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release GLIDE := $(shell command -v glide 2>/dev/null) STATIK := $(shell command -v statik 2>/dev/null) @@ -19,7 +19,10 @@ ifndef GLIDE curl https://glide.sh/get | sh endif -vendor: glide glide.yaml +$(GLIDE): + make glide + +vendor: $(GLIDE) glide.yaml glide install glide.lock: glide glide.yaml @@ -31,14 +34,21 @@ test: vendor go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) pilosa: vendor - go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa crossbuild: vendor mkdir -p build/pilosa-$(IDENTIFIER) make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" + cp {LICENSE,README.md} build/pilosa-$(IDENTIFIER) + tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ + @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + +release: + make crossbuild GOOS=linux GOARCH=amd64 + make crossbuild GOOS=darwin GOARCH=amd64 install: vendor - go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa .protoc-gen-gofast: vendor ifndef PROTOC @@ -61,7 +71,5 @@ ifndef STATIK endif docker: - docker build -t "pilosa:$(VERSION)" \ - --build-arg ldflags="-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) \ - -X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" . + docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . @echo "Created image: pilosa:$(VERSION)" diff --git a/client.go b/client.go index 5df0385d6..80090a07a 100644 --- a/client.go +++ b/client.go @@ -400,11 +400,13 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { } // ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired + } else if !(view == ViewStandard || view == ViewInverse) { + return ErrInvalidView } // Retrieve a list of nodes that own the slice. @@ -418,7 +420,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 for _, i := range rand.Perm(len(nodes)) { node := nodes[i] - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { + if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil { e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err) continue } else { @@ -430,7 +432,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6 } // exportNode copies a CSV export from a node to w. -func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { +func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error { // Create URL. u := url.URL{ Scheme: "http", @@ -439,6 +441,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame str RawQuery: url.Values{ "index": {index}, "frame": {frame}, + "view": {view}, "slice": {strconv.FormatUint(slice, 10)}, }.Encode(), } diff --git a/cmd/export.go b/cmd/export.go index 310e3f05c..e55f84e59 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -51,8 +51,9 @@ The file does not contain any headers. flags := exportCmd.Flags() flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.") + flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") + flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") return exportCmd diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..2d01cf4d6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -44,6 +44,7 @@ frame = "f1" v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.View, "standard") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, @@ -51,3 +52,10 @@ frame = "f1" } executeDry(t, tests) } + +func TestExportInvalidView(t *testing.T) { + output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test") + if !strings.Contains(err.Error(), "invalid view") { + t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output) + } +} diff --git a/ctl/export.go b/ctl/export.go index 8f67b7896..7da310623 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -31,7 +31,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - + View string // Filename to export to. Path string @@ -55,6 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired + } else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) { + return pilosa.ErrInvalidView } // Use output file, if specified. @@ -85,7 +87,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { return err } } diff --git a/executor.go b/executor.go index 585e0a420..6cb0e9e7a 100644 --- a/executor.go +++ b/executor.go @@ -282,7 +282,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - rowIDs, _, err := c.UintSliceArg("ids") + idsArg, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } @@ -299,7 +299,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote { + if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote { return pairs, nil } // Only the original caller should refetch the full counts. @@ -347,6 +347,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) + inverse, _ := c.Args["inverse"].(bool) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -383,7 +384,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca frame = DefaultFrame } - f := e.Holder.Fragment(index, frame, ViewStandard, slice) + // Determine view. + view := ViewStandard + if inverse { + view = ViewInverse + } + + f := e.Holder.Fragment(index, frame, view, slice) if f == nil { return nil, nil } diff --git a/executor_test.go b/executor_test.go index b69aaac37..b9b4c0bd7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "reflect" + "strconv" "strings" "testing" @@ -274,32 +275,55 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { func TestExecutor_Execute_TopN(t *testing.T) { hldr := MustOpenHolder() defer hldr.Close() + e := NewExecutor(hldr.Holder, NewCluster(1)) // Set bits for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0) + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", MustParse(` + SetBit(frame=f, rowID=0, columnID=0) + SetBit(frame=f, rowID=0, columnID=1) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(frame=f, rowID=10, columnID=0) + SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`) + SetBit(frame=other, rowID=0, columnID=0) + `), nil, nil); err != nil { + t.Fatal(err) + } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - // Execute query. - e := NewExecutor(hldr.Holder, NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) - } + t.Run("Standard", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("Inverse", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ + {ID: SliceWidth, Count: 3}, + {ID: 0, Count: 2}, + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := MustOpenHolder() diff --git a/pql/ast.go b/pql/ast.go index 19b5381ed..cf404ad12 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -184,16 +184,20 @@ func (c *Call) String() string { // SupportsInverse indicates that the call may be on an inverse frame. func (c *Call) SupportsInverse() bool { - if c.Name == "Bitmap" { - return true - } - return false + return c.Name == "Bitmap" || c.Name == "TopN" } // IsInverse specifies if the call is for an inverse view. // Return defaults to false unless absolutely sure of inversion. func (c *Call) IsInverse(rowLabel, columnLabel string) bool { if c.SupportsInverse() { + // Top-n has an explicit inverse flag. + if c.Name == "TopN" { + inverse, _ := c.Args["inverse"].(bool) + return inverse + } + + // Bitmap calls use the row/column labels to determine whether inverse. _, rowOK, rowErr := c.UintArg(rowLabel) _, columnOK, columnErr := c.UintArg(columnLabel) if rowErr != nil || columnErr != nil { diff --git a/webui/assets/main.js b/webui/assets/main.js index e18c562fb..887181338 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -20,7 +20,8 @@ class REPL { // keyword: length of substring that comes after cursor "SetBit()": 1, "ClearBit()": 1, - "SetBitmapAttrs()": 1, + "SetRowAttrs()": 1, + "SetColumnAttrs()": 1, "Bitmap()": 1, "Union()": 1, "Intersect()": 1, diff --git a/webui/assets/style.css b/webui/assets/style.css index 515335ef9..3a5bcaf39 100644 --- a/webui/assets/style.css +++ b/webui/assets/style.css @@ -295,4 +295,47 @@ td{ .expand { text-align: center; -} \ No newline at end of file +} + +.query h2 { + display: inline-block; +} + +.query-tooltip { + position: relative; + display: inline; + color: #000; + margin-left: 5px; +} + +.query-tooltip:hover { + color: #000; +} + +.query-tooltip-content { + background-color: rgb(250, 250, 250); + border: solid 1.5px #e4eff4; + color: #102445; + border-radius: 2px; + padding: 15px; + margin-bottom: 15px; + + position: absolute; + left: 80px; + top: -30px; + z-index: 1; +} + +.query-tooltip-container { + position: relative; + visibility: hidden; +} + +.query-tooltip:hover+.query-tooltip-container{ + visibility: visible; +} + +.code{ + font-family: monospace; +} + diff --git a/webui/index.html b/webui/index.html index 8422d4c45..09e39d17a 100644 --- a/webui/index.html +++ b/webui/index.html @@ -35,28 +35,60 @@

Query

+ ? +
+
+
PQL
+
+ SetBit(frame=foo, rowID=0, columnID=0)
+ ClearBit(frame=foo, rowID=0, columnID=0)
+ SetRowAttrs(frame=foo, rowID=0, color="blue")
+ SetColumnAttrs(frame=foo, columnID=0, shape="circle")
+ Bitmap(frame=foo, rowID=0)
+ Range(frame=foo, rowID=0, start="2010-01", end="2017-03")
+ Count(<BITMAP_CALL>)
+ TopN([BITMAP_CALL], frame=foo, n=20)
+ Union([BITMAP_CALL, ...])
+ Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
+ Difference(<BITMAP_CALL>, <BITMAP_CALL>) +
+
+
Special commands
+
+ :create index test [columnLabel=column]
+ :use index test
+ :create frame foo [rowLabel=row]
+ :delete index test
+ :delete frame foo +
+
+ <tab>: autocomplete
+ <up>/<down>: history
+
+
- - -     - +
+ +     + +
-
-
+
-

Output

-
+

Output

+
+ + +
-
-