Merge branch 'master' into metrics

2026-09-12 23:51:03 +00:00 · 2017-05-16 14:03:16 -05:00 · 2017-05-16 14:03:16 -05:00 · 143cc14b1a
commit 143cc14b1a
parent 60cd24b368 f3d0aec46d
13 changed files with 194 additions and 57 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,6 +1,7 @@
 language: go
 go:
-  - 1.7.x
+  - 1.7
+  - 1.8
  - master
 addons:
  apt:
@ -8,7 +9,10 @@ addons:
      - sourceline: 'ppa:masterminds/glide'
    packages:
      - glide
-script: make test
+before_install:
+  - go get github.com/mattn/goveralls
+script:
+  - make vendor && $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go"
 notifications:
  slack:
    secure: k0m8IqQIjb8L5b1plR9+w4hGo5iWlGtrR9nxvPsHL7gShwez74dpHCmFUCjR/uJoZa5Xq6849IWmiBB71ZHE/wNVN60HpMeui9VKkYvUDVpgBuqoIo969ySPDyQo4jNsk0wRRv/dG5/wXo7ArY8qOp5UDjdVuEf4nEbDlBtKjo0A2W8qXBTdZ5EJ3VeP5nrtrT7knuWxXgeUyVmLkpnUaLgkY7icvH6evqkftfcb56n3l5ulKtlQkI/ij2XCUK+5xtNB9m0e1QbiqIFS7sjwlSly/GCstPVnTEp4oHNHE7LtS6TJ8lgRV90Qy6LPtROjbSdfVjhJ5cYKus8DdddSfcsm2qgHCHQThyYtVAfOXaFPYPb50WXEu3HOckHk1tYChfkdxwEGdn/cN8/xxa7Khd7T/+SaS1B4CPxOFp0ie+rD4lF6IhApmpr/UwtiW6srHcPB04lF5mioYWmyS2eyGKTzSZR82nifg5KYUgBL1FjywQDYb8t9QaAn/6svu08k4H29TBnYpOQWREb38pYmEnMtRqiRmbfQL6IyV8405TJkl/PZmIElX6IQGjf53M1KoFdCUuTL1we6rh8gen5zRVUwmM6BJxkVt4NJ2wGkrXg/HD3feYCMrv+DY3lfDN7tADx90lPb+ukmanB8oSIQHLuLCJ2XcFixEC3QSvVlnH0=
--- a/2
+++ b/2
@ -9,7 +9,7 @@ COPY . /go/src/github.com/pilosa/pilosa
 RUN wget ${GLIDE} -O /go/glide.tar.gz -q \
    && tar xf /go/glide.tar.gz \
    && mv /go/linux-amd64/glide /go/bin \
-    && [ "$(sha256sum /go/bin/glide | cut -d' ' -f1)" = "$GLIDE_HASH" ] \
+    && echo "${GLIDE_HASH}  /go/bin/glide" | shasum -a 256 -c - \
    && cd /go/src/github.com/pilosa/pilosa \
    && make vendor \
    && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa
--- a/22
+++ b/22
@ -1,4 +1,4 @@
-.PHONY: glide vendor-update docker pilosa crossbuild install generate statik
+.PHONY: glide vendor-update docker pilosa crossbuild install generate statik release

 GLIDE := $(shell command -v glide 2>/dev/null)
 STATIK := $(shell command -v statik 2>/dev/null)
@ -19,7 +19,10 @@ ifndef GLIDE
 	curl https://glide.sh/get | sh
 endif

-vendor: glide glide.yaml
+$(GLIDE):
+	make glide
+
+vendor: $(GLIDE) glide.yaml
 	glide install

 glide.lock: glide glide.yaml
@ -31,14 +34,21 @@ test: vendor
 	go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)

 pilosa: vendor
-	go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+	go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa

 crossbuild: vendor
 	mkdir -p build/pilosa-$(IDENTIFIER)
 	make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
+	cp {LICENSE,README.md} build/pilosa-$(IDENTIFIER)
+	tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
+	@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
+
+release:
+	make crossbuild GOOS=linux GOARCH=amd64
+	make crossbuild GOOS=darwin GOARCH=amd64

 install: vendor
-	go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+	go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa

 .protoc-gen-gofast: vendor
 ifndef PROTOC
@ -61,7 +71,5 @@ ifndef STATIK
 endif

 docker:
-	docker build -t "pilosa:$(VERSION)" \
-		--build-arg ldflags="-X github.com/pilosa/pilosa/cmd.Version=$(VERSION) \
-		-X github.com/pilosa/pilosa/cmd.BuildTime=$(BUILD_TIME)" .
+	docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) .
 	@echo "Created image: pilosa:$(VERSION)"
--- a/client.go
+++ b/client.go
@ -400,11 +400,13 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error {
 }

 // ExportCSV bulk exports data for a single slice from a host to CSV format.
-func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error {
+func (c *Client) ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error {
 	if index == "" {
 		return ErrIndexRequired
 	} else if frame == "" {
 		return ErrFrameRequired
+	} else if !(view == ViewStandard || view == ViewInverse) {
+		return ErrInvalidView
 	}

 	// Retrieve a list of nodes that own the slice.
@ -418,7 +420,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6
 	for _, i := range rand.Perm(len(nodes)) {
 		node := nodes[i]

-		if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil {
+		if err := c.exportNodeCSV(ctx, node, index, frame, view, slice, w); err != nil {
 			e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
 			continue
 		} else {
@ -430,7 +432,7 @@ func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint6
 }

 // exportNode copies a CSV export from a node to w.
-func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error {
+func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame, view string, slice uint64, w io.Writer) error {
 	// Create URL.
 	u := url.URL{
 		Scheme: "http",
@ -439,6 +441,7 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame str
 		RawQuery: url.Values{
 			"index": {index},
 			"frame": {frame},
+			"view":  {view},
 			"slice": {strconv.FormatUint(slice, 10)},
 		}.Encode(),
 	}
--- a/cmd/export.go
+++ b/cmd/export.go
@ -51,8 +51,9 @@ The file does not contain any headers.
 	flags := exportCmd.Flags()

 	flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
-	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.")
-	flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.")
+	flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
+	flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
+	flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard")
 	flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")

 	return exportCmd
--- a/cmd/export_test.go
+++ b/cmd/export_test.go
@ -44,6 +44,7 @@ frame = "f1"
 				v.Check(cmd.Exporter.Host, "localhost:12345")
 				v.Check(cmd.Exporter.Index, "myindex")
 				v.Check(cmd.Exporter.Frame, "f1")
+				v.Check(cmd.Exporter.View, "standard")
 				v.Check(cmd.Exporter.Path, "/somefile")
 				return v.Error()
 			},
@ -51,3 +52,10 @@ frame = "f1"
 	}
 	executeDry(t, tests)
 }
+
+func TestExportInvalidView(t *testing.T) {
+	output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test")
+	if !strings.Contains(err.Error(), "invalid view") {
+		t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output)
+	}
+}
--- a/ctl/export.go
+++ b/ctl/export.go
@ -31,7 +31,7 @@ type ExportCommand struct {
 	// Name of the index & frame to export from.
 	Index string
 	Frame string
-
+	View string
 	// Filename to export to.
 	Path string

@ -55,6 +55,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 		return pilosa.ErrIndexRequired
 	} else if cmd.Frame == "" {
 		return pilosa.ErrFrameRequired
+	} else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) {
+		return pilosa.ErrInvalidView
 	}

 	// Use output file, if specified.
@ -85,7 +87,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 	// Export each slice.
 	for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
 		logger.Printf("exporting slice: %d", slice)
-		if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
+		if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
 			return err
 		}
 	}
--- a/executor.go
+++ b/executor.go
@ -282,7 +282,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
 // This first performs the TopN() to determine the top results and then
 // requeries to retrieve the full counts for each of the top results.
 func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
-	rowIDs, _, err := c.UintSliceArg("ids")
+	idsArg, _, err := c.UintSliceArg("ids")
 	if err != nil {
 		return nil, fmt.Errorf("executeTopN: %v", err)
 	}
@ -299,7 +299,7 @@ func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, s

 	// If this call is against specific ids, or we didn't get results,
 	// or we are part of a larger distributed query then don't refetch.
-	if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote {
+	if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote {
 		return pairs, nil
 	}
 	// Only the original caller should refetch the full counts.
@ -347,6 +347,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
 // executeTopNSlice executes a TopN call for a single slice.
 func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
 	frame, _ := c.Args["frame"].(string)
+	inverse, _ := c.Args["inverse"].(bool)
 	n, _, err := c.UintArg("n")
 	if err != nil {
 		return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -383,7 +384,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
 		frame = DefaultFrame
 	}

-	f := e.Holder.Fragment(index, frame, ViewStandard, slice)
+	// Determine view.
+	view := ViewStandard
+	if inverse {
+		view = ViewInverse
+	}
+
+	f := e.Holder.Fragment(index, frame, view, slice)
 	if f == nil {
 		return nil, nil
 	}
--- a/executor_test.go
+++ b/executor_test.go
@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"

@ -274,32 +275,55 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
 func TestExecutor_Execute_TopN(t *testing.T) {
 	hldr := MustOpenHolder()
 	defer hldr.Close()
+	e := NewExecutor(hldr.Holder, NewCluster(1))

 	// Set bits for rows 0, 10, & 20 across two slices.
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
-	hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
+	if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
+		t.Fatal(err)
+	} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
+		t.Fatal(err)
+	} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
+		t.Fatal(err)
+	} else if _, err := e.Execute(context.Background(), "i", MustParse(`
+		SetBit(frame=f, rowID=0, columnID=0)
+		SetBit(frame=f, rowID=0, columnID=1)
+		SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`)
+		SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`)
+		SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
+		SetBit(frame=f, rowID=10, columnID=0)
+		SetBit(frame=f, rowID=10, columnID=`+strconv.Itoa(SliceWidth)+`)
+		SetBit(frame=f, rowID=20, columnID=`+strconv.Itoa(SliceWidth)+`)
+		SetBit(frame=other, rowID=0, columnID=0)
+	`), nil, nil); err != nil {
+		t.Fatal(err)
+	}

 	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
+	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache()
 	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
 	hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()

-	// Execute query.
-	e := NewExecutor(hldr.Holder, NewCluster(1))
-	if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
-		{ID: 0, Count: 5},
-		{ID: 10, Count: 2},
-	}) {
-		t.Fatalf("unexpected result: %s", spew.Sdump(result))
-	}
+	t.Run("Standard", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
+			{ID: 0, Count: 5},
+			{ID: 10, Count: 2},
+		}) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
+
+	t.Run("Inverse", func(t *testing.T) {
+		if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
+			{ID: SliceWidth, Count: 3},
+			{ID: 0, Count: 2},
+		}) {
+			t.Fatalf("unexpected result: %s", spew.Sdump(result))
+		}
+	})
 }
 func TestExecutor_Execute_TopN_fill(t *testing.T) {
 	hldr := MustOpenHolder()
--- a/pql/ast.go
+++ b/pql/ast.go
@ -184,16 +184,20 @@ func (c *Call) String() string {

 // SupportsInverse indicates that the call may be on an inverse frame.
 func (c *Call) SupportsInverse() bool {
-	if c.Name == "Bitmap" {
-		return true
-	}
-	return false
+	return c.Name == "Bitmap" || c.Name == "TopN"
 }

 // IsInverse specifies if the call is for an inverse view.
 // Return defaults to false unless absolutely sure of inversion.
 func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
 	if c.SupportsInverse() {
+		// Top-n has an explicit inverse flag.
+		if c.Name == "TopN" {
+			inverse, _ := c.Args["inverse"].(bool)
+			return inverse
+		}
+
+		// Bitmap calls use the row/column labels to determine whether inverse.
 		_, rowOK, rowErr := c.UintArg(rowLabel)
 		_, columnOK, columnErr := c.UintArg(columnLabel)
 		if rowErr != nil || columnErr != nil {
--- a/webui/assets/main.js
+++ b/webui/assets/main.js
@ -20,7 +20,8 @@ class REPL {
          // keyword: length of substring that comes after cursor
          "SetBit()": 1,
          "ClearBit()": 1,
-          "SetBitmapAttrs()": 1,
+          "SetRowAttrs()": 1,
+          "SetColumnAttrs()": 1,
          "Bitmap()": 1,
          "Union()": 1,
          "Intersect()": 1,
--- a/webui/assets/style.css
+++ b/webui/assets/style.css
@ -295,4 +295,47 @@ td{

 .expand {
  text-align: center;
-}
+}
+
+.query h2 {
+  display: inline-block;
+}
+
+.query-tooltip {
+  position: relative;
+  display: inline;
+  color: #000;
+  margin-left: 5px;
+}
+
+.query-tooltip:hover {
+  color: #000;
+}
+
+.query-tooltip-content {
+  background-color: rgb(250, 250, 250);
+  border: solid 1.5px #e4eff4;
+  color: #102445;
+  border-radius: 2px;
+  padding: 15px;
+  margin-bottom: 15px;
+
+  position: absolute;
+  left: 80px;
+  top: -30px;
+  z-index: 1;
+}
+
+.query-tooltip-container {
+  position: relative;
+  visibility: hidden;
+}
+
+.query-tooltip:hover+.query-tooltip-container{
+  visibility: visible;
+}
+
+.code{
+  font-family: monospace;
+}
+
--- a/webui/index.html
+++ b/webui/index.html
@ -35,28 +35,60 @@

 			<div class="query">
 				<h2>Query</h2>
+				<a class="query-tooltip" href="https://www.pilosa.com/docs/webui/" target="_blank"><sup>?</sup></a>
+				<div class="query-tooltip-container">
+					<div class='query-tooltip-content'>
+						<h5>PQL</h5>
+						<div class="code">
+							SetBit(frame=foo, rowID=0, columnID=0)<br />
+							ClearBit(frame=foo, rowID=0, columnID=0)<br />
+							SetRowAttrs(frame=foo, rowID=0, color="blue")<br />
+							SetColumnAttrs(frame=foo, columnID=0, shape="circle")<br />
+							Bitmap(frame=foo, rowID=0)<br />
+							Range(frame=foo, rowID=0, start="2010-01", end="2017-03")<br />
+							Count(&lt;BITMAP_CALL&gt;)<br />
+							TopN([BITMAP_CALL], frame=foo, n=20)<br />
+							Union([BITMAP_CALL, ...])<br />
+							Intersect(&lt;BITMAP_CALL&gt;, [BITMAP_CALL, ...])<br />
+							Difference(&lt;BITMAP_CALL&gt;, &lt;BITMAP_CALL&gt;)
+						</div>
+						<br />
+						<h5>Special commands</h5>
+					    <div class="code">
+							:create index test [columnLabel=column]<br />
+							:use index test<br />
+							:create frame foo [rowLabel=row]<br />
+							:delete index test<br />
+							:delete frame foo
+						</div>
+						<br />
+						&lt;tab&gt;: autocomplete<br />
+						&lt;up&gt;/&lt;down&gt;: history<br />
+					</div>
+				</div>
 				<textarea name="name" id='query' rows="5"  required></textarea>
 				<div class='input-controls'>
-
-					<select class="" name="" id="index-dropdown">
-						<option value="0">Select index</option>
-					</select>
-					&nbsp;&nbsp;&nbsp;
-					<button  id='query-btn'>Run Query</button>
+					<div class="input-hints">
+						<select class="" name="" id="index-dropdown">
+							<option value="0">Select index</option>
+						</select>
+						&nbsp;&nbsp;&nbsp;
+						<button  id='query-btn'>Run Query</button>
+					</div>
 				</div>
-			</div>


-			<div  class="output-container">
+				<div  class="output-container">

-				<h2>Output</h2>
-				<div id="outputs">
+					<h2>Output</h2>
+					<div id="outputs">
+
+						<!-- results -->
+					</div>

-					<!-- results -->
 				</div>

 			</div>
-
 		</div>

 		<div class="interface" id="interface-cluster">