diff --git a/README.md b/README.md
index 9dcf580bd..ef54f7553 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,14 @@
-
- 
-
+
+
+
+
+
[](https://travis-ci.com/pilosa/pilosa)
+[](https://godoc.org/github.com/pilosa/pilosa)
+[](https://goreportcard.com/report/github.com/pilosa/pilosa)
+[](https://github.com/pilosa/pilosa/blob/master/LICENSE)
+[](https://github.com/pilosa/pilosa/releases)
## An open source, distributed bitmap index.
- [Docs](#docs)
diff --git a/client_test.go b/client_test.go
index a89a41cce..6b5f43195 100644
--- a/client_test.go
+++ b/client_test.go
@@ -75,39 +75,65 @@ func TestClient_MultiNode(t *testing.T) {
}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(100, (SliceWidth*9)+10)
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12)
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12, (SliceWidth*9)+13, (SliceWidth*9)+14, (SliceWidth*9)+15)
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(2, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4)
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(3, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4, (SliceWidth*9)+5)
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(22, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+10)
+ sliceNums := []uint64{1, 2, 6}
+ for i, num := range sliceNums {
+ owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host())
+ ownsNum := false
+ for _, ownNum := range owns {
+ if ownNum == num {
+ ownsNum = true
+ break
+ }
+ }
+ if !ownsNum {
+ t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %s", num, s[i].Host(), owns)
+ }
+ }
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(1, 4)
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
+ baseBit0 := SliceWidth * sliceNums[0]
+ baseBit1 := SliceWidth * sliceNums[1]
+ baseBit2 := SliceWidth * sliceNums[2]
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
+ maxSlice := uint64(0)
+ for _, x := range sliceNums {
+ if x > maxSlice {
+ maxSlice = x
+ }
+ }
+
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
+
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
+
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
- hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
- hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 10).RecalculateCache()
- hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).RecalculateCache()
+ hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache()
+ hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache()
+ hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
client[0] = MustNewClient(s[0].Host())
- client[1] = MustNewClient(s[0].Host())
- client[2] = MustNewClient(s[0].Host())
+ client[1] = MustNewClient(s[1].Host())
+ client[2] = MustNewClient(s[2].Host())
topN := 4
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
@@ -120,15 +146,15 @@ func TestClient_MultiNode(t *testing.T) {
// Check the results before every node has the correct max slice value.
pairs := result.(internal.QueryResponse).Results[0].Pairs
for _, pair := range pairs {
- if pair.Key == 22 && pair.Count != 11 {
+ if pair.Key == 22 && pair.Count != 3 {
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
}
}
// Set max slice to correct value.
- hldr[0].Index("i").SetRemoteMaxSlice(10)
- hldr[1].Index("i").SetRemoteMaxSlice(10)
- hldr[2].Index("i").SetRemoteMaxSlice(10)
+ hldr[0].Index("i").SetRemoteMaxSlice(maxSlice)
+ hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
+ hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
result, err = client[0].ExecuteQuery(context.Background(), "i", q, true)
if err != nil {
diff --git a/cluster.go b/cluster.go
index 4cd0321bc..e7f4a67ae 100644
--- a/cluster.go
+++ b/cluster.go
@@ -23,7 +23,7 @@ import (
const (
// DefaultPartitionN is the default number of partitions in a cluster.
- DefaultPartitionN = 16
+ DefaultPartitionN = 256
// DefaultReplicaN is the default number of replicas per partition.
DefaultReplicaN = 1
diff --git a/cluster_test.go b/cluster_test.go
index 3caea3fe2..5e4668fe2 100644
--- a/cluster_test.go
+++ b/cluster_test.go
@@ -137,7 +137,7 @@ func TestCluster_OwnsSlices(t *testing.T) {
c := NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
- if !reflect.DeepEqual(slices, []uint64{1, 3, 10}) {
+ if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
diff --git a/executor_test.go b/executor_test.go
index 084511fb0..1e0cd8343 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -276,15 +276,19 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
+
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
@@ -302,12 +306,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
@@ -325,23 +329,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
@@ -360,19 +364,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
// Create an intersecting row.
- hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
- hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
- hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
+
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
+ hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
@@ -392,9 +400,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//
hldr := MustOpenHolder()
defer hldr.Close()
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
@@ -415,9 +423,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
//
hldr := MustOpenHolder()
defer hldr.Close()
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
@@ -676,8 +684,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Create local executor data on slice 2 & 4.
hldr := MustOpenHolder()
defer hldr.Close()
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
- hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
+ hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
diff --git a/handler.go b/handler.go
index 67cc14869..fb2a35a0f 100644
--- a/handler.go
+++ b/handler.go
@@ -138,7 +138,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
statikFS, err := fs.New()
if err != nil {
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
- fmt.Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
+ h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
return
}
http.FileServer(statikFS).ServeHTTP(w, r)
diff --git a/handler_test.go b/handler_test.go
index c2aa50927..6744e74ea 100644
--- a/handler_test.go
+++ b/handler_test.go
@@ -777,7 +777,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
- } else if w.Body.String() != `[{"host":"host1","internalHost":""},{"host":"host2","internalHost":""}]`+"\n" {
+ } else if w.Body.String() != `[{"host":"host2","internalHost":""},{"host":"host0","internalHost":""}]`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
diff --git a/holder_test.go b/holder_test.go
index d7c90fccd..52288670a 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -246,3 +246,21 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice
}
return &Fragment{Fragment: frag}
}
+
+// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
+func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
+ idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
+ f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
+ if err != nil {
+ panic(err)
+ }
+ v, err := f.CreateViewIfNotExists(view)
+ if err != nil {
+ panic(err)
+ }
+ frag, err := v.CreateFragmentIfNotExists(slice)
+ if err != nil {
+ panic(err)
+ }
+ return &Fragment{Fragment: frag}
+}
diff --git a/server/server_test.go b/server/server_test.go
index d981a4cf3..d677569e0 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -291,14 +291,14 @@ func TestMain_FrameRestore(t *testing.T) {
// Create frames.
client := m0.Client()
- if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+ if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
- } else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
+ } else if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first cluster.
- if _, err := m0.Query("x", "", `
+ if _, err := m0.Query("i", "", `
SetBit(rowID=1, frame="f", columnID=100)
SetBit(rowID=1, frame="f", columnID=1000)
SetBit(rowID=1, frame="f", columnID=100000)
@@ -311,7 +311,7 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Query row on first cluster.
- if res, err := m0.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil {
+ if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@@ -325,16 +325,16 @@ func TestMain_FrameRestore(t *testing.T) {
client, err := pilosa.NewClient(m2.Server.Host)
if err != nil {
t.Fatal(err)
- } else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
+ } else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
- } else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
+ } else if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
- } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil {
+ } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "i", "f"); err != nil {
t.Fatal(err)
}
// Query row on second cluster.
- if res, err := m2.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil {
+ if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
diff --git a/webui/assets/main.js b/webui/assets/main.js
index 4e17db19a..78ed7470c 100644
--- a/webui/assets/main.js
+++ b/webui/assets/main.js
@@ -116,42 +116,90 @@ class REPL {
process_query(query) {
var xhr = new XMLHttpRequest();
+ var url, data, request, command_name;
var e = document.getElementById("index-dropdown");
var indexname = e.options[e.selectedIndex].text;
- xhr.open('POST', '/index/' + indexname + '/query');
+ var repl = this;
+ if (query.startsWith(":")) {
+ var parsed_query = parse_query(query, indexname);
+ if (Object.keys(parsed_query).length === 0) {
+ repl.create_single_output({
+ "input": query,
+ "output": "invalid query",
+ "status": 400,
+ "indexname": indexname,
+ });
+ return;
+ } else {
+ // set selectedIndex from dropdown list
+ if (parsed_query.command === "use") {
+ for (var i = 0; i < e.options.length; i++) {
+ if (e.options[i].text === parsed_query.command_name) {
+ e.selectedIndex = i;
+ break;
+ }
+ }
+ return;
+ }
+ url = parsed_query.url;
+ data = parsed_query.data;
+ request = parsed_query.request;
+ command_name = parsed_query.command_name;
+ }
+ }
+ else {
+ url = '/index/' + indexname + '/query'
+ request = "POST"
+ data = query
+ }
+ xhr.open(request, url);
xhr.setRequestHeader('Content-Type', 'application/text');
- var repl = this
var start_time = new Date().getTime();
- xhr.send(query)
- xhr.onload = function() {
- var end_time = new Date().getTime()
+ xhr.onload = function () {
+ var end_time = new Date().getTime();
repl.result_number++
- repl.createSingleOutput({
- "input": query,
- "output": xhr.responseText,
- "indexname": indexname,
- "querytime_ms": end_time - start_time,
- })
- }
+ repl.create_single_output({
+ "input": query,
+ "output": xhr.responseText,
+ "status": xhr.status,
+ "indexname": indexname,
+ "querytime_ms": end_time - start_time,
+ });
+ };
+ xhr.send(data);
+ // Remove index from dropdown with delete index command
+ if (request === 'DELETE' && url === '/index/' + command_name){
+ for (var i = 0; i < e.options.length; i++) {
+ if (e.options[i].text === command_name) {
+ e.remove(i);
+ break;
+ }
+ }
+ }
}
- createSingleOutput(res) {
- var node = document.createElement("div");
- node.classList.add('output');
- var output_string = res['output']
- var output_json = JSON.parse(output_string)
- var result_class = "result-output"
- var getting_started_errors = [
- 'index not found',
- 'frame not found',
- ]
-
- if("error" in output_json) {
- result_class = "result-error"
- if(getting_started_errors.indexOf(output_json['error']) >= 0) {
- output_string += `
+ create_single_output(res) {
+ var node = document.createElement("div");
+ node.classList.add('output');
+ var output_string = res['output']
+ var result_class = "result-output"
+ var getting_started_errors = [
+ 'index not found',
+ 'frame not found',
+ ]
+ var output_json;
+ if (isJSON(output_string)) {
+ output_json = JSON.parse(output_string)
+ }
+ // handle output formatting
+ if (res["status"] != 200) {
+ result_class = "result-error";
+ if (output_json) {
+ if ("error" in output_json) {
+ if (getting_started_errors.indexOf(output_json['error']) >= 0) {
+ output_string += `
Just getting started? Try this:
$ curl -XPOST "http://127.0.0.1:10101/index/test" -d '{"options": {"columnLabel": "col"}}' # create index "test"
@@ -159,8 +207,10 @@ class REPL {
# Select "test" in the index dropdown above
SetBit(row=0, col=0, frame=foo) # Use PQL to set a bit
`
+ }
+ }
+ }
}
- }
var markup =`
@@ -184,7 +234,9 @@ class REPL {
${output_string}
-
+
+ Expand
+
@@ -195,8 +247,22 @@ class REPL {
`
- node.innerHTML = markup;
- this.output.insertBefore(node, this.output.firstChild)
+ node.innerHTML = markup;
+ this.output.insertBefore(node, this.output.firstChild);
+
+ // Expand when overflow
+ var element = this.output.firstChild.getElementsByClassName(result_class)[0];
+ var expand = this.output.firstChild.getElementsByClassName("expand")[0];
+ if (element.clientHeight < element.scrollHeight) {
+ expand.style.display = 'block';
+ } else {
+ expand.style.display = 'none';
+ }
+ expand.onclick = function () {
+ element.style.height = element.scrollHeight + "px";
+ expand.style.display = 'none';
+ return false;
+ };
}
populate_index_dropdown() {
@@ -367,7 +433,7 @@ function check_anchor_uri() {
}
}
-Date.prototype.today = function () {
+Date.prototype.today = function () {
return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate();
}
@@ -388,3 +454,64 @@ repl.bind_events()
input.focus()
check_anchor_uri()
+
+function isJSON(str) {
+ try {
+ JSON.parse(str)
+ } catch (e) {
+ return false
+ }
+ return true
+}
+
+function parse_query(query, indexname) {
+ var keys = query.replace(/\s+/g, " ").split(" ");
+ var command = keys[0];
+ var command_type = keys[1];
+ var command_name = keys[2];
+
+ if (command !== ":use") {
+ if (!command_name){
+ return {}
+ }
+ }
+
+
+ var parsed_query = {};
+ parsed_query["command"] = command.substr(1, command.length);
+ parsed_query["command_name"] = command_name;
+ switch (command) {
+ case ":create":
+ parsed_query["request"] = "POST";
+ switch (command_type){
+ case "index":
+ parsed_query["url"] = '/index/' + command_name;
+ parsed_query["data"] = "";
+ break;
+ case "frame":
+ parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
+ parsed_query["data"] = "";
+ break
+ }
+ break;
+ case ":delete":
+ parsed_query["request"] = "DELETE";
+ switch (command_type){
+ case "index":
+ parsed_query["url"] = '/index/' + command_name;
+ parsed_query["data"] = "";
+ break;
+ case "frame":
+ parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
+ parsed_query["data"] = "";
+ break;
+ }
+ break;
+ case ":use":
+ parsed_query["command_name"] = keys[1];
+ break;
+ default:
+ return {}
+ }
+ return parsed_query;
+}
diff --git a/webui/assets/style.css b/webui/assets/style.css
index a3a493e16..515335ef9 100644
--- a/webui/assets/style.css
+++ b/webui/assets/style.css
@@ -203,8 +203,6 @@ em{
display: block;
}
-
-
.result-io-header{
display: flex;
align-items: center;
@@ -214,6 +212,7 @@ em{
.result-input,
.result-output,
.result-error{
+ height: 60px;
border-radius: 2px;
background-color: #fafafa;
border: solid 1.5px #e4eff4;
@@ -224,6 +223,8 @@ em{
color: #102445;
padding: 15px;
margin-bottom: 15px;
+ word-break: break-all;
+ overflow:hidden;
}
@@ -290,4 +291,8 @@ td{
.number { color: darkorange; }
.boolean { color: blue; }
.null { color: magenta; }
-.key { color: red; }
\ No newline at end of file
+.key { color: red; }
+
+.expand {
+ text-align: center;
+}
\ No newline at end of file