mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
Merge branch 'master' of github.com:pilosa/pilosa
This commit is contained in:
Linh Vo
commit
0bef10d4df
11 changed files with 314 additions and 124 deletions
12
README.md
12
README.md
|
|
@ -1,8 +1,14 @@
|
|||
<a href="https://www.pilosa.com">
|
||||
<img src="https://www.pilosa.com/img/logo.svg" width="50%">
|
||||
</a>
|
||||
<p>
|
||||
<a href="https://www.pilosa.com">
|
||||
<img src="https://www.pilosa.com/img/logo.svg" width="50%">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
[](https://travis-ci.com/pilosa/pilosa)
|
||||
[](https://godoc.org/github.com/pilosa/pilosa)
|
||||
[](https://goreportcard.com/report/github.com/pilosa/pilosa)
|
||||
[](https://github.com/pilosa/pilosa/blob/master/LICENSE)
|
||||
[](https://github.com/pilosa/pilosa/releases)
|
||||
|
||||
## An open source, distributed bitmap index.
|
||||
- [Docs](#docs)
|
||||
|
|
|
|||
|
|
@ -75,39 +75,65 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(100, (SliceWidth*9)+10)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12, (SliceWidth*9)+13, (SliceWidth*9)+14, (SliceWidth*9)+15)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(2, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(3, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4, (SliceWidth*9)+5)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(22, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+10)
|
||||
sliceNums := []uint64{1, 2, 6}
|
||||
for i, num := range sliceNums {
|
||||
owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].Host())
|
||||
ownsNum := false
|
||||
for _, ownNum := range owns {
|
||||
if ownNum == num {
|
||||
ownsNum = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ownsNum {
|
||||
t.Fatalf("Trying to use slice %d on host %s, but it doesn't own that slice. It owns %s", num, s[i].Host(), owns)
|
||||
}
|
||||
}
|
||||
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(1, 4)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
|
||||
baseBit0 := SliceWidth * sliceNums[0]
|
||||
baseBit1 := SliceWidth * sliceNums[1]
|
||||
baseBit2 := SliceWidth * sliceNums[2]
|
||||
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
maxSlice := uint64(0)
|
||||
for _, x := range sliceNums {
|
||||
if x > maxSlice {
|
||||
maxSlice = x
|
||||
}
|
||||
}
|
||||
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
|
||||
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
|
||||
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
|
||||
|
||||
// Rebuild the RankCache.
|
||||
// We have to do this to avoid the 10-second cache invalidation delay
|
||||
// built into cache.Invalidate()
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 10).RecalculateCache()
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).RecalculateCache()
|
||||
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).RecalculateCache()
|
||||
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).RecalculateCache()
|
||||
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
|
||||
|
||||
// Connect to each node to compare results.
|
||||
client := make([]*Client, 3)
|
||||
client[0] = MustNewClient(s[0].Host())
|
||||
client[1] = MustNewClient(s[0].Host())
|
||||
client[2] = MustNewClient(s[0].Host())
|
||||
client[1] = MustNewClient(s[1].Host())
|
||||
client[2] = MustNewClient(s[2].Host())
|
||||
|
||||
topN := 4
|
||||
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
|
||||
|
|
@ -120,15 +146,15 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
// Check the results before every node has the correct max slice value.
|
||||
pairs := result.(internal.QueryResponse).Results[0].Pairs
|
||||
for _, pair := range pairs {
|
||||
if pair.Key == 22 && pair.Count != 11 {
|
||||
if pair.Key == 22 && pair.Count != 3 {
|
||||
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
|
||||
}
|
||||
}
|
||||
|
||||
// Set max slice to correct value.
|
||||
hldr[0].Index("i").SetRemoteMaxSlice(10)
|
||||
hldr[1].Index("i").SetRemoteMaxSlice(10)
|
||||
hldr[2].Index("i").SetRemoteMaxSlice(10)
|
||||
hldr[0].Index("i").SetRemoteMaxSlice(maxSlice)
|
||||
hldr[1].Index("i").SetRemoteMaxSlice(maxSlice)
|
||||
hldr[2].Index("i").SetRemoteMaxSlice(maxSlice)
|
||||
|
||||
result, err = client[0].ExecuteQuery(context.Background(), "i", q, true)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import (
|
|||
|
||||
const (
|
||||
// DefaultPartitionN is the default number of partitions in a cluster.
|
||||
DefaultPartitionN = 16
|
||||
DefaultPartitionN = 256
|
||||
|
||||
// DefaultReplicaN is the default number of replicas per partition.
|
||||
DefaultReplicaN = 1
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ func TestCluster_OwnsSlices(t *testing.T) {
|
|||
c := NewCluster(5)
|
||||
slices := c.OwnsSlices("test", 10, "host2")
|
||||
|
||||
if !reflect.DeepEqual(slices, []uint64{1, 3, 10}) {
|
||||
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
|
||||
t.Fatalf("unexpected slices for node's index: %v", slices)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
102
executor_test.go
102
executor_test.go
|
|
@ -276,15 +276,19 @@ func TestExecutor_Execute_TopN(t *testing.T) {
|
|||
defer hldr.Close()
|
||||
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
|
@ -302,12 +306,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
|
|||
defer hldr.Close()
|
||||
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
|
@ -325,23 +329,23 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
|
|||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
|
@ -360,19 +364,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
|
|||
defer hldr.Close()
|
||||
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
|
||||
|
||||
// Create an intersecting row.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
|
||||
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
|
@ -392,9 +400,9 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
|
|||
//
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
|
||||
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -415,9 +423,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
|
|||
//
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
|
||||
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -676,8 +684,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
|
|||
// Create local executor data on slice 2 & 4.
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
|
||||
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
|
||||
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
|
|||
statikFS, err := fs.New()
|
||||
if err != nil {
|
||||
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
|
||||
fmt.Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
|
||||
h.logger().Println("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
|
||||
return
|
||||
}
|
||||
http.FileServer(statikFS).ServeHTTP(w, r)
|
||||
|
|
|
|||
|
|
@ -777,7 +777,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
|
|||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if w.Body.String() != `[{"host":"host1","internalHost":""},{"host":"host2","internalHost":""}]`+"\n" {
|
||||
} else if w.Body.String() != `[{"host":"host2","internalHost":""},{"host":"host0","internalHost":""}]`+"\n" {
|
||||
t.Fatalf("unexpected body: %q", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -246,3 +246,21 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice
|
|||
}
|
||||
return &Fragment{Fragment: frag}
|
||||
}
|
||||
|
||||
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
|
||||
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
|
||||
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
|
||||
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
v, err := f.CreateViewIfNotExists(view)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Fragment{Fragment: frag}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -291,14 +291,14 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
|
||||
// Create frames.
|
||||
client := m0.Client()
|
||||
if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write data on first cluster.
|
||||
if _, err := m0.Query("x", "", `
|
||||
if _, err := m0.Query("i", "", `
|
||||
SetBit(rowID=1, frame="f", columnID=100)
|
||||
SetBit(rowID=1, frame="f", columnID=1000)
|
||||
SetBit(rowID=1, frame="f", columnID=100000)
|
||||
|
|
@ -311,7 +311,7 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
}
|
||||
|
||||
// Query row on first cluster.
|
||||
if res, err := m0.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil {
|
||||
if res, err := m0.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
|
|
@ -325,16 +325,16 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
client, err := pilosa.NewClient(m2.Server.Host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
} else if err := m2.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := m2.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil {
|
||||
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "i", "f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query row on second cluster.
|
||||
if res, err := m2.Query("x", "", `Bitmap(rowID=1, frame="f")`); err != nil {
|
||||
if res, err := m2.Query("i", "", `Bitmap(rowID=1, frame="f")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
|
|
|
|||
|
|
@ -116,42 +116,90 @@ class REPL {
|
|||
|
||||
process_query(query) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
var url, data, request, command_name;
|
||||
var e = document.getElementById("index-dropdown");
|
||||
var indexname = e.options[e.selectedIndex].text;
|
||||
xhr.open('POST', '/index/' + indexname + '/query');
|
||||
var repl = this;
|
||||
if (query.startsWith(":")) {
|
||||
var parsed_query = parse_query(query, indexname);
|
||||
if (Object.keys(parsed_query).length === 0) {
|
||||
repl.create_single_output({
|
||||
"input": query,
|
||||
"output": "invalid query",
|
||||
"status": 400,
|
||||
"indexname": indexname,
|
||||
});
|
||||
return;
|
||||
} else {
|
||||
// set selectedIndex from dropdown list
|
||||
if (parsed_query.command === "use") {
|
||||
for (var i = 0; i < e.options.length; i++) {
|
||||
if (e.options[i].text === parsed_query.command_name) {
|
||||
e.selectedIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
url = parsed_query.url;
|
||||
data = parsed_query.data;
|
||||
request = parsed_query.request;
|
||||
command_name = parsed_query.command_name;
|
||||
}
|
||||
}
|
||||
else {
|
||||
url = '/index/' + indexname + '/query'
|
||||
request = "POST"
|
||||
data = query
|
||||
}
|
||||
xhr.open(request, url);
|
||||
xhr.setRequestHeader('Content-Type', 'application/text');
|
||||
|
||||
var repl = this
|
||||
var start_time = new Date().getTime();
|
||||
xhr.send(query)
|
||||
xhr.onload = function() {
|
||||
var end_time = new Date().getTime()
|
||||
xhr.onload = function () {
|
||||
var end_time = new Date().getTime();
|
||||
repl.result_number++
|
||||
repl.createSingleOutput({
|
||||
"input": query,
|
||||
"output": xhr.responseText,
|
||||
"indexname": indexname,
|
||||
"querytime_ms": end_time - start_time,
|
||||
})
|
||||
}
|
||||
repl.create_single_output({
|
||||
"input": query,
|
||||
"output": xhr.responseText,
|
||||
"status": xhr.status,
|
||||
"indexname": indexname,
|
||||
"querytime_ms": end_time - start_time,
|
||||
});
|
||||
};
|
||||
|
||||
xhr.send(data);
|
||||
// Remove index from dropdown with delete index command
|
||||
if (request === 'DELETE' && url === '/index/' + command_name){
|
||||
for (var i = 0; i < e.options.length; i++) {
|
||||
if (e.options[i].text === command_name) {
|
||||
e.remove(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
createSingleOutput(res) {
|
||||
var node = document.createElement("div");
|
||||
node.classList.add('output');
|
||||
var output_string = res['output']
|
||||
var output_json = JSON.parse(output_string)
|
||||
var result_class = "result-output"
|
||||
var getting_started_errors = [
|
||||
'index not found',
|
||||
'frame not found',
|
||||
]
|
||||
|
||||
if("error" in output_json) {
|
||||
result_class = "result-error"
|
||||
if(getting_started_errors.indexOf(output_json['error']) >= 0) {
|
||||
output_string += `<br />
|
||||
create_single_output(res) {
|
||||
var node = document.createElement("div");
|
||||
node.classList.add('output');
|
||||
var output_string = res['output']
|
||||
var result_class = "result-output"
|
||||
var getting_started_errors = [
|
||||
'index not found',
|
||||
'frame not found',
|
||||
]
|
||||
var output_json;
|
||||
if (isJSON(output_string)) {
|
||||
output_json = JSON.parse(output_string)
|
||||
}
|
||||
// handle output formatting
|
||||
if (res["status"] != 200) {
|
||||
result_class = "result-error";
|
||||
if (output_json) {
|
||||
if ("error" in output_json) {
|
||||
if (getting_started_errors.indexOf(output_json['error']) >= 0) {
|
||||
output_string += `<br />
|
||||
<br />
|
||||
Just getting started? Try this:<br />
|
||||
$ curl -XPOST "http://127.0.0.1:10101/index/test" -d '{"options": {"columnLabel": "col"}}' # create index "test"<br />
|
||||
|
|
@ -159,8 +207,10 @@ class REPL {
|
|||
# Select "test" in the index dropdown above<br />
|
||||
SetBit(row=0, col=0, frame=foo) # Use PQL to set a bit
|
||||
`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var markup =`
|
||||
|
|
@ -184,7 +234,9 @@ class REPL {
|
|||
</div>
|
||||
<div class="${result_class}">
|
||||
${output_string}
|
||||
</div>
|
||||
</div>
|
||||
<a href="#" class="expand"><h5>Expand</h5></a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="pane">
|
||||
|
|
@ -195,8 +247,22 @@ class REPL {
|
|||
</div>
|
||||
</div>
|
||||
`
|
||||
node.innerHTML = markup;
|
||||
this.output.insertBefore(node, this.output.firstChild)
|
||||
node.innerHTML = markup;
|
||||
this.output.insertBefore(node, this.output.firstChild);
|
||||
|
||||
// Expand when overflow
|
||||
var element = this.output.firstChild.getElementsByClassName(result_class)[0];
|
||||
var expand = this.output.firstChild.getElementsByClassName("expand")[0];
|
||||
if (element.clientHeight < element.scrollHeight) {
|
||||
expand.style.display = 'block';
|
||||
} else {
|
||||
expand.style.display = 'none';
|
||||
}
|
||||
expand.onclick = function () {
|
||||
element.style.height = element.scrollHeight + "px";
|
||||
expand.style.display = 'none';
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
populate_index_dropdown() {
|
||||
|
|
@ -367,7 +433,7 @@ function check_anchor_uri() {
|
|||
}
|
||||
}
|
||||
|
||||
Date.prototype.today = function () {
|
||||
Date.prototype.today = function () {
|
||||
return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate();
|
||||
}
|
||||
|
||||
|
|
@ -388,3 +454,64 @@ repl.bind_events()
|
|||
input.focus()
|
||||
|
||||
check_anchor_uri()
|
||||
|
||||
function isJSON(str) {
|
||||
try {
|
||||
JSON.parse(str)
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function parse_query(query, indexname) {
|
||||
var keys = query.replace(/\s+/g, " ").split(" ");
|
||||
var command = keys[0];
|
||||
var command_type = keys[1];
|
||||
var command_name = keys[2];
|
||||
|
||||
if (command !== ":use") {
|
||||
if (!command_name){
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var parsed_query = {};
|
||||
parsed_query["command"] = command.substr(1, command.length);
|
||||
parsed_query["command_name"] = command_name;
|
||||
switch (command) {
|
||||
case ":create":
|
||||
parsed_query["request"] = "POST";
|
||||
switch (command_type){
|
||||
case "index":
|
||||
parsed_query["url"] = '/index/' + command_name;
|
||||
parsed_query["data"] = "";
|
||||
break;
|
||||
case "frame":
|
||||
parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
|
||||
parsed_query["data"] = "";
|
||||
break
|
||||
}
|
||||
break;
|
||||
case ":delete":
|
||||
parsed_query["request"] = "DELETE";
|
||||
switch (command_type){
|
||||
case "index":
|
||||
parsed_query["url"] = '/index/' + command_name;
|
||||
parsed_query["data"] = "";
|
||||
break;
|
||||
case "frame":
|
||||
parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
|
||||
parsed_query["data"] = "";
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case ":use":
|
||||
parsed_query["command_name"] = keys[1];
|
||||
break;
|
||||
default:
|
||||
return {}
|
||||
}
|
||||
return parsed_query;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -203,8 +203,6 @@ em{
|
|||
display: block;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.result-io-header{
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
|
@ -214,6 +212,7 @@ em{
|
|||
.result-input,
|
||||
.result-output,
|
||||
.result-error{
|
||||
height: 60px;
|
||||
border-radius: 2px;
|
||||
background-color: #fafafa;
|
||||
border: solid 1.5px #e4eff4;
|
||||
|
|
@ -224,6 +223,8 @@ em{
|
|||
color: #102445;
|
||||
padding: 15px;
|
||||
margin-bottom: 15px;
|
||||
word-break: break-all;
|
||||
overflow:hidden;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -290,4 +291,8 @@ td{
|
|||
.number { color: darkorange; }
|
||||
.boolean { color: blue; }
|
||||
.null { color: magenta; }
|
||||
.key { color: red; }
|
||||
.key { color: red; }
|
||||
|
||||
.expand {
|
||||
text-align: center;
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue