mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-06 19:07:50 +00:00
Merge branch 'master' into flip
This commit is contained in:
tgruben
GitHub
commit
a58206bb4d
18 changed files with 449 additions and 146 deletions
30
attr.go
30
attr.go
|
|
@ -41,7 +41,7 @@ const (
|
|||
|
||||
// AttrStore represents a storage layer for attributes.
|
||||
type AttrStore struct {
|
||||
mu sync.Mutex
|
||||
mu sync.RWMutex
|
||||
path string
|
||||
db *bolt.DB
|
||||
|
||||
|
|
@ -92,8 +92,8 @@ func (s *AttrStore) Close() error {
|
|||
|
||||
// Attrs returns a set of attributes by ID.
|
||||
func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
// Check cache for map.
|
||||
if m = s.attrs[id]; m != nil {
|
||||
|
|
@ -119,6 +119,19 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
|
|||
|
||||
// SetAttrs sets attribute values for a given ID.
|
||||
func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
|
||||
// Ignore empty maps.
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if the attributes already exist under a read-only lock.
|
||||
if attr, err := s.Attrs(id); err != nil {
|
||||
return err
|
||||
} else if attr != nil && mapContains(attr, m) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Obtain write lock.
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
|
|
@ -493,3 +506,14 @@ func (cur *blockCursor) next() (key, value []byte) {
|
|||
|
||||
return key, value
|
||||
}
|
||||
|
||||
// mapContains returns true if all keys & values of subset are in m.
|
||||
func mapContains(m, subset map[string]interface{}) bool {
|
||||
for k, v := range subset {
|
||||
value, ok := m[k]
|
||||
if !ok || value != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
34
attr_test.go
34
attr_test.go
|
|
@ -18,6 +18,8 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
|
|
@ -143,6 +145,38 @@ func NewAttrStore() *AttrStore {
|
|||
return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
|
||||
}
|
||||
|
||||
func BenchmarkAttrStore_Duplicate(b *testing.B) {
|
||||
s := MustOpenAttrStore()
|
||||
defer s.Close()
|
||||
|
||||
// Set attributes.
|
||||
const n = 5
|
||||
for i := 0; i < n; i++ {
|
||||
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
// Update attributes with an existing subset.
|
||||
cpuN := runtime.GOMAXPROCS(0)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < cpuN; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < b.N/cpuN; j++ {
|
||||
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
|
||||
func MustOpenAttrStore() *AttrStore {
|
||||
s := NewAttrStore()
|
||||
|
|
|
|||
13
cmd/root.go
13
cmd/root.go
|
|
@ -109,6 +109,11 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
|
|||
v.AutomaticEnv()
|
||||
|
||||
c := v.GetString("config")
|
||||
var flagErr error
|
||||
validTags := make(map[string]bool)
|
||||
flags.VisitAll(func(f *pflag.Flag) {
|
||||
validTags[f.Name] = true
|
||||
})
|
||||
|
||||
// add config file to viper
|
||||
if c != "" {
|
||||
|
|
@ -118,10 +123,16 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
|
|||
if err != nil {
|
||||
return fmt.Errorf("error reading configuration file '%s': %v", c, err)
|
||||
}
|
||||
|
||||
for _, key := range v.AllKeys() {
|
||||
if _, ok := validTags[key]; !ok {
|
||||
return fmt.Errorf("invalid option in configuration file: %v", key)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// set all values from viper
|
||||
var flagErr error
|
||||
flags.VisitAll(func(f *pflag.Flag) {
|
||||
if flagErr != nil {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -171,3 +171,27 @@ func TestRootCommand(t *testing.T) {
|
|||
t.Fatalf("Expected standard usage message from RootCommand, but err: '%v', output: '%s'", err, outStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRootCommand_Config(t *testing.T) {
|
||||
file, err := ioutil.TempFile("", "test.conf")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
config := `data-dir = "/tmp/pil5_0"
|
||||
bind = "127.0.0.1:10101"
|
||||
|
||||
[cluster]
|
||||
poll-interval = "2m0s"
|
||||
replicas = 2
|
||||
partitions = 128
|
||||
hosts = [
|
||||
"127.0.0.1:10101",
|
||||
"127.0.0.1:10111",
|
||||
]`
|
||||
file.Write([]byte(config))
|
||||
file.Close()
|
||||
_, err = ExecNewRootCommand(t, "server", "--config", file.Name())
|
||||
if err.Error() != "invalid option in configuration file: cluster.partitions" {
|
||||
t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ type ExportCommand struct {
|
|||
// Name of the index & frame to export from.
|
||||
Index string
|
||||
Frame string
|
||||
View string
|
||||
View string
|
||||
// Filename to export to.
|
||||
Path string
|
||||
|
||||
|
|
|
|||
14
fragment.go
14
fragment.go
|
|
@ -86,6 +86,9 @@ type Fragment struct {
|
|||
cache Cache
|
||||
CacheSize uint32
|
||||
|
||||
// Stats reporting.
|
||||
maxRowID uint64
|
||||
|
||||
// Cache containing full rows (not just counts).
|
||||
rowCache BitmapCache
|
||||
|
||||
|
|
@ -166,6 +169,11 @@ func (f *Fragment) Open() error {
|
|||
// Clear checksums.
|
||||
f.checksums = make(map[int][]byte)
|
||||
|
||||
// Read last bit to determine max row.
|
||||
pos := f.storage.Max()
|
||||
f.maxRowID = pos / SliceWidth
|
||||
f.stats.Gauge("rows", float64(f.maxRowID))
|
||||
|
||||
return nil
|
||||
}(); err != nil {
|
||||
f.close()
|
||||
|
|
@ -409,6 +417,12 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
|
|||
|
||||
f.stats.Count("setN", 1)
|
||||
|
||||
// Update row count if they have increased.
|
||||
if rowID > f.maxRowID {
|
||||
f.maxRowID = rowID
|
||||
f.stats.Gauge("rows", float64(f.maxRowID))
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
22
handler.go
22
handler.go
|
|
@ -42,6 +42,7 @@ import (
|
|||
|
||||
_ "github.com/pilosa/pilosa/statik"
|
||||
"github.com/rakyll/statik/fs"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Handler represents an HTTP handler.
|
||||
|
|
@ -843,6 +844,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro
|
|||
// readURLQueryRequest parses query parameters from URL parameters from r.
|
||||
func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
|
||||
q := r.URL.Query()
|
||||
validQuery := validOptions(QueryRequest{})
|
||||
for key, _ := range q {
|
||||
if _, ok := validQuery[key]; !ok {
|
||||
return nil, errors.New("invalid query params")
|
||||
}
|
||||
}
|
||||
|
||||
// Parse query string.
|
||||
buf, err := ioutil.ReadAll(r.Body)
|
||||
|
|
@ -875,6 +882,21 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// validOptions return all attributes of an interface with lower first character.
|
||||
func validOptions(v interface{}) map[string]bool {
|
||||
validQuery := make(map[string]bool)
|
||||
argsType := reflect.ValueOf(v).Type()
|
||||
|
||||
for i := 0; i < argsType.NumField(); i++ {
|
||||
fieldName := argsType.Field(i).Name
|
||||
chars := []rune(fieldName)
|
||||
chars[0] = unicode.ToLower(chars[0])
|
||||
fieldName = string(chars)
|
||||
validQuery[fieldName] = true
|
||||
}
|
||||
return validQuery
|
||||
}
|
||||
|
||||
// writeQueryResponse writes the response from the executor to w.
|
||||
func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error {
|
||||
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
|
||||
|
|
|
|||
|
|
@ -209,6 +209,16 @@ func TestHandler_Query_Args_Err(t *testing.T) {
|
|||
t.Fatalf("unexpected body: %q", body)
|
||||
}
|
||||
}
|
||||
func TestHandler_Query_Params_Err(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
|
||||
t.Fatalf("unexpected body: %q", body)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Ensure the handler can execute a query with a uint64 response as JSON.
|
||||
func TestHandler_Query_Uint64_JSON(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ var (
|
|||
)
|
||||
|
||||
// Regular expression to validate index and frame names.
|
||||
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,64}$`)
|
||||
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
|
||||
|
||||
// Regular expression to validate row and column labels.
|
||||
var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,64}$`)
|
||||
var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`)
|
||||
|
||||
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
|
||||
// Can have a set of attributes attached to it.
|
||||
|
|
|
|||
55
pilosa_test.go
Normal file
55
pilosa_test.go
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
func TestValidateName(t *testing.T) {
|
||||
names := []string{
|
||||
"a", "ab", "ab1", "b-c", "d_e",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
}
|
||||
for _, name := range names {
|
||||
if pilosa.ValidateName(name) != nil {
|
||||
t.Fatalf("Should be valid index name: %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNameInvalid(t *testing.T) {
|
||||
names := []string{
|
||||
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
|
||||
}
|
||||
for _, name := range names {
|
||||
if pilosa.ValidateName(name) == nil {
|
||||
t.Fatalf("Should be invalid index name: %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLabel(t *testing.T) {
|
||||
labels := []string{
|
||||
"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
}
|
||||
for _, label := range labels {
|
||||
if pilosa.ValidateLabel(label) != nil {
|
||||
t.Fatalf("Should be valid label: %s", label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateLabelInvalid(t *testing.T) {
|
||||
labels := []string{
|
||||
"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
|
||||
}
|
||||
for _, label := range labels {
|
||||
if pilosa.ValidateLabel(label) == nil {
|
||||
t.Fatalf("Should be invalid label: %s", label)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
|
|||
case uint64:
|
||||
return tval, true, nil
|
||||
default:
|
||||
return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Calll.UintArg", tval, tval)
|
||||
return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -441,6 +441,39 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
|
|||
return output
|
||||
}
|
||||
|
||||
// Xor returns the bitwise exclusive or of b and other.
|
||||
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
|
||||
output := &Bitmap{}
|
||||
|
||||
ki, ci := b.keys, b.containers
|
||||
kj, cj := other.keys, other.containers
|
||||
|
||||
for {
|
||||
var key uint64
|
||||
var container *container
|
||||
|
||||
ni, nj := len(ki), len(kj)
|
||||
if ni == 0 && nj == 0 { // eof(i,j)
|
||||
break
|
||||
} else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
|
||||
key, container = kj[0], cj[0].clone()
|
||||
kj, cj = kj[1:], cj[1:]
|
||||
} else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
|
||||
key, container = ki[0], ci[0].clone()
|
||||
ki, ci = ki[1:], ci[1:]
|
||||
} else { // i == j
|
||||
key, container = ki[0], xor(ci[0], cj[0])
|
||||
ki, ci = ki[1:], ci[1:]
|
||||
kj, cj = kj[1:], cj[1:]
|
||||
}
|
||||
|
||||
output.keys = append(output.keys, key)
|
||||
output.containers = append(output.containers, container)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
// removeEmptyContainers deletes all containers that have a count of zero.
|
||||
func (b *Bitmap) removeEmptyContainers() {
|
||||
for i := 0; i < len(b.containers); {
|
||||
|
|
@ -1249,7 +1282,7 @@ func intersectionCountArrayArray(a, b *container) (n uint64) {
|
|||
return n
|
||||
}
|
||||
|
||||
func intersectionCountArrayBitmap(a, b *container) (n uint64) {
|
||||
func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
|
||||
// Copy array header so we can shrink it.
|
||||
array := a.array
|
||||
if len(array) == 0 {
|
||||
|
|
@ -1291,6 +1324,18 @@ func intersectionCountArrayBitmap(a, b *container) (n uint64) {
|
|||
return n
|
||||
}
|
||||
|
||||
func intersectionCountArrayBitmap(a, b *container) (n uint64) {
|
||||
for _, val := range a.array {
|
||||
i := val / 64
|
||||
if i >= uint32(len(b.bitmap)) {
|
||||
break
|
||||
}
|
||||
off := val % 64
|
||||
n += (b.bitmap[i] & (1 << off)) >> off
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func intersectionCountBitmapBitmap(a, b *container) (n uint64) {
|
||||
return popcntAndSlice(a.bitmap, b.bitmap)
|
||||
}
|
||||
|
|
@ -1584,37 +1629,6 @@ func differenceBitmapBitmap(a, b *container) *container {
|
|||
}
|
||||
return output
|
||||
}
|
||||
func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
|
||||
output := &Bitmap{}
|
||||
|
||||
ki, ci := b.keys, b.containers
|
||||
kj, cj := other.keys, other.containers
|
||||
|
||||
for {
|
||||
var key uint64
|
||||
var container *container
|
||||
|
||||
ni, nj := len(ki), len(kj)
|
||||
if ni == 0 && nj == 0 { // eof(i,j)
|
||||
break
|
||||
} else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
|
||||
key, container = kj[0], cj[0].clone()
|
||||
kj, cj = kj[1:], cj[1:]
|
||||
} else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
|
||||
key, container = ki[0], ci[0].clone()
|
||||
ki, ci = ki[1:], ci[1:]
|
||||
} else { // i == j
|
||||
key, container = ki[0], xor(ci[0], cj[0])
|
||||
ki, ci = ki[1:], ci[1:]
|
||||
kj, cj = kj[1:], cj[1:]
|
||||
}
|
||||
|
||||
output.keys = append(output.keys, key)
|
||||
output.containers = append(output.containers, container)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
func xor(a, b *container) *container {
|
||||
if a.isArray() {
|
||||
|
|
@ -1635,10 +1649,8 @@ func xor(a, b *container) *container {
|
|||
func xorArrayArray(a, b *container) *container {
|
||||
output := &container{}
|
||||
na, nb := len(a.array), len(b.array)
|
||||
for i, j := 0, 0; ; {
|
||||
if i >= na && j >= nb {
|
||||
break
|
||||
} else if i < na && j >= nb {
|
||||
for i, j := 0, 0; i < na || j < nb; {
|
||||
if i < na && j >= nb {
|
||||
output.add(a.array[i])
|
||||
i++
|
||||
continue
|
||||
|
|
@ -1664,32 +1676,19 @@ func xorArrayArray(a, b *container) *container {
|
|||
}
|
||||
|
||||
func xorArrayBitmap(a, b *container) *container {
|
||||
output := &container{}
|
||||
itr := newBufIterator(newBitmapIterator(b.bitmap))
|
||||
for i := 0; ; {
|
||||
vb, eof := itr.next()
|
||||
if i >= len(a.array) && eof {
|
||||
break
|
||||
} else if i >= len(a.array) {
|
||||
output.add(vb)
|
||||
continue
|
||||
} else if eof {
|
||||
output.add(a.array[i])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
va := a.array[i]
|
||||
if va < vb {
|
||||
output.add(va)
|
||||
i++
|
||||
itr.unread()
|
||||
} else if va > vb {
|
||||
output.add(vb)
|
||||
output := b.clone()
|
||||
for _, v := range a.array {
|
||||
if b.bitmapContains(v) {
|
||||
output.remove(v)
|
||||
} else {
|
||||
i++
|
||||
output.add(v)
|
||||
}
|
||||
}
|
||||
|
||||
if output.count() < ArrayMaxSize {
|
||||
output.convertToArray()
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
@ -1703,6 +1702,10 @@ func xorBitmapBitmap(a, b *container) *container {
|
|||
output.bitmap[i] = v
|
||||
output.n += int(popcnt(v))
|
||||
}
|
||||
|
||||
if output.count() < ArrayMaxSize {
|
||||
output.convertToArray()
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,7 @@
|
|||
|
||||
package roaring
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
func TestBitmapCountRange(t *testing.T) {
|
||||
c := container{}
|
||||
|
|
@ -41,3 +39,48 @@ func TestBitmapCountRange(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntersectionCountArrayBitmap2(t *testing.T) {
|
||||
a, b := &container{}, &container{}
|
||||
tests := []struct {
|
||||
array []uint32
|
||||
bitmap []uint64
|
||||
exp uint64
|
||||
}{
|
||||
{
|
||||
array: []uint32{0},
|
||||
bitmap: []uint64{1},
|
||||
exp: 1,
|
||||
},
|
||||
{
|
||||
array: []uint32{0, 1},
|
||||
bitmap: []uint64{3},
|
||||
exp: 2,
|
||||
},
|
||||
{
|
||||
array: []uint32{64, 128, 129, 2000},
|
||||
bitmap: []uint64{932421, 2},
|
||||
exp: 0,
|
||||
},
|
||||
{
|
||||
array: []uint32{0, 65, 130, 195},
|
||||
bitmap: []uint64{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
|
||||
exp: 4,
|
||||
},
|
||||
{
|
||||
array: []uint32{63, 120, 543, 639, 12000},
|
||||
bitmap: []uint64{0x8000000000000000, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000000000000000},
|
||||
exp: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
a.array = test.array
|
||||
b.bitmap = test.bitmap
|
||||
ret1 := intersectionCountArrayBitmapOld(a, b)
|
||||
ret2 := intersectionCountArrayBitmap(a, b)
|
||||
if ret1 != ret2 || ret2 != test.exp {
|
||||
t.Fatalf("test #%v intersectCountArrayBitmap fail orig: %v new: %v exp: %v", i, ret1, ret2, test.exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,8 +137,25 @@ func TestBitmap_Xor_ArrayArray(t *testing.T) {
|
|||
if n := result.Count(); n != 2 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
//equivalence array test
|
||||
result = result.Xor(result)
|
||||
if n := result.Count(); n > 0 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//empty array test
|
||||
func TestBitmap_Xor_Empty(t *testing.T) {
|
||||
bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
|
||||
empty := roaring.NewBitmap()
|
||||
result := bm1.Xor(empty)
|
||||
|
||||
if n := result.Count(); n != 4 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
|
||||
bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098)
|
||||
bm1 := roaring.NewBitmap()
|
||||
|
|
@ -150,14 +167,28 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
|
|||
if n := result.Count(); n != 4999 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
//equivalence bitmap test
|
||||
result = result.Xor(result)
|
||||
if n := result.Count(); n > 0 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
|
||||
empty := roaring.NewBitmap()
|
||||
result = bm1.Xor(empty)
|
||||
if n := result.Count(); n != 5000 {
|
||||
t.Fatalf("unexpected n: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Xor_BitmapBitmap(t *testing.T) {
|
||||
bm0 := roaring.NewBitmap()
|
||||
bm1 := roaring.NewBitmap()
|
||||
|
||||
for i := uint64(0); i < 10000; i += 2 {
|
||||
bm1.Add(i)
|
||||
}
|
||||
|
||||
for i := uint64(1); i < 10000; i += 2 {
|
||||
bm0.Add(i)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ func (s *Server) Open() error {
|
|||
// Open HTTP listener to determine port (if specified as :0).
|
||||
ln, err := net.Listen("tcp", ":"+port)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("net.Listen: %v", err)
|
||||
}
|
||||
s.ln = ln
|
||||
|
||||
|
|
@ -115,16 +115,16 @@ func (s *Server) Open() error {
|
|||
|
||||
// Open holder.
|
||||
if err := s.Holder.Open(); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("opening Holder: %v", err)
|
||||
}
|
||||
|
||||
if err := s.BroadcastReceiver.Start(s); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("starting BroadcastReceiver: %v", err)
|
||||
}
|
||||
|
||||
// Open NodeSet communication
|
||||
if err := s.Cluster.NodeSet.Open(); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("opening NodeSet: %v", err)
|
||||
}
|
||||
|
||||
// Create executor for executing queries.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
class REPL {
|
||||
constructor(input, output, button) {
|
||||
constructor(input, output, button, completer) {
|
||||
this.input = input
|
||||
this.output = output
|
||||
this.button = button
|
||||
this.completer = completer
|
||||
this.history = []
|
||||
this.history_index = 0
|
||||
this.history_buffer = ''
|
||||
|
|
@ -16,21 +17,6 @@ class REPL {
|
|||
UP_ARROW: 38,
|
||||
DOWN_ARROW: 40
|
||||
}
|
||||
var keywords = {
|
||||
// keyword: length of substring that comes after cursor
|
||||
"SetBit()": 1,
|
||||
"ClearBit()": 1,
|
||||
"SetRowAttrs()": 1,
|
||||
"SetColumnAttrs()": 1,
|
||||
"Bitmap()": 1,
|
||||
"Union()": 1,
|
||||
"Intersect()": 1,
|
||||
"Difference()": 1,
|
||||
"Count()": 1,
|
||||
"Range()": 1,
|
||||
"TopN()": 1,
|
||||
"frame=": 0,
|
||||
}
|
||||
|
||||
this.input.addEventListener("keydown", function(e) {
|
||||
if (e.keyCode == keys.UP_ARROW) {
|
||||
|
|
@ -70,35 +56,7 @@ class REPL {
|
|||
}
|
||||
if (e.keyCode == keys.TAB) {
|
||||
e.preventDefault()
|
||||
|
||||
// extract word fragment ending at cursor. a word fragment:
|
||||
// - starts with last nonalpha character before cursor (or beginning of string)
|
||||
// - ends at cursor
|
||||
var word_start = repl.input.selectionEnd-1
|
||||
while(word_start>0) {
|
||||
var c = repl.input.value.charCodeAt(word_start)
|
||||
if(!((c>64 && c<91) || (c>96 && c<123))) {
|
||||
word_start++
|
||||
break
|
||||
}
|
||||
word_start--
|
||||
}
|
||||
var input_word = repl.input.value.substring(word_start, repl.input.selectionEnd)
|
||||
|
||||
// check for keyword match and insert
|
||||
// this just stops at the first match
|
||||
for(var keyword in keywords) {
|
||||
if(keyword.startsWith(input_word)){
|
||||
var cursor_pos = repl.input.selectionEnd
|
||||
var completion = keyword.substring(input_word.length)
|
||||
var before = repl.input.value.substring(0, cursor_pos)
|
||||
var after = repl.input.value.substring(cursor_pos)
|
||||
repl.input.value = before + completion + after
|
||||
var new_pos = cursor_pos + completion.length - keywords[keyword]
|
||||
repl.input.setSelectionRange(new_pos, new_pos)
|
||||
break
|
||||
}
|
||||
}
|
||||
repl.completer.complete()
|
||||
}
|
||||
})
|
||||
repl.button.onclick = function() {
|
||||
|
|
@ -203,10 +161,10 @@ class REPL {
|
|||
output_string += `<br />
|
||||
<br />
|
||||
Just getting started? Try this:<br />
|
||||
$ curl -XPOST "http://127.0.0.1:10101/index/test" -d '{"options": {"columnLabel": "col"}}' # create index "test"<br />
|
||||
$ curl -XPOST "http://127.0.0.1:10101/index/test/frame/foo" -d '{"options": {"rowLabel": "row"}}' # create frame "foo"<br />
|
||||
# Select "test" in the index dropdown above<br />
|
||||
SetBit(row=0, col=0, frame=foo) # Use PQL to set a bit
|
||||
:create index test<br />
|
||||
:use test<br />
|
||||
:create frame foo<br />
|
||||
SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit
|
||||
`
|
||||
}
|
||||
}
|
||||
|
|
@ -444,11 +402,94 @@ Date.prototype.timeNow = function () {
|
|||
|
||||
populate_version()
|
||||
|
||||
|
||||
class Autocompleter {
|
||||
constructor(input, output) {
|
||||
this.input = input
|
||||
this.output = output
|
||||
this.keyword_map = this.static_keywords
|
||||
this.init_dynamic_keywords()
|
||||
}
|
||||
|
||||
get static_keywords() {
|
||||
return {
|
||||
// keyword: length of substring that comes after cursor
|
||||
"SetBit()": 1,
|
||||
"ClearBit()": 1,
|
||||
"SetRowAttrs()": 1,
|
||||
"SetColumnAttrs()": 1,
|
||||
"Bitmap()": 1,
|
||||
"Union()": 1,
|
||||
"Intersect()": 1,
|
||||
"Difference()": 1,
|
||||
"Count()": 1,
|
||||
"Range()": 1,
|
||||
"TopN()": 1,
|
||||
"frame=": 0,
|
||||
}
|
||||
}
|
||||
|
||||
complete() {
|
||||
var completer = this
|
||||
// extract word fragment ending at cursor. a word fragment:
|
||||
// - starts with last nonalpha character before cursor (or beginning of string)
|
||||
// - ends at cursor
|
||||
var word_start = completer.input.selectionEnd-1
|
||||
while(word_start>0) {
|
||||
var c = completer.input.value.charCodeAt(word_start)
|
||||
if(!((c>64 && c<91) || (c>96 && c<123))) {
|
||||
word_start++
|
||||
break
|
||||
}
|
||||
word_start--
|
||||
}
|
||||
var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)
|
||||
|
||||
// check for keyword match and insert if exactly one match
|
||||
var matches = []
|
||||
for(var keyword in this.keyword_map) {
|
||||
if(keyword.startsWith(input_word)){
|
||||
matches.push(keyword)
|
||||
}
|
||||
}
|
||||
if(matches.length > 1) {
|
||||
// completer.output.innerHTML = whatever
|
||||
}
|
||||
|
||||
if(matches.length == 1) {
|
||||
// completer.output.innerHTML = ""
|
||||
var cursor_pos = completer.input.selectionEnd
|
||||
var completion = matches[0].substring(input_word.length)
|
||||
var before = completer.input.value.substring(0, cursor_pos)
|
||||
var after = completer.input.value.substring(cursor_pos)
|
||||
completer.input.value = before + completion + after
|
||||
var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]
|
||||
completer.input.setSelectionRange(new_pos, new_pos)
|
||||
}
|
||||
}
|
||||
|
||||
init_dynamic_keywords() {
|
||||
// hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list
|
||||
}
|
||||
|
||||
add_keyword() {
|
||||
// call when index or frame created in webui
|
||||
}
|
||||
|
||||
remove_keyword() {
|
||||
// call when index or frame deleted in webui
|
||||
// issue: if e.g. multiple indexes have same frame, removing one removes all.
|
||||
// solution: maintain count. requires more elaborate representation of keywords.
|
||||
}
|
||||
}
|
||||
|
||||
var input = document.getElementById('query')
|
||||
var output = document.getElementById('outputs')
|
||||
var button = document.getElementById('query-btn')
|
||||
var autocomplete_output = document.getElementById('autocomplete-container')
|
||||
|
||||
repl = new REPL(input, output, button)
|
||||
autocompleter = new Autocompleter(input, autocomplete_output)
|
||||
repl = new REPL(input, output, button, autocompleter)
|
||||
repl.populate_index_dropdown()
|
||||
repl.bind_events()
|
||||
|
||||
|
|
|
|||
|
|
@ -287,12 +287,6 @@ td{
|
|||
padding: 21px 30px;
|
||||
}
|
||||
|
||||
.string { color: green; }
|
||||
.number { color: darkorange; }
|
||||
.boolean { color: blue; }
|
||||
.null { color: magenta; }
|
||||
.key { color: red; }
|
||||
|
||||
.expand {
|
||||
text-align: center;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,11 +6,11 @@
|
|||
<link rel="stylesheet" href="/assets/style.css">
|
||||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||
<title>Pilosa WebUI</title>
|
||||
<link rel="icon" type="image/png" href="https://dc3kpxyuw05cb.cloudfront.net/img/favicon.png">
|
||||
<link rel="icon" type="image/png" href="https://www.pilosa.com/img/favicon.png">
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<img src="https://dc3kpxyuw05cb.cloudfront.net/img/logo.svg" width="110" alt="">
|
||||
<img src="https://www.pilosa.com/img/logo.svg" width="110" alt="">
|
||||
<div id="server-version" class=""></div>
|
||||
</div>
|
||||
<div class="container">
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
<h5>Special commands</h5>
|
||||
<div class="code">
|
||||
:create index test [columnLabel=column]<br />
|
||||
:use index test<br />
|
||||
:use test<br />
|
||||
:create frame foo [rowLabel=row]<br />
|
||||
:delete index test<br />
|
||||
:delete frame foo
|
||||
|
|
@ -76,19 +76,15 @@
|
|||
<button id='query-btn'>Run Query</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="output-container">
|
||||
|
||||
<h2>Output</h2>
|
||||
<div id="outputs">
|
||||
|
||||
<!-- results -->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="output-container">
|
||||
<h2>Output</h2>
|
||||
<div id="outputs">
|
||||
<!-- results -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="interface" id="interface-cluster">
|
||||
|
|
@ -103,10 +99,11 @@
|
|||
</div>
|
||||
</div>
|
||||
<div class="status-container">
|
||||
<!--
|
||||
<h2>Slice Distribution</h2>
|
||||
<div class="status-table-container" id="status-slices">
|
||||
...
|
||||
</div>
|
||||
-->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue