scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into slicecount-optimization

Browse source
This commit is contained in:
Todd Gruben 2018-06-19 10:36:05 -05:00
commit 1488ac0e3a
116 changed files with 10937 additions and 10692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.travis.yml
View file

@ -13,7 +13,7 @@ env:
- GOARCH=amd64
- GOARCH=amd64 ENTERPRISE=1
install:
- make install-dep install-statik vendor generate-statik
- make install-dep vendor
script:
- make test
before_deploy:
@ -27,9 +27,16 @@ deploy:
go: "1.10"
condition: $GOARCH = amd64
matrix:
# Excluding or allowing failures on non-primary matrix configurations due to long running times.
fast_finish: true
allow_failures:
- go: master
fast_finish: true
- go: 1.9
exclude:
- go: 1.9
env: GOARCH=386
- go: 1.9
env: GOARCH=amd64
notifications:
slack:
secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

8
CONTRIBUTING.md
View file

@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks:
make install
```
- Install build dependencies (dep, statik, and protoc):
- Install build dependencies (dep and protoc):
```sh
make install-build-deps
@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks:
make release
```
- Generate static assets for the WebUI:
```sh
make generate-statik
```
- Regenerate protocol buffer files in `internal/`:
```sh

2
Dockerfile
View file

@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
&& CGO_ENABLED=0 make install-dep install FLAGS="-a"
FROM scratch

28
Gopkg.lock generated
View file

@ -1,12 +1,6 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"
[[projects]]
branch = "master"
name = "github.com/CAFxX/gcnotifier"
@ -76,6 +70,18 @@
packages = ["proto"]
revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9"
[[projects]]
name = "github.com/google/go-cmp"
packages = [
"cmp",
"cmp/cmpopts",
"cmp/internal/diff",
"cmp/internal/function",
"cmp/internal/value"
]
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
version = "v0.2.0"
[[projects]]
name = "github.com/gorilla/context"
packages = ["."]
@ -212,14 +218,22 @@
[[projects]]
name = "github.com/shirou/gopsutil"
packages = [
"cpu",
"host",
"internal/common",
"mem",
"net",
"process"
]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
[[projects]]
branch = "master"
name = "github.com/shirou/w32"
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
@ -302,6 +316,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba"
inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5"
solver-name = "gps-cdcl"
solver-version = 1

16
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -88,16 +88,12 @@ install: vendor
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
# `go generate` statik assets (WebUI)
generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa
# `go generate` all needed packages
generate: generate-protoc generate-statik generate-stringer
generate: generate-protoc generate-stringer
# Create Docker image from Dockerfile
docker:
@ -126,23 +122,17 @@ endef
require-dep:
$(call require,dep)
require-statik:
$(call require,statik)
require-protoc-gen-gofast:
$(call require,protoc-gen-gofast)
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer
install-dep:
go get -u github.com/golang/dep/cmd/dep
install-statik:
go get -u github.com/rakyll/statik
install-stringer:
go get -u golang.org/x/tools/cmd/stringer

268
api.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
@ -45,7 +44,7 @@ type API struct {
BroadcastHandler BroadcastHandler
StatusHandler StatusHandler
Cluster *Cluster
RemoteClient *http.Client
TranslateStore TranslateStore
Logger Logger
}
@ -126,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if err != nil {
return resp, errors.Wrap(err, "reading column attrs")
}
// Translate column attributes, if necessary.
if api.TranslateStore != nil {
for _, col := range resp.ColumnAttrSets {
v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID)
if err != nil {
return resp, err
}
col.Key, col.ID = v, 0
}
}
resp.ColumnAttrSets = columnAttrSets
}
return resp, nil
@ -217,9 +228,9 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
return nil
}
// CreateFrame makes the named frame in the named index with the given options.
func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) {
if err := api.validate(apiCreateFrame); err != nil {
// CreateField makes the named field in the named index with the given options.
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, options FieldOptions) (*Field, error) {
if err := api.validate(apiCreateField); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -229,32 +240,32 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str
return nil, ErrIndexNotFound
}
// Create frame.
frame, err := index.CreateFrame(frameName, options)
// Create field.
field, err := index.CreateField(fieldName, options)
if err != nil {
return nil, errors.Wrap(err, "creating frame")
return nil, errors.Wrap(err, "creating field")
}
// Send the create frame message to all nodes.
// Send the create field message to all nodes.
err = api.Broadcaster.SendSync(
&internal.CreateFrameMessage{
&internal.CreateFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
Meta: options.Encode(),
})
if err != nil {
api.Logger.Printf("problem sending CreateFrame message: %s", err)
return nil, errors.Wrap(err, "sending CreateFrame message")
api.Logger.Printf("problem sending CreateField message: %s", err)
return nil, errors.Wrap(err, "sending CreateField message")
}
api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return frame, nil
api.Holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return field, nil
}
// DeleteFrame removes the named frame from the named index. If the index is not
// found, an error is returned. If the frame is not found, it is ignored and no
// DeleteField removes the named field from the named index. If the index is not
// found, an error is returned. If the field is not found, it is ignored and no
// action is taken.
func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error {
if err := api.validate(apiDeleteFrame); err != nil {
func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error {
if err := api.validate(apiDeleteField); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -264,40 +275,40 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str
return ErrIndexNotFound
}
// Delete frame from the index.
if err := index.DeleteFrame(frameName); err != nil {
return errors.Wrap(err, "deleting frame")
// Delete field from the index.
if err := index.DeleteField(fieldName); err != nil {
return errors.Wrap(err, "deleting field")
}
// Send the delete frame message to all nodes.
// Send the delete field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.DeleteFrameMessage{
&internal.DeleteFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
})
if err != nil {
api.Logger.Printf("problem sending DeleteFrame message: %s", err)
return errors.Wrap(err, "sending DeleteFrame message")
api.Logger.Printf("problem sending DeleteField message: %s", err)
return errors.Wrap(err, "sending DeleteField message")
}
api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
api.Holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return nil
}
// ExportCSV encodes the fragment designated by the index,frame,slice as
// ExportCSV encodes the fragment designated by the index,field,slice as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error {
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error {
if err := api.validate(apiExportCSV); err != nil {
return errors.Wrap(err, "validating api method")
}
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return ErrClusterDoesNotOwnSlice
}
// Find the fragment.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return ErrFragmentNotFound
}
@ -306,7 +317,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
cw := csv.NewWriter(w)
// Iterate over each column.
if err := f.ForEachBit(func(rowID, columnID uint64) error {
if err := f.forEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(rowID, 10),
strconv.FormatUint(columnID, 10),
@ -327,19 +338,19 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64)
return nil, errors.Wrap(err, "validating api method")
}
return api.Cluster.SliceNodes(indexName, slice), nil
return api.Cluster.sliceNodes(indexName, slice), nil
}
// MarshalFragment returns an object which can write the specified fragment's data
// to an io.Writer. The serialized data can be read back into a fragment with
// the UnmarshalFragment API call.
func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) {
func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) {
if err := api.validate(apiMarshalFragment); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -349,15 +360,15 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName
// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
// Reader which was previously written by MarshalFragment to populate the
// fragment's data.
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error {
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error {
if err := api.validate(apiUnmarshalFragment); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
// Retrieve field.
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Retrieve view.
@ -366,7 +377,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa
return errors.Wrap(err, "creating view")
}
// Retrieve fragment from frame.
// Retrieve fragment from field.
frag, err := view.CreateFragmentIfNotExists(slice)
if err != nil {
return errors.Wrap(err, "creating fragment")
@ -397,13 +408,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice)
f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice)
if f == nil {
return nil, ErrFragmentNotFound
}
var resp = internal.BlockDataResponse{}
resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block))
resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block))
// Encode response.
buf, err := proto.Marshal(&resp)
@ -415,13 +426,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) {
func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) {
if err := api.validate(apiFragmentBlocks); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -482,101 +493,22 @@ func (api *API) LocalID() string {
return api.Cluster.Node.ID
}
// Schema returns information about each index in Pilosa including which frames
// Schema returns information about each index in Pilosa including which fields
// and views they contain.
func (api *API) Schema(ctx context.Context) []*IndexInfo {
return api.Holder.Schema()
}
// CreateField creates a new BSI field in the given index and frame.
func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error {
if err := api.validate(apiCreateField); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame by name.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Create new field.
if err := f.CreateField(field); err != nil {
return errors.Wrap(err, "creating field")
}
// Send the create field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.CreateFieldMessage{
Index: indexName,
Frame: frameName,
Field: encodeField(field),
})
if err != nil {
api.Logger.Printf("problem sending CreateField message: %s", err)
}
return errors.Wrap(err, "sending CreateField message")
}
// DeleteField deletes the given field.
func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error {
if err := api.validate(apiDeleteField); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame by name.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Delete field.
if err := f.DeleteField(fieldName); err != nil {
return errors.Wrap(err, "deleting field")
}
// Send the delete field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.DeleteFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
})
if err != nil {
api.Logger.Printf("problem sending DeleteField message: %s", err)
}
return errors.Wrap(err, "sending DeleteField message")
}
// Fields returns the fields in the given frame.
func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) {
if err := api.validate(apiFields); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
index := api.Holder.index(indexName)
if index == nil {
return nil, ErrIndexNotFound
}
frame := index.frame(frameName)
if frame == nil {
return nil, ErrFrameNotFound
}
return frame.GetFields()
}
// Views returns the views in the given frame.
func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) {
// Views returns the views in the given field.
func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*View, error) {
if err := api.validate(apiViews); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve views.
f := api.Holder.Frame(indexName, frameName)
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
// Fetch views.
@ -585,15 +517,15 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) (
}
// DeleteView removes the given view.
func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error {
func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error {
if err := api.validate(apiDeleteView); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
// Retrieve field.
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Delete the view.
@ -608,7 +540,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri
err := api.Broadcaster.SendSync(
&internal.DeleteViewMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
View: viewName,
})
if err != nil {
@ -653,15 +585,15 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At
return attrs, nil
}
func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
if err := api.validate(apiFrameAttrDiff); err != nil {
func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
if err := api.validate(apiFieldAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve index from holder.
f := api.Holder.Frame(indexName, frameName)
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
// Retrieve local blocks.
@ -687,15 +619,15 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s
return attrs, nil
}
// Import bulk imports data into a particular index,frame,slice.
// Import bulk imports data into a particular index,field,slice.
func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
if err != nil {
return errors.Wrap(err, "getting frame")
return errors.Wrap(err, "getting field")
}
// Convert timestamps to time.Time.
@ -709,9 +641,9 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
}
// Import into fragment.
err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps)
err = field.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -722,15 +654,15 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
return errors.Wrap(err, "validating api method")
}
_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
if err != nil {
return errors.Wrap(err, "getting frame")
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values)
err = field.ImportValue(req.ColumnIDs, req.Values)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -758,28 +690,28 @@ func (api *API) LongQueryTime() time.Duration {
return api.Cluster.LongQueryTime
}
func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) {
func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) {
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
return nil, nil, ErrClusterDoesNotOwnSlice
}
// Find the Index.
api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice)
api.Logger.Printf("importing: %v %v %v", indexName, fieldName, slice)
index := api.Holder.Index(indexName)
if index == nil {
api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error())
api.Logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error())
return nil, nil, ErrIndexNotFound
}
// Retrieve frame.
frame := index.Frame(frameName)
if frame == nil {
api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error())
return nil, nil, ErrFrameNotFound
// Retrieve field.
field := index.Field(fieldName)
if field == nil {
api.Logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error())
return nil, nil, ErrFieldNotFound
}
return index, frame, nil
return index, field, nil
}
// SetCoordinator makes a new Node the cluster coordinator.
@ -788,15 +720,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode
return nil, nil, errors.Wrap(err, "validating api method")
}
oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.NodeByID(id)
oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator)
newNode = api.Cluster.nodeByID(id)
if newNode == nil {
return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node")
}
// If the new coordinator is this node, do the SetCoordinator directly.
if newNode.ID == api.LocalID() {
return oldNode, newNode, api.Cluster.SetCoordinator(newNode)
return oldNode, newNode, api.Cluster.setCoordinator(newNode)
}
// Send the set-coordinator message to new node.
@ -818,13 +750,13 @@ func (api *API) RemoveNode(id string) (*Node, error) {
return nil, errors.Wrap(err, "validating api method")
}
removeNode := api.Cluster.nodeByID(id)
removeNode := api.Cluster.unprotectedNodeByID(id)
if removeNode == nil {
return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove")
}
// Start the resize process (similar to NodeJoin)
err := api.Cluster.NodeLeave(removeNode)
err := api.Cluster.nodeLeave(removeNode)
if err != nil {
return removeNode, errors.Wrap(err, "calling node leave")
}
@ -837,7 +769,7 @@ func (api *API) ResizeAbort() error {
return errors.Wrap(err, "validating api method")
}
err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted)
err := api.Cluster.completeCurrentJob(resizeJobStateAborted)
return errors.Wrap(err, "complete current job")
}
@ -870,17 +802,14 @@ type apiMethod int
const (
apiClusterMessage apiMethod = iota
apiCreateField
apiCreateFrame
apiCreateIndex
apiDeleteField
apiDeleteFrame
apiDeleteIndex
apiDeleteView
apiExportCSV
apiFields
apiFragmentBlockData
apiFragmentBlocks
apiFrameAttrDiff
apiFieldAttrDiff
//apiHosts // not implemented
apiImport
apiImportValue
@ -916,17 +845,14 @@ var methodsResizing = map[apiMethod]struct{}{
var methodsNormal = map[apiMethod]struct{}{
apiCreateField: struct{}{},
apiCreateFrame: struct{}{},
apiCreateIndex: struct{}{},
apiDeleteField: struct{}{},
apiDeleteFrame: struct{}{},
apiDeleteIndex: struct{}{},
apiDeleteView: struct{}{},
apiExportCSV: struct{}{},
apiFields: struct{}{},
apiFragmentBlockData: struct{}{},
apiFragmentBlocks: struct{}{},
apiFrameAttrDiff: struct{}{},
apiFieldAttrDiff: struct{}{},
apiImport: struct{}{},
apiImportValue: struct{}{},
apiIndex: struct{}{},

8
apimethod_string.go
View file

@ -2,15 +2,15 @@
package pilosa
import "strconv"
import "fmt"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
return fmt.Sprintf("apiMethod(%d)", i)
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

79
attr.go
View file

@ -24,10 +24,10 @@ import (
// Attribute data type enum.
const (
AttrTypeString = 1
AttrTypeInt = 2
AttrTypeBool = 3
AttrTypeFloat = 4
attrTypeString = 1
attrTypeInt = 2
attrTypeBool = 3
attrTypeFloat = 4
)
// AttrStore represents an interface for handling row/column attributes.
@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
pb := &internal.Attr{Key: key}
switch value := value.(type) {
case string:
pb.Type = AttrTypeString
pb.Type = attrTypeString
pb.StringValue = value
case float64:
pb.Type = AttrTypeFloat
pb.Type = attrTypeFloat
pb.FloatValue = value
case uint64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = int64(value)
case int64:
pb.Type = AttrTypeInt
pb.Type = attrTypeInt
pb.IntValue = value
case bool:
pb.Type = AttrTypeBool
pb.Type = attrTypeBool
pb.BoolValue = value
}
return pb
@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr {
// decodeAttr converts from an Attr internal representation to a key/value pair.
func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
switch attr.Type {
case AttrTypeString:
case attrTypeString:
return attr.Key, attr.StringValue
case AttrTypeInt:
case attrTypeInt:
return attr.Key, attr.IntValue
case AttrTypeBool:
case attrTypeBool:
return attr.Key, attr.BoolValue
case AttrTypeFloat:
case attrTypeFloat:
return attr.Key, attr.FloatValue
default:
return attr.Key, nil
@ -221,3 +221,56 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) {
}
return decodeAttrs(pb.GetAttrs()), nil
}
func newMemAttrStore() AttrStore {
return &memAttrStore{
store: make(map[uint64]map[string]interface{}),
}
}
// memAttrStore represents an in-memory implementation of the AttrStore interface.
type memAttrStore struct {
store map[uint64]map[string]interface{}
}
// Path is an in-memory implementation of AttrStore Path method.
func (s *memAttrStore) Path() string { return "" }
// Open is an in-memory implementation of AttrStore Open method.
func (s *memAttrStore) Open() error {
return nil
}
// Close is an in-memory implementation of AttrStore Close method.
func (s *memAttrStore) Close() error {
return nil
}
// Attrs returns a set of attributes by ID.
func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return s.store[id], nil
}
// SetAttrs sets attribute values for a given ID.
func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
s.store[id] = m
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
for id, v := range m {
s.store[id] = v
}
return nil
}
// Blocks is an in-memory implementation of AttrStore Blocks method.
func (s *memAttrStore) Blocks() ([]AttrBlock, error) {
return nil, nil
}
// BlockData is an in-memory implementation of AttrStore BlockData method.
func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}

79
attr_test.go
View file

@ -15,15 +15,20 @@
package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) {
// Ensure database returns a non-nil empty map if unset.
func TestAttrStore_Attrs_Empty(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
if m, err := s.Attrs(100); err != nil {
@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
// Ensure database can unset attributes if explicitly set to nil.
func TestAttrStore_Attrs_Unset(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
// Ensure attribute block checksums can be returned.
func TestAttrStore_Blocks(t *testing.T) {
s := test.MustOpenAttrStore()
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) {
t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
}
}
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

108
broadcast.go
View file

@ -120,23 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error {
// Broadcast message types.
const (
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateFrame
MessageTypeDeleteFrame
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
MessageTypeSetCoordinator
MessageTypeUpdateCoordinator
MessageTypeNodeState
MessageTypeRecalculateCaches
MessageTypeNodeEvent
messageTypeCreateSlice = iota
messageTypeCreateIndex
messageTypeDeleteIndex
messageTypeCreateField
messageTypeDeleteField
messageTypeCreateView
messageTypeDeleteView
messageTypeClusterStatus
messageTypeResizeInstruction
messageTypeResizeInstructionComplete
messageTypeSetCoordinator
messageTypeUpdateCoordinator
messageTypeNodeState
messageTypeRecalculateCaches
messageTypeNodeEvent
)
// MarshalMessage encodes the protobuf message into a byte slice.
@ -144,39 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
var typ uint8
switch obj := m.(type) {
case *internal.CreateSliceMessage:
typ = MessageTypeCreateSlice
typ = messageTypeCreateSlice
case *internal.CreateIndexMessage:
typ = MessageTypeCreateIndex
typ = messageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = MessageTypeDeleteIndex
case *internal.CreateFrameMessage:
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
typ = MessageTypeDeleteFrame
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
typ = messageTypeDeleteIndex
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
typ = messageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
typ = messageTypeDeleteField
case *internal.CreateViewMessage:
typ = messageTypeCreateView
case *internal.DeleteViewMessage:
typ = messageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
typ = messageTypeClusterStatus
case *internal.ResizeInstruction:
typ = MessageTypeResizeInstruction
typ = messageTypeResizeInstruction
case *internal.ResizeInstructionComplete:
typ = MessageTypeResizeInstructionComplete
typ = messageTypeResizeInstructionComplete
case *internal.SetCoordinatorMessage:
typ = MessageTypeSetCoordinator
typ = messageTypeSetCoordinator
case *internal.UpdateCoordinatorMessage:
typ = MessageTypeUpdateCoordinator
typ = messageTypeUpdateCoordinator
case *internal.NodeStateMessage:
typ = MessageTypeNodeState
typ = messageTypeNodeState
case *internal.RecalculateCaches:
typ = MessageTypeRecalculateCaches
typ = messageTypeRecalculateCaches
case *internal.NodeEventMessage:
typ = MessageTypeNodeEvent
typ = messageTypeNodeEvent
default:
return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
}
@ -193,39 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
var m proto.Message
switch typ {
case MessageTypeCreateSlice:
case messageTypeCreateSlice:
m = &internal.CreateSliceMessage{}
case MessageTypeCreateIndex:
case messageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
case MessageTypeDeleteIndex:
case messageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
case MessageTypeCreateFrame:
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:
m = &internal.DeleteFrameMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeCreateField:
case messageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
case messageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeClusterStatus:
case messageTypeCreateView:
m = &internal.CreateViewMessage{}
case messageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case messageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:
case messageTypeResizeInstruction:
m = &internal.ResizeInstruction{}
case MessageTypeResizeInstructionComplete:
case messageTypeResizeInstructionComplete:
m = &internal.ResizeInstructionComplete{}
case MessageTypeSetCoordinator:
case messageTypeSetCoordinator:
m = &internal.SetCoordinatorMessage{}
case MessageTypeUpdateCoordinator:
case messageTypeUpdateCoordinator:
m = &internal.UpdateCoordinatorMessage{}
case MessageTypeNodeState:
case messageTypeNodeState:
m = &internal.NodeStateMessage{}
case MessageTypeRecalculateCaches:
case messageTypeRecalculateCaches:
m = &internal.RecalculateCaches{}
case MessageTypeNodeEvent:
case messageTypeNodeEvent:
m = &internal.NodeEventMessage{}
default:
return nil, fmt.Errorf("invalid message type: %d", typ)

8
cache.go
View file

@ -27,8 +27,8 @@ import (
)
const (
// ThresholdFactor is used to calculate the threshold for new items entering the cache
ThresholdFactor = 1.1
// thresholdFactor is used to calculate the threshold for new items entering the cache
thresholdFactor = 1.1
)
// Cache represents a cache of counts.
@ -158,7 +158,7 @@ type RankCache struct {
func NewRankCache(maxEntries uint32) *RankCache {
return &RankCache{
maxEntries: maxEntries,
thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
entries: make(map[uint64]uint64),
stats: NopStatsClient,
}
@ -409,7 +409,7 @@ func (p Pairs) String() string {
return buf.String()
}
func encodePairs(a Pairs) []*internal.Pair {
func EncodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])

1127
client.go
View file

File diff suppressed because it is too large Load diff

475
cluster.go
View file

File diff suppressed because it is too large Load diff

488
cluster_internal_test.go
View file

@ -15,11 +15,15 @@
package pilosa
import (
"bytes"
"io/ioutil"
"math/rand"
"reflect"
"strings"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa/internal"
)
@ -45,13 +49,13 @@ func TestFragCombos(t *testing.T) {
tests := []struct {
idx string
maxSlice uint64
frameViews viewsByFrame
fieldViews viewsByField
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
fieldViews: viewsByField{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
@ -60,7 +64,7 @@ func TestFragCombos(t *testing.T) {
{
idx: "foo",
maxSlice: uint64(3),
frameViews: viewsByFrame{"f": []string{"v0"}},
fieldViews: viewsByField{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
@ -69,7 +73,7 @@ func TestFragCombos(t *testing.T) {
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
@ -145,23 +149,23 @@ func TestFragSources(t *testing.T) {
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
field, err := idx.CreateFieldIfNotExists("f", FieldOptions{})
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 101, nil)
_, err = field.SetBit("standard", 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 1300000, nil)
_, err = field.SetBit("standard", 1, 1300000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 2600000, nil)
_, err = field.SetBit("standard", 1, 2600000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 3900000, nil)
_, err = field.SetBit("standard", 1, 3900000, nil)
if err != nil {
t.Fatal(err)
}
@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) {
{
existingNodes: []*Node{node0, node1},
node: node2,
action: ResizeJobActionAdd,
action: resizeJobActionAdd,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false},
},
{
existingNodes: []*Node{node0, node1, node2},
node: node2,
action: ResizeJobActionRemove,
action: resizeJobActionRemove,
expectedIDs: map[string]bool{node0.ID: false, node1.ID: false},
},
}
for _, test := range tests {
actual := NewResizeJob(test.existingNodes, test.node, test.action)
actual := newResizeJob(test.existingNodes, test.node, test.action)
if err != nil {
t.Fatal(err)
}
@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) {
}
}
}
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.containsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
// NEXT: move this test to internal and unexport IsCoordinator
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.isCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.isCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.addNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.addNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.nodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.updateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.updateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

528
cluster_test.go
View file

@ -1,528 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := Cluster{
Nodes: []*Node{
{URI: NewTestURIFromHostPort("serverA", 1000)},
{URI: NewTestURIFromHostPort("serverB", 1000)},
{URI: NewTestURIFromHostPort("serverC", 1000)},
},
Hasher: NewTestModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0], _ = quick.Value(reflect.TypeOf(""), rand)
values[1] = reflect.ValueOf(uint64(rand.Uint32()))
values[2] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// Ensure the hasher can hash correctly.
func TestHasher(t *testing.T) {
for _, tt := range []struct {
key uint64
bucket []int
}{
// Generated from the reference C++ code
{0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
{1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}},
{0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}},
{0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}},
} {
for i, v := range tt.bucket {
if got := NewHasher().Hash(tt.key, i+1); got != v {
t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v)
}
}
}
}
// Ensure OwnsSlices can find the actual slice list for node and index.
func TestCluster_OwnsSlices(t *testing.T) {
c := NewTestCluster(5)
slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0))
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// Ensure ContainsSlices can find the actual slice list for node and index.
func TestCluster_ContainsSlices(t *testing.T) {
c := NewTestCluster(5)
c.ReplicaN = 3
slices := c.ContainsSlices("test", 10, c.Nodes[2])
if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
func TestCluster_Nodes(t *testing.T) {
uri0 := NewTestURIFromHostPort("node0", 0)
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
uri3 := NewTestURIFromHostPort("node3", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
node3 := &Node{ID: "node3", URI: uri3}
nodes := []*Node{node0, node1, node2}
t.Run("NodeIDs", func(t *testing.T) {
actual := Nodes(nodes).IDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Filter", func(t *testing.T) {
actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("FilterURI", func(t *testing.T) {
actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs()
expected := []URI{uri0, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("Contains", func(t *testing.T) {
actualTrue := Nodes(nodes).Contains(node1)
actualFalse := Nodes(nodes).Contains(node3)
if !reflect.DeepEqual(actualTrue, true) {
t.Errorf("expected: %v, but got: %v", true, actualTrue)
}
if !reflect.DeepEqual(actualFalse, false) {
t.Errorf("expected: %v, but got: %v", false, actualTrue)
}
})
t.Run("Clone", func(t *testing.T) {
clone := Nodes(nodes).Clone()
actual := Nodes(clone).URIs()
expected := []URI{uri0, uri1, uri2}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
}
func TestCluster_Coordinator(t *testing.T) {
uri1 := NewTestURIFromHostPort("node1", 0)
uri2 := NewTestURIFromHostPort("node2", 0)
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
c1 := *NewCluster()
c1.Node = node1
c1.Coordinator = node1.ID
c2 := *NewCluster()
c2.Node = node2
c2.Coordinator = node1.ID
t.Run("IsCoordinator", func(t *testing.T) {
if !c1.IsCoordinator() {
t.Errorf("!IsCoordinator error: %v", c1.Node)
} else if c2.IsCoordinator() {
t.Errorf("IsCoordinator error: %v", c2.Node)
}
})
}
func TestCluster_Topology(t *testing.T) {
c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
uri0 := NewTestURIFromHostPort("host0", 0)
uri1 := NewTestURIFromHostPort("host1", 0)
uri2 := NewTestURIFromHostPort("host2", 0)
invalid := NewTestURIFromHostPort("invalid", 0)
node0 := &Node{ID: "node0", URI: uri0}
node1 := &Node{ID: "node1", URI: uri1}
node2 := &Node{ID: "node2", URI: uri2}
nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid}
t.Run("AddNode", func(t *testing.T) {
err := c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
// add the same host.
err = c1.AddNode(node1)
if err != nil {
t.Fatal(err)
}
err = c1.AddNode(node2)
if err != nil {
t.Fatal(err)
}
actual := c1.NodeIDs()
expected := []string{node0.ID, node1.ID, node2.ID}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("expected: %v, but got: %v", expected, actual)
}
})
t.Run("ContainsID", func(t *testing.T) {
if !c1.Topology.ContainsID(node1.ID) {
t.Errorf("!ContainsHost error: %v", node1.ID)
} else if c1.Topology.ContainsID(nodeinvalid.ID) {
t.Errorf("ContainsHost error: %v", nodeinvalid.ID)
}
})
}
// Ensure that general cluster functionality works as expected.
func TestCluster_ResizeStates(t *testing.T) {
t.Run("Single node, no data", func(t *testing.T) {
tc := NewClusterCluster(1)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
node := tc.Clusters[0]
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
expectedTop := &Topology{
NodeIDs: []string{node.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{node.Node.ID},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node comes up in state NORMAL.
if node.State() != ClusterStateNormal {
t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Single node, not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"some-other-host"},
}
tc.WriteTopology(node.Path, top)
// Open TestCluster.
expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]"
err := tc.Open()
if err == nil || err.Error() != expected {
t.Errorf("did not receive expected error: %s", expected)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, no data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
tc.AddNode(false)
node0 := tc.Clusters[0]
node1 := tc.Clusters[1]
// Ensure that nodes comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// write topology to data file
top := &Topology{
NodeIDs: []string{"node0", "node2"},
}
tc.WriteTopology(node0.Path, top)
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Ensure that node is in state STARTING before the other node joins.
if node0.State() != ClusterStateStarting {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State())
}
// Expect an error by adding a node not in the topology.
expectedError := "host is not in topology: node1"
err := tc.AddNode(false)
if err == nil || err.Error() != expectedError {
t.Errorf("did not receive expected error: %s", expectedError)
}
tc.AddNode(false)
node2 := tc.Clusters[2]
// Ensure that node comes up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node2.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State())
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
t.Run("Multiple nodes, with data", func(t *testing.T) {
tc := NewClusterCluster(0)
tc.AddNode(false)
node0 := tc.Clusters[0]
// Open TestCluster.
if err := tc.Open(); err != nil {
t.Fatal(err)
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
Fields: []*Field{
{
Name: "fld0",
Type: FieldTypeInt,
Min: -100,
Max: 100,
},
},
}); err != nil {
t.Fatal(err)
}
tc.SetFieldValue("i", "fields", 1, "fld0", -10)
tc.SetFieldValue("i", "fields", 1, "fld0", 10)
tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Frame := node0.Holder.Frame("i", "f")
node0View := node0Frame.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
node0Frame = node0.Holder.Frame("i", "fields")
node0View = node0Frame.View("field_fld0")
node0Fragment = node0View.Fragment(1)
node0ChecksumFld := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
if node0.State() != ClusterStateNormal {
t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State())
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
expectedTop := &Topology{
NodeIDs: []string{node0.Node.ID, node1.Node.ID},
}
// Verify topology file.
if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs)
} else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Values
// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
node1Frame = node1.Holder.Frame("i", "fields")
node1View = node1Frame.View("field_fld0")
node1Fragment = node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}
// Ensures that coordinator can be changed.
func TestCluster_UpdateCoordinator(t *testing.T) {
t.Run("UpdateCoordinator", func(t *testing.T) {
c := NewTestCluster(2)
oldNode := c.Nodes[0]
newNode := c.Nodes[1]
// Update coordinator to the same value.
if c.UpdateCoordinator(oldNode) {
t.Errorf("did not expect coordinator to change")
} else if c.Coordinator != oldNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI)
}
// Update coordinator to a new value.
if !c.UpdateCoordinator(newNode) {
t.Errorf("expected coordinator to change")
} else if c.Coordinator != newNode.ID {
t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI)
}
})
}

57
cmd/bench.go
View file

@ -1,57 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
)
var Bencher *ctl.BenchCommand
func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr)
benchCmd := &cobra.Command{
Use: "bench",
Short: "Benchmark operations.",
Long: `
Executes a benchmark for a given operation against the index.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Bencher.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := benchCmd.Flags()
flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")
ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify)
return benchCmd
}
func init() {
subcommandFns["bench"] = NewBenchCommand
}

54
cmd/bench_test.go
View file

@ -1,54 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestBenchHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "bench", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa bench") || err != nil {
t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestBenchConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"bench", "--operation", "set-bit"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Bencher.Host, "localhost:12345")
v.Check(cmd.Bencher.Index, "myindex")
v.Check(cmd.Bencher.Frame, "f1")
v.Check(cmd.Bencher.Op, "set-bit")
v.Check(cmd.Bencher.N, 0)
return v.Error()
},
},
}
executeDry(t, tests)
}

2
cmd/export.go
View file

@ -52,7 +52,7 @@ The file does not contain any headers.
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)

4
cmd/export_test.go
View file

@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) {
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
field = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.Field, "f1")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()
},

11
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and frame. The data
Long: `Bulk imports one or more CSV files to a host's index and field. The data
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -54,15 +54,14 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags := importCmd.Flags()
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.")
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field")
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)
return importCmd

4
cmd/import_test.go
View file

@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) {
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
field = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Importer.Host, "localhost:12345")
v.Check(cmd.Importer.Index, "myindex")
v.Check(cmd.Importer.Frame, "f1")
v.Check(cmd.Importer.Field, "f1")
return v.Error()
},
},

116
ctl/bench.go
View file

@ -1,116 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"fmt"
"io"
"math/rand"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BenchCommand represents a command for benchmarking index operations.
type BenchCommand struct {
// Destination host and port.
Host string
// Name of the index & frame to execute against.
Index string
Frame string
// Type of operation and number to execute.
Op string
N int
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewBenchCommand returns a new instance of BenchCommand.
func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
return &BenchCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the bench command.
func (cmd *BenchCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
switch cmd.Op {
case "set-bit":
return cmd.runSetBit(ctx, client)
case "":
return errors.New("op required")
default:
return fmt.Errorf("unknown bench op: %q", cmd.Op)
}
}
// runSetBit executes a benchmark of random SetBit() operations.
func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error {
if cmd.N == 0 {
return errors.New("operation count required")
} else if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
}
const maxRowID = 1000
const maxColumnID = 100000
startTime := time.Now()
// Execute operation continuously.
for i := 0; i < cmd.N; i++ {
rowID := rand.Intn(maxRowID)
columnID := rand.Intn(maxColumnID)
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID),
Remote: false,
}
if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil {
return err
}
}
// Print results.
elapsed := time.Since(startTime)
fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds())
return nil
}
func (cmd *BenchCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

98
ctl/bench_test.go
View file

@ -1,98 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"os"
"testing"
"github.com/pilosa/pilosa"
"github.com/pkg/errors"
)
func TestBenchCommand_InvalidOption(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewBenchCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
if errors.Cause(err) != pilosa.ErrHostRequired {
t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err)
}
cm.Host = "localhost:10101"
err = cm.Run(context.Background())
if err.Error() != "op required" {
t.Fatalf("Expect err: %s, actual err: %s", "op required", err)
}
cm.Op = "test"
err = cm.Run(context.Background())
if err.Error() != "unknown bench op: \"test\"" {
t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err)
}
}
func TestBenchCommand_Run(t *testing.T) {
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewBenchCommand(stdin, w, w)
cm.Op = "set-bit"
cm.Host = "localhost:10101"
err := cm.Run(context.Background())
if err.Error() != "operation count required" {
t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err)
}
cm.N = 1
err = cm.Run(context.Background())
if err != pilosa.ErrIndexRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err)
}
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err)
}
cm.Frame = "f"
err = cm.Run(context.Background())
w.Close()
var buf bytes.Buffer
io.Copy(&buf, r)
fmt.Println(buf.String())
if err != nil {
fmt.Println(buf.String())
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

6
ctl/common.go
View file

@ -17,7 +17,7 @@ package ctl
import (
"crypto/tls"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
"github.com/spf13/pflag"
@ -37,7 +37,7 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP
}
// CommandClient returns a pilosa.InternalHTTPClient for the command
func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) {
func CommandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) {
tlsConfig := cmd.TLSConfiguration()
var TLSConfig *tls.Config
if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" {
@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error
InsecureSkipVerify: tlsConfig.SkipVerify,
}
}
client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig))
client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig))
if err != nil {
return nil, errors.Wrap(err, "getting internal client")
}

10
ctl/export.go
View file

@ -30,9 +30,9 @@ type ExportCommand struct {
// Remote host and port.
Host string
// Name of the index & frame to export from.
// Name of the index & field to export from.
Index string
Frame string
Field string
// Filename to export to.
Path string
@ -57,8 +57,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if cmd.Field == "" {
return pilosa.ErrFieldRequired
}
// Use output file, if specified.
@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

8
ctl/export_test.go
View file

@ -38,8 +38,8 @@ func TestExportCommand_Validation(t *testing.T) {
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
if err != pilosa.ErrFieldRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
}
}
@ -59,10 +59,10 @@ func TestExportCommand_Run(t *testing.T) {
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader("")))
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}

33
ctl/generate_config.go
View file

@ -18,9 +18,11 @@ import (
"context"
"fmt"
"io"
"strings"
"github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// GenerateConfigCommand represents a command for printing a default config.
@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera
// Run prints out the default config.
func (cmd *GenerateConfigCommand) Run(ctx context.Context) error {
fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
data-dir = "~/.pilosa"
bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
replicas = 1
hosts = [
"localhost:10101",
]
[anti-entropy]
interval = "10m0s"
[profile]
cpu = ""
cpu-time = "30s"
[metric]
service = "statsd"
host = "127.0.0.1:8125"
poll-interval = "0m15s"
`)+"\n")
conf := server.NewConfig()
ret, err := toml.Marshal(*conf)
if err != nil {
return errors.Wrap(err, "unmarshaling default config")
}
fmt.Fprintf(cmd.Stdout, "%s\n", ret)
return nil
}

2
ctl/generate_config_test.go
View file

@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
io.Copy(&buf, r)
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), "localhost:10101") {
} else if !strings.Contains(buf.String(), ":10101") {
t.Fatalf("Unexpected config: %s", buf.String())
}
}

80
ctl/import.go
View file

@ -26,6 +26,7 @@ import (
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
@ -35,20 +36,17 @@ type ImportCommand struct {
// Destination host and port.
Host string `json:"host"`
// Name of the index & frame to import into.
// Name of the index & field to import into.
Index string `json:"index"`
Frame string `json:"frame"`
Field string `json:"field"`
// Options for index & frame to be created if they don't exist
// Options for index & field to be created if they don't exist
IndexOptions pilosa.IndexOptions
FrameOptions pilosa.FrameOptions
FieldOptions pilosa.FieldOptions
// CreateSchema ensures the schema exists before import
CreateSchema bool
// For Range-Encoded fields, name of the Field to import into.
Field string `json:"field"`
// Indicates that the payload should be treated as string keys.
StringKeys bool `json:"StringKeys"`
@ -83,11 +81,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Validate arguments.
// Index and frame are validated early before the files are parsed.
// Index and field are validated early before the files are parsed.
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if cmd.Field == "" {
return pilosa.ErrFieldRequired
} else if len(cmd.Paths) == 0 {
return errors.New("path required")
}
@ -105,10 +103,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
}
}
// Determine the field type in order to correctly handle the input data.
fieldType := pilosa.DefaultFieldType
schema, err := cmd.Client.Schema(ctx)
if err != nil {
return errors.Wrap(err, "getting schema")
}
for _, index := range schema {
if index.Name == cmd.Index {
for _, field := range index.Fields {
if field.Name == cmd.Field {
fieldType = field.Options.Type
}
}
}
}
// Import each path and import by slice.
for _, path := range cmd.Paths {
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, path); err != nil {
if err := cmd.importPath(ctx, fieldType, path); err != nil {
return err
}
}
@ -121,18 +135,18 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
if err != nil {
return fmt.Errorf("Error Creating Index: %s", err)
}
err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions)
err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field, cmd.FieldOptions)
if err != nil {
return fmt.Errorf("Error Creating Frame: %s", err)
return fmt.Errorf("Error Creating Field: %s", err)
}
return nil
}
// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
// If a field is provided, treat the import data as values to be range-encoded.
if cmd.Field != "" {
return cmd.bufferFieldValues(ctx, path)
func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error {
// If fieldType is `int`, treat the import data as values to be range-encoded.
if fieldType == pilosa.FieldTypeInt {
return cmd.bufferValues(ctx, path)
} else {
if cmd.StringKeys {
return cmd.bufferBitsK(ctx, path)
@ -232,16 +246,16 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
// Group bits by slice.
logger.Printf("grouping %d bits", len(bits))
bitsBySlice := pilosa.Bits(bits).GroupBySlice()
bitsBySlice := http.Bits(bits).GroupBySlice()
// Parse path into bits.
for slice, chunk := range bitsBySlice {
if cmd.Sort {
sort.Sort(pilosa.BitsByPos(chunk))
sort.Sort(http.BitsByPos(chunk))
}
logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil {
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}
@ -338,15 +352,15 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
// TODO: does it help to sort the rowKeys?
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil {
return errors.Wrap(err, "importing keys")
}
return nil
}
// bufferFieldValues buffers slices of fieldValues to be imported as a batch.
func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error {
// bufferValues buffers slices of FieldValues to be imported as a batch.
func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error {
a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
var r *csv.Reader
@ -394,7 +408,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
}
val.ColumnID = columnID
// Parse field value.
// Parse FieldValue.
value, err := strconv.ParseInt(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
@ -403,9 +417,9 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
a = append(a, val)
// If we've reached the buffer size then import field values.
// If we've reached the buffer size then import FieldValues.
if len(a) == cmd.BufferSize {
if err := cmd.importFieldValues(ctx, a); err != nil {
if err := cmd.importValues(ctx, a); err != nil {
return err
}
a = a[:0]
@ -413,29 +427,29 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
}
// If there are still values in the buffer then flush them.
if err := cmd.importFieldValues(ctx, a); err != nil {
if err := cmd.importValues(ctx, a); err != nil {
return err
}
return nil
}
// importFieldValues sends batches of fieldValues to the server.
func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error {
// importValues sends batches of FieldValues to the server.
func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Group vals by slice.
logger.Printf("grouping %d vals", len(vals))
valsBySlice := pilosa.FieldValues(vals).GroupBySlice()
valsBySlice := http.FieldValues(vals).GroupBySlice()
// Parse path into field values.
// Parse path into FieldValues.
for slice, vals := range valsBySlice {
if cmd.Sort {
sort.Sort(pilosa.FieldValues(vals))
sort.Sort(http.FieldValues(vals))
}
logger.Printf("importing slice: %d, n=%d", slice, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil {
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil {
return errors.Wrap(err, "importing values")
}
}

39
ctl/import_test.go
View file

@ -15,6 +15,7 @@
package ctl
import (
"bufio"
"bytes"
"context"
"io"
@ -38,11 +39,11 @@ func TestImportCommand_Validation(t *testing.T) {
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
if err != pilosa.ErrFieldRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
}
cm.Frame = "f"
cm.Field = "f"
err = cm.Run(context.Background())
if err.Error() != "path required" {
t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err)
@ -72,7 +73,7 @@ func TestImportCommand_Run(t *testing.T) {
cm.Host = s.Host()
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
cm.CreateSchema = true
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -81,8 +82,7 @@ func TestImportCommand_Run(t *testing.T) {
}
}
// Ensure that the ImportValue path runs (note: we have specified a value
// for cm.Field.)
// Ensure that the ImportValue path runs.
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
@ -106,11 +106,10 @@ func TestImportCommand_RunValue(t *testing.T) {
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
cm.Index = "i"
cm.Frame = "f"
cm.Field = "foo"
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
@ -120,12 +119,21 @@ func TestImportCommand_RunValue(t *testing.T) {
func TestImportCommand_InvalidFile(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = "anyhost"
cm.Host = s.Host()
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
file, err := ioutil.TempFile("", "import.csv")
file.Write([]byte("a,2\n3,5\n5,6"))
if err != nil {
@ -180,3 +188,12 @@ func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
}
return req
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

3
ctl/server.go
View file

@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.")
flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.")
// Translation
flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.")
// Gossip
flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")
flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.")

14
diagnostics.go
View file

@ -217,7 +217,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
numFrames := 0
numFields := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
@ -225,19 +225,19 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
for _, index := range d.server.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
for _, field := range index.Fields() {
numFields += 1
if field.Type() == FieldTypeInt {
bsiFieldCount += 1
}
if frame.TimeQuantum() != "" {
if field.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumFields", numFields)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)

4
docs/getting-started.md
View file

@ -112,10 +112,6 @@ Note that both the user IDs and the repository IDs were remapped to sequential i
#### Make Some Queries
<div class="note">
<p>Note the Pilosa server comes with a <a href="../webui/">WebUI</a> for constructing queries in a browser. In local development, it is available at <a href="http://localhost:10101">localhost:10101</a>.</p>
</div>
Which repositories did user 14 star:
``` request
curl localhost:10101/index/repository/query \

11
docs/installation.md
View file

@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -144,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -173,7 +170,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -262,7 +258,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -298,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -327,7 +321,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.

22
docs/webui.md
View file

@ -9,13 +9,27 @@ nav = [
## WebUI
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status.
This can be used for constructing queries and viewing the cluster status.
### Installation
Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
Installing on a Mac with Homebrew is simple; just run:
```
brew install pilosa-webui
```
You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running:
```
make install
```
### Console
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
@ -39,4 +53,4 @@ Frame creation also supports options like `timeQuantum`. When creating a new fra
### Cluster Admin
Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.
Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.

554
executor.go
View file

File diff suppressed because it is too large Load diff

746
executor_test.go
View file

File diff suppressed because it is too large Load diff

1258
field.go Normal file
View file

File diff suppressed because it is too large Load diff

149
field_internal_test.go Normal file
View file

@ -0,0 +1,149 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"reflect"
"testing"
"github.com/pilosa/pilosa/pql"
)
// Ensure a bsiGroup can adjust to its baseValue.
func TestBSIGroup_BaseValue(t *testing.T) {
b0 := &bsiGroup{
Name: "b0",
Type: bsiGroupTypeInt,
Min: -100,
Max: 900,
}
b1 := &bsiGroup{
Name: "b1",
Type: bsiGroupTypeInt,
Min: 0,
Max: 1000,
}
b2 := &bsiGroup{
Name: "b2",
Type: bsiGroupTypeInt,
Min: 100,
Max: 1100,
}
t.Run("Normal Condition", func(t *testing.T) {
for _, tt := range []struct {
f *bsiGroup
op pql.Token
val int64
expBaseValue uint64
expOutOfRange bool
}{
// LT
{b0, pql.LT, 5, 105, false},
{b0, pql.LT, -8, 92, false},
{b0, pql.LT, -108, 0, true},
{b0, pql.LT, 1005, 1000, false},
{b0, pql.LT, 0, 100, false},
{b1, pql.LT, 5, 5, false},
{b1, pql.LT, -8, 0, true},
{b1, pql.LT, 1005, 1000, false},
{b1, pql.LT, 0, 0, false},
{b2, pql.LT, 5, 0, true},
{b2, pql.LT, -8, 0, true},
{b2, pql.LT, 105, 5, false},
{b2, pql.LT, 1105, 1000, false},
// GT
{b0, pql.GT, -105, 0, false},
{b0, pql.GT, 5, 105, false},
{b0, pql.GT, 905, 0, true},
{b0, pql.GT, 0, 100, false},
{b1, pql.GT, 5, 5, false},
{b1, pql.GT, -8, 0, false},
{b1, pql.GT, 1005, 0, true},
{b1, pql.GT, 0, 0, false},
{b2, pql.GT, 5, 0, false},
{b2, pql.GT, -8, 0, false},
{b2, pql.GT, 105, 5, false},
{b2, pql.GT, 1105, 0, true},
// EQ
{b0, pql.EQ, -105, 0, true},
{b0, pql.EQ, 5, 105, false},
{b0, pql.EQ, 905, 0, true},
{b0, pql.EQ, 0, 100, false},
{b1, pql.EQ, 5, 5, false},
{b1, pql.EQ, -8, 0, true},
{b1, pql.EQ, 1005, 0, true},
{b1, pql.EQ, 0, 0, false},
{b2, pql.EQ, 5, 0, true},
{b2, pql.EQ, -8, 0, true},
{b2, pql.EQ, 105, 5, false},
{b2, pql.EQ, 1105, 0, true},
} {
bv, oor := tt.f.baseValue(tt.op, tt.val)
if oor != tt.expOutOfRange {
t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
}
}
})
t.Run("Betwween Condition", func(t *testing.T) {
for _, tt := range []struct {
f *bsiGroup
predMin int64
predMax int64
expBaseValueMin uint64
expBaseValueMax uint64
expOutOfRange bool
}{
{b0, -205, -105, 0, 0, true},
{b0, -105, 80, 0, 180, false},
{b0, 5, 20, 105, 120, false},
{b0, 20, 1005, 120, 1000, false},
{b0, 1005, 2000, 0, 0, true},
{b1, -105, -5, 0, 0, true},
{b1, -5, 20, 0, 20, false},
{b1, 5, 20, 5, 20, false},
{b1, 20, 1005, 20, 1000, false},
{b1, 1005, 2000, 0, 0, true},
{b2, 5, 95, 0, 0, true},
{b2, 95, 120, 0, 20, false},
{b2, 105, 120, 5, 20, false},
{b2, 120, 1105, 20, 1000, false},
{b2, 1105, 2000, 0, 0, true},
} {
min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax)
if oor != tt.expOutOfRange {
t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
}
}
})
}

287
field_test.go Normal file
View file

@ -0,0 +1,287 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure field can open and retrieve a view.
func TestField_CreateViewIfNotExists(t *testing.T) {
f := test.MustOpenField()
defer f.Close()
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
// Ensure field can set its time quantum.
func TestField_SetTimeQuantum(t *testing.T) {
fo := pilosa.FieldOptions{
Type: "time",
}
f := test.MustOpenField(pilosa.OptFieldFieldOptions(fo))
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value on field.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100); err != nil {
t.Fatal(err)
} else if value != 21 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
// Setting value should return no change.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
}
})
t.Run("Overwrite", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set different value.
if changed, err := f.SetValue(100, 23); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100); err != nil {
t.Fatal(err)
} else if value != 23 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
})
t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeSet,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestField_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
field, err := pilosa.NewField(path, "i", ".meta")
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
}
// Ensure that field name validation is consistent.
func TestField_NameValidation(t *testing.T) {
validFieldNames := []string{
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
}
invalidFieldNames := []string{
"",
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"camelCase",
"UPPERCASE",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
for _, name := range validFieldNames {
_, err := pilosa.NewField(path, "i", name)
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
_, err := pilosa.NewField(path, "i", name)
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}
}
}
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := test.MustOpenField()
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in field")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}

337
fragment.go
View file

@ -25,7 +25,6 @@ import (
"hash"
"io"
"io/ioutil"
"net/http"
"os"
"sort"
"sync"
@ -48,31 +47,29 @@ const (
// SliceWidth is the number of column IDs in a slice.
SliceWidth = 1048576
// SnapshotExt is the file extension used for an in-process snapshot.
SnapshotExt = ".snapshotting"
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
// CopyExt is the file extension used for the temp file used while copying.
CopyExt = ".copying"
// copyExt is the file extension used for the temp file used while copying.
copyExt = ".copying"
// CacheExt is the file extension for persisted cache ids.
CacheExt = ".cache"
// cacheExt is the file extension for persisted cache ids.
cacheExt = ".cache"
// HashBlockSize is the number of rows in a merkle hash block.
HashBlockSize = 100
// defaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
defaultFragmentMaxOpN = 2000
)
const (
// DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN.
DefaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a frame and slice in an index.
// Fragment represents the intersection of a field and slice in an index.
type Fragment struct {
mu sync.RWMutex
// Composite identifiers
index string
frame string
field string
view string
slice uint64
@ -84,7 +81,7 @@ type Fragment struct {
opN int // number of ops since snapshot
// Cache for row counts.
CacheType string // passed in by frame
CacheType string // passed in by field
cache Cache
CacheSize uint32
@ -106,51 +103,32 @@ type Fragment struct {
Logger Logger
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
// This is set by the parent field unless overridden for testing.
RowAttrStore AttrStore
stats StatsClient
}
// NewFragment returns a new instance of Fragment.
func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
func NewFragment(path, index, field, view string, slice uint64) *Fragment {
return &Fragment{
path: path,
index: index,
frame: frame,
field: field,
view: view,
slice: slice,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
CacheType: defaultCacheType,
CacheSize: defaultCacheSize,
Logger: NopLogger,
MaxOpN: DefaultFragmentMaxOpN,
MaxOpN: defaultFragmentMaxOpN,
stats: NopStatsClient,
}
}
// Path returns the path the fragment was initialized with.
func (f *Fragment) Path() string { return f.path }
// CachePath returns the path to the fragment's cache data.
func (f *Fragment) CachePath() string { return f.path + CacheExt }
// Index returns the index that the fragment was initialized with.
func (f *Fragment) Index() string { return f.index }
// Frame returns the frame the fragment was initialized with.
func (f *Fragment) Frame() string { return f.frame }
// View returns the view the fragment was initialized with.
func (f *Fragment) View() string { return f.view }
// Slice returns the slice the fragment was initialized with.
func (f *Fragment) Slice() uint64 { return f.slice }
// Cache returns the fragment's cache.
// This is not safe for concurrent use.
func (f *Fragment) Cache() Cache { return f.cache }
// cachePath returns the path to the fragment's cache data.
func (f *Fragment) cachePath() string { return f.path + cacheExt }
// Open opens the underlying storage.
func (f *Fragment) Open() error {
@ -247,7 +225,7 @@ func (f *Fragment) openStorage() error {
// openCache initializes the cache from row ids persisted to disk.
func (f *Fragment) openCache() error {
// Determine cache type from frame name.
// Determine cache type from field name.
switch f.CacheType {
case CacheTypeRanked:
f.cache = NewRankCache(f.CacheSize)
@ -261,7 +239,7 @@ func (f *Fragment) openCache() error {
}
// Read cache data from disk.
path := f.CachePath()
path := f.cachePath()
buf, err := ioutil.ReadFile(path)
if os.IsNotExist(err) {
return nil
@ -342,14 +320,14 @@ func (f *Fragment) closeStorage() error {
return nil
}
// Row returns a row by ID.
func (f *Fragment) Row(rowID uint64) *Row {
// row returns a row by ID.
func (f *Fragment) row(rowID uint64) *Row {
f.mu.Lock()
defer f.mu.Unlock()
return f.row(rowID, true, true)
return f.unprotectedRow(rowID, true, true)
}
func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row {
func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row {
if checkRowCache {
r, ok := f.rowCache.Fetch(rowID)
if ok && r != nil {
@ -380,15 +358,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R
return row
}
// SetBit sets a bit for a given column & row within the fragment.
// setBit sets a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.setBit(rowID, columnID)
return f.unprotectedSetBit(rowID, columnID)
}
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
@ -415,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
}
// Get the row from row cache or fragment.storage.
row := f.row(rowID, true, true)
row := f.unprotectedRow(rowID, true, true)
row.SetBit(columnID)
// Update the cache.
@ -432,15 +410,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
return changed, nil
}
// ClearBit clears a bit for a given column & row within the fragment.
// clearBit clears a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.clearBit(rowID, columnID)
return f.unprotectedClearBit(rowID, columnID)
}
func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
@ -467,7 +445,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
}
// Get the row from cache or fragment.storage.
row := f.row(rowID, true, true)
row := f.unprotectedRow(rowID, true, true)
row.ClearBit(columnID)
// Update the cache.
@ -486,8 +464,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
return f.storage.Contains(pos), nil
}
// FieldValue uses a column of bits to read a multi-bit value.
func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// value uses a column of bits to read a multi-bit value.
func (f *Fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -510,20 +488,20 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi
return value, true, nil
}
// SetFieldValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// setValue uses a column of bits to set a multi-bit value.
func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
for i := uint(0); i < bitDepth; i++ {
if value&(1<<i) != 0 {
if c, err := f.setBit(uint64(i), columnID); err != nil {
if c, err := f.unprotectedSetBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
}
} else {
if c, err := f.clearBit(uint64(i), columnID); err != nil {
if c, err := f.unprotectedClearBit(uint64(i), columnID); err != nil {
return changed, err
} else if c {
changed = true
@ -532,7 +510,7 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
}
// Mark value as set.
if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
if c, err := f.unprotectedSetBit(uint64(bitDepth), columnID); err != nil {
return changed, errors.Wrap(err, "marking not-null")
} else if c {
changed = true
@ -541,8 +519,8 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
return changed, nil
}
// importSetFieldValue is a more efficient SetFieldValue just for imports.
func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// importSetValue is a more efficient SetValue just for imports.
func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
for i := uint(0); i < bitDepth; i++ {
if value&(1<<i) != 0 {
@ -582,11 +560,11 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
return changed, nil
}
// FieldSum returns the sum of a given field as well as the number of columns involved.
// sum returns the sum of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
func (f *Fragment) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence row.
row := f.Row(uint64(bitDepth))
row := f.row(uint64(bitDepth))
if filter != nil {
count = row.IntersectionCount(filter)
} else {
@ -601,7 +579,7 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
row := f.Row(uint64(i))
row := f.row(uint64(i))
cnt := uint64(0)
if filter != nil {
cnt = row.IntersectionCount(filter)
@ -614,11 +592,11 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err
return sum, count, nil
}
// FieldMin returns the min of a given field as well as the number of columns involved.
// min returns the min of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) {
func (f *Fragment) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -630,7 +608,7 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
row := f.row(uint64(ii))
x := consider.Difference(row)
count = x.Count()
@ -647,11 +625,11 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
return min, count, nil
}
// FieldMax returns the max of a given field as well as the number of columns involved.
// max returns the max of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) {
func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.row(uint64(bitDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -663,7 +641,7 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
row := f.row(uint64(ii))
x := row.Intersect(consider)
count = x.Count()
@ -678,29 +656,29 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
switch op {
case pql.EQ:
return f.fieldRangeEQ(bitDepth, predicate)
return f.rangeEQ(bitDepth, predicate)
case pql.NEQ:
return f.fieldRangeNEQ(bitDepth, predicate)
return f.rangeNEQ(bitDepth, predicate)
case pql.LT, pql.LTE:
return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
return f.rangeLT(bitDepth, predicate, op == pql.LTE)
case pql.GT, pql.GTE:
return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE)
return f.rangeGT(bitDepth, predicate, op == pql.GTE)
default:
return nil, ErrInvalidRangeOperation
}
}
func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
if bit == 1 {
@ -713,12 +691,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
return b, nil
}
func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Get the equal bitmap.
eq, err := f.fieldRangeEQ(bitDepth, predicate)
eq, err := f.rangeEQ(bitDepth, predicate)
if err != nil {
return nil, err
}
@ -729,16 +707,16 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error)
return b, nil
}
func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
keep := NewRow()
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.row(uint64(bitDepth))
// Filter any bits that don't match the current bit value.
leadingZeros := true
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
// Remove any columns with higher bits set.
@ -777,13 +755,13 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.Row(uint64(bitDepth))
func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.row(uint64(bitDepth))
keep := NewRow()
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit := (predicate >> uint(i)) & 1
// Handle last bit differently.
@ -812,20 +790,20 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
// FieldNotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) {
return f.Row(uint64(bitDepth)), nil
// notNull returns the not-null row (stored at bitDepth).
func (f *Fragment) notNull(bitDepth uint) (*Row, error) {
return f.row(uint64(bitDepth)), nil
}
// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.Row(uint64(bitDepth))
// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.row(uint64(bitDepth))
keep1 := NewRow() // GTE
keep2 := NewRow() // LTE
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
row := f.row(uint64(i))
bit1 := (predicateMin >> uint(i)) & 1
bit2 := (predicateMax >> uint(i)) & 1
@ -864,12 +842,12 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
if columnID < minColumnID || columnID >= minColumnID+SliceWidth {
return 0, errors.New("column out of bounds")
}
return Pos(rowID, columnID), nil
return pos(rowID, columnID), nil
}
// ForEachBit executes fn for every bit set in the fragment.
// forEachBit executes fn for every bit set in the fragment.
// Errors returned from fn are passed through.
func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -886,10 +864,10 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
return err
}
// Top returns the top rows from the fragment.
// top returns the top rows from the fragment.
// If opt.Src is specified then only rows which intersect src are returned.
// If opt.FilterValues exist then the row attribute specified by field is matched.
func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
func (f *Fragment) top(opt TopOptions) ([]Pair, error) {
// Retrieve pairs. If no row ids specified then return from cache.
pairs := f.topBitmapPairs(opt.RowIDs)
@ -900,7 +878,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Create a fast lookup of filter values.
var filters map[interface{}]struct{}
if opt.FilterField != "" && len(opt.FilterValues) > 0 {
if opt.FilterName != "" && len(opt.FilterValues) > 0 {
filters = make(map[interface{}]struct{})
for _, v := range opt.FilterValues {
filters[v] = struct{}{}
@ -948,7 +926,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
return nil, errors.Wrap(err, "getting attrs")
} else if attr == nil {
continue
} else if attrValue := attr[opt.FilterField]; attrValue == nil {
} else if attrValue := attr[opt.FilterName]; attrValue == nil {
continue
} else if _, ok := filters[attrValue]; !ok {
continue
@ -960,7 +938,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Calculate count and append.
count := cnt
if opt.Src != nil {
count = opt.Src.IntersectionCount(f.Row(rowID))
count = opt.Src.IntersectionCount(f.row(rowID))
}
if count == 0 {
continue
@ -1004,7 +982,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
count := opt.Src.IntersectionCount(f.row(rowID))
if count < threshold {
continue
}
@ -1048,7 +1026,7 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair {
continue
}
row := f.Row(rowID)
row := f.row(rowID)
if row.Count() > 0 {
// Otherwise load from storage.
pairs = append(pairs, BitmapPair{
@ -1074,7 +1052,7 @@ type TopOptions struct {
MinThreshold uint64
// Filter field name & values.
FilterField string
FilterName string
FilterValues []interface{}
TanimotoThreshold uint64
}
@ -1089,13 +1067,6 @@ func (f *Fragment) Checksum() []byte {
return h.Sum(nil)
}
// BlockN returns the number of blocks in the fragment.
func (f *Fragment) BlockN() int {
f.mu.Lock()
defer f.mu.Unlock()
return int(f.storage.Max() / (HashBlockSize * SliceWidth))
}
// InvalidateChecksums clears all cached block checksums.
func (f *Fragment) InvalidateChecksums() {
f.mu.Lock()
@ -1184,8 +1155,8 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
}
}
// BlockData returns bits in a block as row & column ID pairs.
func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
// blockData returns bits in a block as row & column ID pairs.
func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) {
f.mu.Lock()
defer f.mu.Unlock()
@ -1196,17 +1167,17 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
return
}
// MergeBlock compares the block's bits and computes a diff with another set of block bits.
// mergeBlock compares the block's bits and computes a diff with another set of block bits.
// The state of a bit is determined by consensus from all blocks being considered.
//
// For example, if 3 blocks are compared and two have a set bit and one has a
// cleared bit then the bit is considered cleared. The function returns the
// diff per incoming block so that all can be in sync.
func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) {
func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) {
// Ensure that all pair sets are of equal length.
for i := range data {
if len(data[i].RowIDs) != len(data[i].ColumnIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs))
if len(data[i].rowIDs) != len(data[i].columnIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs))
}
}
@ -1214,8 +1185,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
defer f.mu.Unlock()
// Track sets and clears for all blocks (including local).
sets = make([]PairSet, len(data)+1)
clears = make([]PairSet, len(data)+1)
sets = make([]pairSet, len(data)+1)
clears = make([]pairSet, len(data)+1)
// Limit upper row/column pair.
maxRowID := uint64(id+1) * HashBlockSize
@ -1231,7 +1202,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Append buffered iterators for each incoming block.
for i := range data {
var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs)
var itr Iterator = NewSliceIterator(data[i].rowIDs, data[i].columnIDs)
itr = NewLimitIterator(itr, maxRowID, maxColumnID)
itrs = append(itrs, NewBufIterator(itr))
}
@ -1296,25 +1267,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Append to either the set or clear diff.
if newValue {
sets[i].RowIDs = append(sets[i].RowIDs, min.rowID)
sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
sets[i].rowIDs = append(sets[i].rowIDs, min.rowID)
sets[i].columnIDs = append(sets[i].columnIDs, min.columnID)
} else {
clears[i].RowIDs = append(sets[i].RowIDs, min.rowID)
clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
clears[i].rowIDs = append(sets[i].rowIDs, min.rowID)
clears[i].columnIDs = append(sets[i].columnIDs, min.columnID)
}
}
}
// Set local bits.
for i := range sets[0].ColumnIDs {
if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil {
for i := range sets[0].columnIDs {
if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "setting")
}
}
// Clear local bits.
for i := range clears[0].ColumnIDs {
if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil {
for i := range clears[0].columnIDs {
if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "clearing")
}
}
@ -1322,9 +1293,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
return sets[1:], clears[1:], nil
}
// Import bulk imports a set of bits and then snapshots the storage.
// bulkImport bulk imports a set of bits and then snapshots the storage.
// This does not affect the fragment's cache.
func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of row ids and column ids.
@ -1373,7 +1344,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
// Import should ALWAYS have row() load a new row from fragment.storage
// because the row that's in rowCache hasn't been updated with
// this import's data.
f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count())
f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count())
}
f.cache.Invalidate()
@ -1392,8 +1363,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
return nil
}
// ImportValue bulk imports a set of range-encoded values.
func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error {
// importValue bulk imports a set of range-encoded values.
func (f *Fragment) importValue(columnIDs, values []uint64, bitDepth uint) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of column ids and values.
@ -1408,7 +1379,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error
for i := range columnIDs {
columnID, value := columnIDs[i], values[i]
_, err := f.importSetFieldValue(columnID, bitDepth, value)
_, err := f.importSetValue(columnID, bitDepth, value)
if err != nil {
return errors.Wrap(err, "setting")
}
@ -1452,13 +1423,13 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) {
}
func (f *Fragment) snapshot() error {
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
start := time.Now()
defer track(start, completeMessage, f.stats, f.Logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
snapshotPath := f.path + snapshotExt
file, err := os.Create(snapshotPath)
if err != nil {
return fmt.Errorf("create snapshot file: %s", err)
@ -1529,7 +1500,7 @@ func (f *Fragment) flushCache() error {
}
// Write to disk.
if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil {
if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil {
return errors.Wrap(err, "writing")
}
@ -1603,7 +1574,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error {
defer f.mu.Unlock()
// Read cache into buffer.
buf, err := ioutil.ReadFile(f.CachePath())
buf, err := ioutil.ReadFile(f.cachePath())
if os.IsNotExist(err) {
return nil
} else if err != nil {
@ -1662,7 +1633,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) {
func (f *Fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path + CopyExt
path := f.path + copyExt
file, err := os.Create(path)
if err != nil {
return errors.Wrap(err, "creating directory")
@ -1697,7 +1668,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error {
buf, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "reading")
} else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil {
} else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil {
return errors.Wrap(err, "writing")
}
@ -1745,9 +1716,8 @@ func (h *blockHasher) WriteValue(v uint64) {
type FragmentSyncer struct {
Fragment *Fragment
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
Closing <-chan struct{}
}
@ -1762,11 +1732,11 @@ func (s *FragmentSyncer) isClosing() bool {
}
}
// SyncFragment compares checksums for the local and remote fragments and
// syncFragment compares checksums for the local and remote fragments and
// then merges any blocks which have differences.
func (s *FragmentSyncer) SyncFragment() error {
func (s *FragmentSyncer) syncFragment() error {
// Determine replica set.
nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice())
nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice)
if len(nodes) == 1 {
return nil
}
@ -1782,8 +1752,7 @@ func (s *FragmentSyncer) SyncFragment() error {
}
// Retrieve remote blocks.
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice())
blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice)
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -1846,9 +1815,9 @@ func (s *FragmentSyncer) syncBlock(id int) error {
f := s.Fragment
// Read pairs from each remote block.
var pairSets []PairSet
var clients []InternalClient
for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) {
var uris []*URI
var pairSets []pairSet
for _, node := range s.Cluster.sliceNodes(f.index, f.slice) {
if s.Node.ID == node.ID {
continue
}
@ -1858,18 +1827,18 @@ func (s *FragmentSyncer) syncBlock(id int) error {
return nil
}
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
clients = append(clients, client)
uri := &node.URI
uris = append(uris, uri)
// Only sync the standard block.
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id)
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id)
if err != nil {
return errors.Wrap(err, "getting block")
}
pairSets = append(pairSets, PairSet{
ColumnIDs: columnIDs,
RowIDs: rowIDs,
pairSets = append(pairSets, pairSet{
columnIDs: columnIDs,
rowIDs: rowIDs,
})
}
@ -1879,23 +1848,23 @@ func (s *FragmentSyncer) syncBlock(id int) error {
}
// Merge blocks together.
sets, clears, err := f.MergeBlock(id, pairSets)
sets, clears, err := f.mergeBlock(id, pairSets)
if err != nil {
return errors.Wrap(err, "merging")
}
// Write updates to remote blocks.
for i := 0; i < len(clients); i++ {
for i := 0; i < len(uris); i++ {
set, clear := sets[i], clears[i]
count := 0
// Ignore if there are no differences.
if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 {
if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 {
continue
}
// Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest.
total := len(set.ColumnIDs) + len(clear.ColumnIDs)
total := len(set.columnIDs) + len(clear.columnIDs)
maxWrites := s.Cluster.MaxWritesPerRequest
if maxWrites <= 0 {
maxWrites = 5000
@ -1903,12 +1872,12 @@ func (s *FragmentSyncer) syncBlock(id int) error {
buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites))))
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
for j := 0; j < len(set.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
for j := 0; j < len(clear.columnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j])
count++
}
@ -1924,7 +1893,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
Query: buffers[k].String(),
Remote: true,
}
_, err := clients[i].Query(context.Background(), f.Index(), queryRequest)
_, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest)
if err != nil {
return errors.Wrap(err, "executing")
}
@ -1942,10 +1911,10 @@ func madvise(b []byte, advice int) (err error) {
return
}
// PairSet is a list of equal length row and column id lists.
type PairSet struct {
RowIDs []uint64
ColumnIDs []uint64
// pairSet is a list of equal length row and column id lists.
type pairSet struct {
rowIDs []uint64
columnIDs []uint64
}
// byteSlicesEqual returns true if all slices are equal.
@ -1962,7 +1931,7 @@ func byteSlicesEqual(a [][]byte) bool {
return true
}
// Pos returns the row position of a row/column pair.
func Pos(rowID, columnID uint64) uint64 {
// pos returns the row position of a row/column pair.
func pos(rowID, columnID uint64) uint64 {
return (rowID * SliceWidth) + (columnID % SliceWidth)
}

583
fragment_test.go → fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

1156
frame.go
View file

File diff suppressed because it is too large Load diff

423
frame_test.go
View file

@ -1,423 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"reflect"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
// Ensure frame can open and retrieve a view.
func TestFrame_CreateViewIfNotExists(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
// Ensure frame can set its time quantum.
func TestFrame_SetTimeQuantum(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload frame and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure a frame can set & read a field value.
func TestFrame_SetFieldValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
},
})
if err != nil {
t.Fatal(err)
}
// Set value on first field.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set value on same column but different field.
if changed, err := f.SetFieldValue(100, "field1", 25); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.FieldValue(100, "field0"); err != nil {
t.Fatal(err)
} else if value != 21 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
// Setting value should return no change.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
}
})
t.Run("Overwrite", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set different value.
if changed, err := f.SetFieldValue(100, "field0", 23); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.FieldValue(100, "field0"); err != nil {
t.Fatal(err)
} else if value != 23 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldValueTooLow", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldValueTooHigh", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestFrame_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", ".meta")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
}
// Ensure that frame name validation is consistent.
func TestFrame_NameValidation(t *testing.T) {
validFrameNames := []string{
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
}
invalidFrameNames := []string{
"",
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"camelCase",
"UPPERCASE",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
for _, name := range validFrameNames {
_, err := pilosa.NewFrame(path, "i", name)
if err != nil {
t.Fatalf("unexpected frame name: %s %s", name, err)
}
}
for _, name := range invalidFrameNames {
_, err := pilosa.NewFrame(path, "i", name)
if err == nil {
t.Fatalf("expected error on frame name: %s", name)
}
}
}
// Ensure frame can open and retrieve a view.
func TestFrame_DeleteView(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in frame")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}
// Ensure a field can adjust to its baseValue.
func TestField_BaseValue(t *testing.T) {
f0 := &pilosa.Field{
Name: "f0",
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 900,
}
f1 := &pilosa.Field{
Name: "f1",
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}
f2 := &pilosa.Field{
Name: "f2",
Type: pilosa.FieldTypeInt,
Min: 100,
Max: 1100,
}
t.Run("Normal Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
op pql.Token
val int64
expBaseValue uint64
expOutOfRange bool
}{
// LT
{f0, pql.LT, 5, 105, false},
{f0, pql.LT, -8, 92, false},
{f0, pql.LT, -108, 0, true},
{f0, pql.LT, 1005, 1000, false},
{f0, pql.LT, 0, 100, false},
{f1, pql.LT, 5, 5, false},
{f1, pql.LT, -8, 0, true},
{f1, pql.LT, 1005, 1000, false},
{f1, pql.LT, 0, 0, false},
{f2, pql.LT, 5, 0, true},
{f2, pql.LT, -8, 0, true},
{f2, pql.LT, 105, 5, false},
{f2, pql.LT, 1105, 1000, false},
// GT
{f0, pql.GT, -105, 0, false},
{f0, pql.GT, 5, 105, false},
{f0, pql.GT, 905, 0, true},
{f0, pql.GT, 0, 100, false},
{f1, pql.GT, 5, 5, false},
{f1, pql.GT, -8, 0, false},
{f1, pql.GT, 1005, 0, true},
{f1, pql.GT, 0, 0, false},
{f2, pql.GT, 5, 0, false},
{f2, pql.GT, -8, 0, false},
{f2, pql.GT, 105, 5, false},
{f2, pql.GT, 1105, 0, true},
// EQ
{f0, pql.EQ, -105, 0, true},
{f0, pql.EQ, 5, 105, false},
{f0, pql.EQ, 905, 0, true},
{f0, pql.EQ, 0, 100, false},
{f1, pql.EQ, 5, 5, false},
{f1, pql.EQ, -8, 0, true},
{f1, pql.EQ, 1005, 0, true},
{f1, pql.EQ, 0, 0, false},
{f2, pql.EQ, 5, 0, true},
{f2, pql.EQ, -8, 0, true},
{f2, pql.EQ, 105, 5, false},
{f2, pql.EQ, 1105, 0, true},
} {
bv, oor := tt.f.BaseValue(tt.op, tt.val)
if oor != tt.expOutOfRange {
t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
}
}
})
t.Run("Betwween Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
predMin int64
predMax int64
expBaseValueMin uint64
expBaseValueMax uint64
expOutOfRange bool
}{
{f0, -205, -105, 0, 0, true},
{f0, -105, 80, 0, 180, false},
{f0, 5, 20, 105, 120, false},
{f0, 20, 1005, 120, 1000, false},
{f0, 1005, 2000, 0, 0, true},
{f1, -105, -5, 0, 0, true},
{f1, -5, 20, 0, 20, false},
{f1, 5, 20, 5, 20, false},
{f1, 20, 1005, 20, 1000, false},
{f1, 1005, 2000, 0, 0, true},
{f2, 5, 95, 0, 0, true},
{f2, 95, 120, 0, 20, false},
{f2, 105, 120, 5, 20, false},
{f2, 120, 1105, 20, 1000, false},
{f2, 1105, 2000, 0, 0, true},
} {
min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax)
if oor != tt.expOutOfRange {
t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
}
}
})
}

67
gossip/gossip.go
View file

@ -18,13 +18,12 @@ import (
"fmt"
"io/ioutil"
"log"
"net"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/sync/errgroup"
"github.com/gogo/protobuf/proto"
"github.com/hashicorp/memberlist"
"github.com/pilosa/pilosa"
@ -35,7 +34,6 @@ import (
// Ensure GossipMemberSet implements interfaces.
var _ pilosa.BroadcastReceiver = &GossipMemberSet{}
var _ pilosa.Gossiper = &GossipMemberSet{}
var _ memberlist.Delegate = &GossipMemberSet{}
// GossipMemberSet represents a gossip implementation of MemberSet using memberlist.
@ -213,7 +211,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
conf.BindAddr = host
conf.BindPort = port
conf.AdvertisePort = port
conf.AdvertiseAddr = pilosa.HostToIP(host)
conf.AdvertiseAddr = hostToIP(host)
//
conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
conf.SuspicionMult = cfg.SuspicionMult
@ -239,49 +237,6 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe
return g, nil
}
// SendSync implementation of the Broadcaster interface.
func (g *GossipMemberSet) SendSync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
mlist := g.memberlist
// Direct sends the message directly to every node.
// An error from any node raises an error on the entire operation.
//
// Gossip uses the gossip protocol to eventually deliver the message
// to every node.
var eg errgroup.Group
for _, n := range mlist.Members() {
// Don't send the message to the local node.
if n == mlist.LocalNode() {
continue
}
node := n
eg.Go(func() error {
return mlist.SendToTCP(node, msg)
})
}
return eg.Wait()
}
// SendAsync implementation of the Gossiper interface.
func (g *GossipMemberSet) SendAsync(pb proto.Message) error {
msg, err := pilosa.MarshalMessage(pb)
if err != nil {
return fmt.Errorf("marshal message: %s", err)
}
b := &broadcast{
msg: msg,
notify: nil,
}
g.broadcasts.QueueBroadcast(b)
return nil
}
// NodeMeta implementation of the memberlist.Delegate interface.
func (g *GossipMemberSet) NodeMeta(limit int) []byte {
buf, err := proto.Marshal(pilosa.EncodeNode(g.node))
@ -580,3 +535,21 @@ type Config struct {
Nodes int `toml:"nodes"`
ToTheDeadTime toml.Duration `toml:"to-the-dead-time"`
}
// hostToIP converts host to an IP4 address based on net.LookupIP().
func hostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}

1300
handler.go
View file

File diff suppressed because it is too large Load diff

143
holder.go
View file

@ -18,7 +18,6 @@ import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
@ -34,8 +33,8 @@ import (
)
const (
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
DefaultCacheFlushInterval = 1 * time.Minute
// defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
defaultCacheFlushInterval = 1 * time.Minute
// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
FileLimit = 262144 // (512^2)
@ -84,7 +83,7 @@ func NewHolder() *Holder {
NewAttrStore: NewNopAttrStore,
CacheFlushInterval: DefaultCacheFlushInterval,
CacheFlushInterval: defaultCacheFlushInterval,
Logger: NopLogger,
}
@ -112,7 +111,8 @@ func (h *Holder) Open() error {
}
for _, fi := range fis {
if !fi.IsDir() {
// Skip files or hidden directories.
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
continue
}
@ -209,20 +209,20 @@ func (h *Holder) MaxSlices() map[string]uint64 {
return a
}
// Schema returns schema information for all indexes, frames, and views.
// Schema returns schema information for all indexes, fields, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.name})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
di.Fields = append(di.Fields, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
sort.Sort(fieldInfoSlice(di.Fields))
a = append(a, di)
}
sort.Sort(indexInfoSlice(a))
@ -238,16 +238,16 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
if err != nil {
return errors.Wrap(err, "creating index")
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
// Create fields that don't exist.
for _, f := range index.Fields {
opt := decodeFieldOptions(f.Meta)
field, err := idx.CreateFieldIfNotExists(f.Name, *opt)
if err != nil {
return errors.Wrap(err, "creating frame")
return errors.Wrap(err, "creating field")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
_, err := field.CreateViewIfNotExists(v)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -339,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
return nil, errors.Wrap(err, "creating")
}
index.keys = opt.Keys
if err := index.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
} else if err := index.saveMeta(); err != nil {
return nil, errors.Wrap(err, "meta")
}
// Update options.
h.indexes[index.Name()] = index
return index, nil
@ -390,27 +393,27 @@ func (h *Holder) DeleteIndex(name string) error {
return nil
}
// Frame returns the frame for an index and name.
func (h *Holder) Frame(index, name string) *Frame {
// Field returns the field for an index and name.
func (h *Holder) Field(index, name string) *Field {
idx := h.Index(index)
if idx == nil {
return nil
}
return idx.Frame(name)
return idx.Field(name)
}
// View returns the view for an index, frame, and name.
func (h *Holder) View(index, frame, name string) *View {
f := h.Frame(index, frame)
// View returns the view for an index, field, and name.
func (h *Holder) View(index, field, name string) *View {
f := h.Field(index, field)
if f == nil {
return nil
}
return f.View(name)
}
// Fragment returns the fragment for an index, frame & slice.
func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment {
v := h.View(index, frame, view)
// Fragment returns the fragment for an index, field & slice.
func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment {
v := h.View(index, field, view)
if v == nil {
return nil
}
@ -435,9 +438,9 @@ func (h *Holder) monitorCacheFlush() {
func (h *Holder) flushCaches() {
for _, index := range h.Indexes() {
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.allFragments() {
select {
case <-h.closing:
return
@ -445,7 +448,7 @@ func (h *Holder) flushCaches() {
}
if err := fragment.FlushCache(); err != nil {
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath())
}
}
}
@ -563,9 +566,8 @@ func (h *Holder) logStartup() error {
type HolderSyncer struct {
Holder *Holder
Node *Node
Cluster *Cluster
RemoteClient *http.Client
Node *Node
Cluster *Cluster
// Stats
Stats StatsClient
@ -600,15 +602,15 @@ func (s *HolderSyncer) SyncHolder() error {
}
tf := time.Now()
for _, fi := range di.Frames {
for _, fi := range di.Fields {
// Verify syncer has not closed.
if s.IsClosing() {
return nil
}
// Sync frame row attributes.
if err := s.syncFrame(di.Name, fi.Name); err != nil {
return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err)
// Sync field row attributes.
if err := s.syncField(di.Name, fi.Name); err != nil {
return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err)
}
for _, vi := range fi.Views {
@ -619,7 +621,7 @@ func (s *HolderSyncer) SyncHolder() error {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) {
if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) {
continue
}
@ -630,11 +632,11 @@ func (s *HolderSyncer) SyncHolder() error {
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
}
}
}
s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0)
s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0)
tf = time.Now() // reset tf
}
s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0)
@ -662,11 +664,9 @@ func (s *HolderSyncer) syncIndex(index string) error {
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.ColumnAttrDiff(context.Background(), index, blks)
m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks)
if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
@ -689,38 +689,36 @@ func (s *HolderSyncer) syncIndex(index string) error {
return nil
}
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve frame reference.
f := s.Holder.Frame(index, name)
// syncField synchronizes field attributes with the rest of the cluster.
func (s *HolderSyncer) syncField(index, name string) error {
// Retrieve field reference.
f := s.Holder.Field(index, name)
if f == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
frameTag := fmt.Sprintf("frame:%s", name)
fieldTag := fmt.Sprintf("field:%s", name)
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.RowAttrDiff(context.Background(), index, name, blks)
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks)
if err == ErrFieldNotFound {
continue // field not created remotely yet, skip
} else if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@ -738,11 +736,11 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error {
// Retrieve local frame.
f := s.Holder.Frame(index, frame)
func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error {
// Retrieve local field.
f := s.Holder.Field(index, field)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Ensure view exists locally.
@ -759,13 +757,12 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err
// Sync fragments together.
fs := FragmentSyncer{
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
RemoteClient: s.RemoteClient,
Fragment: frag,
Node: s.Node,
Cluster: s.Cluster,
Closing: s.Closing,
}
if err := fs.SyncFragment(); err != nil {
if err := fs.syncFragment(); err != nil {
return errors.Wrap(err, "syncing fragment")
}
@ -803,19 +800,19 @@ func (c *HolderCleaner) CleanHolder() error {
}
// Get the fragments that node is responsible for (based on hash(index, node)).
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.allFragments() {
fragSlice := fragment.slice
// Ignore fragments that should be present.
if uint64InSlice(fragSlice, containedSlices) {
continue
}
// Delete fragment.
if err := view.DeleteFragment(fragSlice); err != nil {
if err := view.deleteFragment(fragSlice); err != nil {
return errors.Wrap(err, "deleting fragment")
}
}

247
holder_test.go
View file

@ -24,8 +24,8 @@ import (
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -91,7 +91,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFramePermission", func(t *testing.T) {
t.Run("ErrFieldPermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -100,7 +100,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -113,13 +113,13 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFrameMetaCorrupt", func(t *testing.T) {
t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -127,17 +127,17 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) {
t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -145,7 +145,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") {
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
@ -159,9 +159,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -183,9 +183,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -208,11 +208,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -231,11 +229,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -257,11 +253,11 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
} else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if err := view.Fragment(0).FlushCache(); err != nil {
t.Fatal(err)
@ -331,14 +327,8 @@ func TestHolder_DeleteIndex(t *testing.T) {
defer hldr.Close()
// Write bits to separate indexes.
f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
if _, err := f0.SetBit(100, 200); err != nil {
t.Fatal(err)
}
f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0)
if _, err := f1.SetBit(100, 200); err != nil {
t.Fatal(err)
}
hldr.SetBit("i0", "f", 100, 200)
hldr.SetBit("i1", "f", 100, 200)
// Ensure i0 exists.
if _, err := os.Stat(hldr.IndexPath("i0")); err != nil {
@ -360,8 +350,19 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
s := test.NewServer()
defer s.Close()
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
t.Fatal(err)
}
cluster := test.NewCluster(2)
client := server.GetHTTPClient(nil)
client := http.GetHTTPClient(nil)
httpClient := http.NewInternalClientFromURI(uri, client)
cluster.InternalClient = httpClient
// Create a local holder.
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
@ -369,11 +370,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Create a remote holder wrapped by an HTTP
hldr1 := test.MustOpenHolder()
defer hldr1.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(client)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr1.Holder
e.Node = cluster.Nodes[1]
e.Cluster = cluster
@ -383,58 +382,35 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
uri, err := pilosa.NewURIFromAddress(s.URL)
if err != nil {
t.Fatal(err)
}
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create frames on nodes.
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0, hldr1} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
hldr.MustCreateFieldIfNotExists("i", "f")
hldr.MustCreateFieldIfNotExists("i", "f0")
hldr.MustCreateFieldIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f", 0, 10)
hldr0.SetBit("i", "f", 2, 20)
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
// Set a bit to create the fragment.
hldr0.SetBit("y", "z", 0, 0)
// Set data on the remote holder.
f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
}
hldr1.SetBit("i", "f", 0, 4000)
hldr1.SetBit("i", "f", 3, 10)
hldr1.SetBit("i", "f", 120, 10)
f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil {
t.Fatal(err)
}
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5)
hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
@ -442,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Set up syncer.
syncer := pilosa.HolderSyncer{
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
RemoteClient: server.GetHTTPClient(nil),
Stats: pilosa.NopStatsClient,
Holder: hldr0.Holder,
Node: cluster.Nodes[0],
Cluster: cluster,
Stats: pilosa.NopStatsClient,
}
if err := syncer.SyncHolder(); err != nil {
@ -455,29 +430,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -485,7 +454,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Ensure holder can clean up orphaned fragments.
func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster := test.NewCluster(2)
cluster := pilosa.NewTestCluster(2)
// Create a local holder.
hldr0 := test.MustOpenHolder()
@ -494,44 +463,28 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0)
// Create frames on nodes.
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
hldr.MustCreateFieldIfNotExists("i", "f")
hldr.MustCreateFieldIfNotExists("i", "f0")
hldr.MustCreateFieldIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(200, 4); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f", 0, 10)
hldr0.SetBit("i", "f", 0, 4000)
hldr0.SetBit("i", "f", 2, 20)
hldr0.SetBit("i", "f", 3, 10)
hldr0.SetBit("i", "f", 120, 10)
hldr0.SetBit("i", "f", 200, 4)
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
hldr0.SetBit("i", "f0", 9, SliceWidth+5)
f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2)
if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil {
t.Fatal(err)
}
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5)
hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7)
// Set highest slice.
hldr0.Index("i").SetRemoteMaxSlice(1)
@ -553,29 +506,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -596,26 +543,24 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
if f != nil {
t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}

1039
http/client.go Normal file
View file

File diff suppressed because it is too large Load diff

137
client_test.go → http/client_test.go
View file

@ -12,20 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package http_test
import (
"context"
"fmt"
"net/http"
gohttp "net/http"
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -43,10 +43,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
return server, hldr
}
var defaultClient *http.Client
var defaultClient *gohttp.Client
func init() {
defaultClient = server.GetHTTPClient(nil)
defaultClient = http.GetHTTPClient(nil)
}
@ -61,21 +61,24 @@ func TestClient_MultiNode(t *testing.T) {
}
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[0].Holder
e.Node = cluster.Nodes[0]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[1].Holder
e.Node = cluster.Nodes[1]
e.Cluster = cluster
return e.Execute(ctx, index, query, slices, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor(defaultClient)
httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient)
e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient))
e.Holder = hldr[2].Holder
e.Node = cluster.Nodes[2]
e.Cluster = cluster
@ -84,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) {
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
sliceNums := []uint64{1, 2, 6}
// This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())`
owns := [][]uint64{
{1, 3, 4, 8, 10, 13, 17, 19},
{2, 5, 7, 11, 12, 14, 18},
{0, 6, 9, 15, 16, 20},
}
for i, num := range sliceNums {
owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())
ownsNum := false
for _, ownNum := range owns {
for _, ownNum := range owns[i] {
if ownNum == num {
ownsNum = true
break
@ -98,9 +108,9 @@ func TestClient_MultiNode(t *testing.T) {
}
}
baseBit0 := SliceWidth * sliceNums[0]
baseBit1 := SliceWidth * sliceNums[1]
baseBit2 := SliceWidth * sliceNums[2]
baseBit0 := pilosa.SliceWidth * sliceNums[0]
baseBit1 := pilosa.SliceWidth * sliceNums[1]
baseBit2 := pilosa.SliceWidth * sliceNums[2]
maxSlice := uint64(0)
for _, x := range sliceNums {
@ -109,26 +119,26 @@ func TestClient_MultiNode(t *testing.T) {
}
}
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[0].MustSetBits("i", "f", 100, baseBit0+10)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustSetBits("i", "f", 1, baseBit1+4)
hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustSetBits("i", "f", 21, baseBit2+10)
hldr[2].MustSetBits("i", "f", 100, baseBit2+10)
hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11)
hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
@ -138,14 +148,14 @@ func TestClient_MultiNode(t *testing.T) {
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
// Connect to each node to compare results.
client := make([]*test.Client, 3)
client[0] = test.MustNewClient(s[0].Host(), defaultClient)
client[1] = test.MustNewClient(s[1].Host(), defaultClient)
client[2] = test.MustNewClient(s[2].Host(), defaultClient)
client := make([]*Client, 3)
client[0] = MustNewClient(s[0].Host(), defaultClient)
client[1] = MustNewClient(s[1].Host(), defaultClient)
client[2] = MustNewClient(s[2].Host(), defaultClient)
topN := 4
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN),
Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN),
Remote: false,
}
result, err := client[0].Query(context.Background(), "i", queryRequest)
@ -211,8 +221,8 @@ func TestClient_Import(t *testing.T) {
defer hldr.Close()
// Load bitmap into cache to ensure cache gets updated.
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f.Row(0)
hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created.
hldr.Row("i", "f", 0)
s := test.NewServer()
defer s.Close()
@ -221,7 +231,7 @@ func TestClient_Import(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
c := MustNewClient(s.Host(), defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -231,10 +241,10 @@ func TestClient_Import(t *testing.T) {
}
// Verify data.
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -244,8 +254,9 @@ func TestClient_ImportValue(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
fld := pilosa.Field{
Name: "fld",
fldName := "f"
fo := pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
@ -253,7 +264,7 @@ func TestClient_ImportValue(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
field, err := index.CreateFieldIfNotExists(fldName, fo)
if err != nil {
t.Fatal(err)
}
@ -265,8 +276,8 @@ func TestClient_ImportValue(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
c := MustNewClient(s.Host(), defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
{ColumnID: 3, Value: 40},
@ -275,7 +286,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Sum.
sum, cnt, err := frame.FieldSum(nil, fld.Name)
sum, cnt, err := field.Sum(nil, fldName)
if err != nil {
t.Fatal(err)
}
@ -284,7 +295,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Min.
min, cnt, err := frame.FieldMin(nil, fld.Name)
min, cnt, err := field.Min(nil, fldName)
if err != nil {
t.Fatal(err)
}
@ -293,11 +304,11 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Min with Filter.
filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
filter, err := field.Range(fldName, pql.GT, 40)
if err != nil {
t.Fatal(err)
}
min, cnt, err = frame.FieldMin(filter, fld.Name)
min, cnt, err = field.Min(filter, fldName)
if err != nil {
t.Fatal(err)
}
@ -306,7 +317,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Max.
max, cnt, err := frame.FieldMax(nil, fld.Name)
max, cnt, err := field.Max(nil, fldName)
if err != nil {
t.Fatal(err)
}
@ -321,11 +332,11 @@ func TestClient_FragmentBlocks(t *testing.T) {
defer hldr.Close()
// Set two bits on blocks 0 & 3.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
hldr.SetBit("i", "f", 0, 1)
s := test.NewServer()
defer s.Close()
@ -334,8 +345,8 @@ func TestClient_FragmentBlocks(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := test.MustNewClient(s.Host(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0)
c := MustNewClient(s.Host(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)
if err != nil {
t.Fatal(err)
} else if len(blocks) != 2 {
@ -351,3 +362,17 @@ func TestClient_FragmentBlocks(t *testing.T) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
}
}
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*http.InternalClient
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string, h *gohttp.Client) *Client {
c, err := http.NewInternalClient(host, h)
if err != nil {
panic(err)
}
return &Client{InternalClient: c}
}

1289
http/handler.go Normal file
View file

File diff suppressed because it is too large Load diff

18
handler_internal_test.go → http/handler_internal_test.go
View file

@ -12,12 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package http
import (
"encoding/json"
"reflect"
"testing"
"github.com/pilosa/pilosa"
)
// Test custom UnmarshalJSON for postIndexRequest object
@ -27,7 +29,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
expected postIndexRequest
err string
}{
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
{json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
@ -55,23 +57,23 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
}
}
// Test custom UnmarshalJSON for postFrameRequest object
func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
// Test custom UnmarshalJSON for postFieldRequest object
func TestPostFieldRequestUnmarshalJSON(t *testing.T) {
tests := []struct {
json string
expected postFrameRequest
expected postFieldRequest
err string
}{
{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
{json: `{"options": {}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
{json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {
actual := &postFrameRequest{}
actual := &postFieldRequest{}
err := json.Unmarshal([]byte(test.json), actual)
if err != nil {
if test.err == "" || test.err != err.Error() {

412
handler_test.go → http/handler_test.go
View file

@ -12,17 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package http_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
gohttp "net/http"
"net/http/httptest"
"reflect"
"strings"
@ -30,9 +29,9 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/test"
)
@ -51,7 +50,7 @@ func TestHandlerPanics(t *testing.T) {
if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("expected panic in log, but got: %s", bufbytes)
}
if w.Code != http.StatusInternalServerError {
if w.Code != gohttp.StatusInternalServerError {
t.Fatalf("expected internal server error, but got: %v", w.Code)
}
bodyBytes := w.Body.Bytes()
@ -71,7 +70,7 @@ func TestHandler_NotFound(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != http.StatusNotFound {
if w.Code != gohttp.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
}
@ -84,17 +83,17 @@ func TestHandler_Schema(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -103,10 +102,10 @@ func TestHandler_Schema(t *testing.T) {
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -121,17 +120,17 @@ func TestHandler_Status(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -144,7 +143,7 @@ func TestHandler_Status(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
@ -158,9 +157,9 @@ func TestHandler_Info(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) {
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
}
@ -175,7 +174,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
} else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" {
@ -190,20 +189,20 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
@ -231,7 +230,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -272,7 +271,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
@ -288,7 +287,7 @@ func TestHandler_Query_Args_Err(t *testing.T) {
h.API.Holder = hldr.Holder
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -297,7 +296,7 @@ func TestHandler_Query_Args_Err(t *testing.T) {
func TestHandler_Query_Params_Err(t *testing.T) {
w := httptest.NewRecorder()
test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -319,7 +318,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -342,14 +341,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 {
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 100 {
t.Fatalf("unexpected n: %d", n)
@ -372,7 +371,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
@ -405,7 +404,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
@ -430,16 +429,16 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) {
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
@ -488,7 +487,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
@ -496,9 +495,9 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
@ -537,8 +536,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
if w.Code != http.StatusOK {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -561,17 +560,17 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs {
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
@ -592,7 +591,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -612,10 +611,10 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
@ -637,7 +636,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) {
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != http.StatusMethodNotAllowed {
if w.Code != gohttp.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
}
@ -652,7 +651,7 @@ func TestHandler_Query_ErrParse(t *testing.T) {
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
@ -674,14 +673,14 @@ func TestHandler_Index_Delete(t *testing.T) {
}
// Send request to delete index.
resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Verify body response.
if resp.StatusCode != http.StatusOK {
if resp.StatusCode != gohttp.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
} else if buf, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
@ -695,12 +694,12 @@ func TestHandler_Index_Delete(t *testing.T) {
}
}
// Ensure handler can delete a frame.
func TestHandler_DeleteFrame(t *testing.T) {
// Ensure handler can delete a field.
func TestHandler_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -708,13 +707,13 @@ func TestHandler_DeleteFrame(t *testing.T) {
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Frame("f1"); f != nil {
t.Fatal("expected nil frame")
} else if f := hldr.Index("i0").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
@ -751,7 +750,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) {
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
resp, err := http.Post(
resp, err := gohttp.Post(
s.URL+"/index/i/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
@ -767,8 +766,8 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) {
}
}
// Ensure the handler can return data in differing blocks for a frame.
func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Ensure the handler can return data in differing blocks for a field.
func TestHandler_Field_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -778,7 +777,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Set attributes on the index.
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{})
f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
@ -801,8 +800,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
resp, err := http.Post(
s.URL+"/index/i/frame/meta/attr/diff",
resp, err := gohttp.Post(
s.URL+"/index/i/field/meta/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
@ -817,245 +816,6 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
}
}
// Ensure the handler can create a new field on an existing frame.
func TestHandler_Frame_AddField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":100,"max":200}`),
)
if err != nil {
t.Fatal(err)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
if field := f.Field("x"); !reflect.DeepEqual(field, &pilosa.Field{Name: "x", Type: "int", Min: 100, Max: 200}) {
t.Fatalf("unexpected field: %#v", field)
}
})
t.Run("ErrInvalidFieldType", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"bad_type","min":100,"max":200}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field type`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":200,"max":100}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field range`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
t.Run("ErrFieldAlreadyExists", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":0,"max":100}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: field already exists`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
}
// Ensure the handler can delete existing fields.
func TestHandler_Frame_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/x", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
if field := f.Field("x"); field != nil {
t.Fatalf("expected nil field, got: %#v", field)
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/y", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) != `deleting field: field not found` {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
}
func TestHandler_Frame_GetFields(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil {
t.Fatal(err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f/fields")
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
var fields FrameFields
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if err = json.Unmarshal([]byte(body), &fields); err != nil {
t.Fatal(err)
}
field := fields.Fields[0]
if field.Name != "x" {
t.Fatalf("expected field's name: x, actuall name: %v", field.Name)
} else if field.Min != 1 {
t.Fatalf("expected field's min: x, actuall min: %v", field.Min)
} else if field.Max != 100 {
t.Fatalf("expected field's max: x, actuall max: %v", field.Max)
}
})
t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{})
if err != nil {
t.Fatalf("creating frame: %v", err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f1/fields")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) == `frame fields not allowed` {
t.Fatalf("shouldn't get frame fields not allowed error: %q", body)
}
})
}
type FrameFields struct {
Fields []pilosa.Field
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()
@ -1072,7 +832,7 @@ func TestHandler_Version(t *testing.T) {
if strings.HasPrefix(version, "v") {
version = version[1:]
}
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
@ -1092,7 +852,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
@ -1102,7 +862,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
@ -1110,7 +870,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
@ -1126,7 +886,7 @@ func TestHandler_Expvars(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
@ -1149,40 +909,12 @@ func TestHandler_RecalculateCaches(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
if w.Code != http.StatusNoContent {
if w.Code != gohttp.StatusNoContent {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
func TestHandler_WebUI(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
if !strings.Contains(w.Body.String(), "<title>Pilosa WebUI</title>") {
t.Fatalf("WebUI is not being served correctly.")
}
// If curl is the client, the response should be different
w = httptest.NewRecorder()
req := test.MustNewHTTPRequest("GET", "/", nil)
req.Header.Add("User-Agent", "curl/7.54.0")
h.ServeHTTP(w, req)
if !strings.Contains(w.Body.String(), "try the WebUI") {
t.Fatalf("WebUI is not being served correctly.")
}
}
func TestHandler_CORS(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -1208,7 +940,7 @@ func TestHandler_CORS(t *testing.T) {
}
// CORS config should allow preflight response
handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"}))
handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"}))
w = httptest.NewRecorder()
handler.ServeHTTP(w, req)
result = w.Result()

87
http/translator.go Normal file
View file

@ -0,0 +1,87 @@
package http
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"github.com/pilosa/pilosa"
)
// Ensure implementation implements inteface.
var _ pilosa.TranslateStore = (*TranslateStore)(nil)
// TranslateStore represents an implementation of TranslateStore that
// communicates over HTTP. This is used with the TranslateHandler.
type TranslateStore struct {
URL string
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(rawurl string) *TranslateStore {
return &TranslateStore{URL: rawurl}
}
// TranslateColumnsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateColumnToString is not currently implemented.
func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// TranslateRowsToUint64 is not currently implemented.
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
return nil, pilosa.ErrNotImplemented
}
// TranslateRowToString is not currently implemented.
func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) {
return "", pilosa.ErrNotImplemented
}
// Reader returns a reader that can stream data from a remote store.
func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
// Generate remote URL.
u, err := url.Parse(s.URL)
if err != nil {
return nil, err
}
u.Path = "/translate/data"
u.RawQuery = (url.Values{
"offset": {strconv.FormatInt(off, 10)},
}).Encode()
// Connect a stream to the remote server.
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
// Connect a stream to the remote server.
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err)
}
// Handle error codes or return body as stream.
switch resp.StatusCode {
case http.StatusOK:
return resp.Body, nil
case http.StatusNotImplemented:
resp.Body.Close()
return nil, pilosa.ErrNotImplemented
default:
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body))
}
}

134
http/translator_test.go Normal file
View file

@ -0,0 +1,134 @@
package http_test
import (
"context"
"io"
"io/ioutil"
"net/http/httptest"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/mock"
"github.com/pilosa/pilosa/test"
)
func TestTranslateStore_Reader(t *testing.T) {
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
t.Run("ServerDisconnect", func(t *testing.T) {
var mrc mock.ReadCloser
var readN int
mrc.ReadFunc = func(p []byte) (int, error) {
readN++
switch readN {
case 1:
copy(p, []byte("foo"))
return 3, nil
case 2:
copy(p, []byte("barbaz"))
return 6, nil
case 3:
return 0, io.EOF
default:
t.Fatal("unexpected read")
return 0, nil
}
}
var closeInvoked bool
mrc.CloseFunc = func() error {
closeInvoked = true
return nil
}
// Setup handler on test server.
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
if off != 100 {
t.Fatalf("unexpected off: %d", off)
}
return &mrc, nil
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
// Connect to server and stream all available data.
store := http.NewTranslateStore(s.URL)
rc, err := store.Reader(context.Background(), 100)
if err != nil {
t.Fatal(err)
} else if data, err := ioutil.ReadAll(rc); err != nil {
t.Fatal(err)
} else if string(data) != `foobarbaz` {
t.Fatalf("unexpected data: %q", data)
} else if err := rc.Close(); err != nil {
t.Fatal(err)
}
if !closeInvoked {
t.Fatal("expected server close")
}
})
// Ensure server closes store reader if client disconnects.
t.Run("ClientDisconnect", func(t *testing.T) {
// Setup mock so that Read() hangs.
done := make(chan struct{})
var mrc mock.ReadCloser
mrc.ReadFunc = func(p []byte) (int, error) {
<-done
return 0, io.EOF
}
var closeInvoked bool
mrc.CloseFunc = func() error {
closeInvoked = true
return nil
}
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return &mrc, nil
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
defer close(done)
// Connect to server and begin streaming.
ctx, cancel := context.WithCancel(context.Background())
store := http.NewTranslateStore(s.URL)
if _, err := store.Reader(ctx, 0); err != nil {
t.Fatal(err)
}
// Cancel the context and check if server is closed.
cancel()
time.Sleep(100 * time.Millisecond)
if !closeInvoked {
t.Fatal("expected server-side close")
}
})
})
// Ensure client is notified if the server doesn't support streaming replication.
t.Run("ErrNotImplemented", func(t *testing.T) {
var translateStore mock.TranslateStore
translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) {
return nil, pilosa.ErrNotImplemented
}
h := test.MustNewHandler()
h.API.TranslateStore = &translateStore
s := httptest.NewServer(h)
defer s.Close()
_, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0)
if err != pilosa.ErrNotImplemented {
t.Fatalf("unexpected error: %s", err)
}
})
}

179
index.go
View file

@ -28,14 +28,15 @@ import (
"github.com/pkg/errors"
)
// Index represents a container for frames.
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
path string
name string
keys bool // use string keys
// Frames by name.
frames map[string]*Frame
// Fields by name.
fields map[string]*Field
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
@ -53,7 +54,7 @@ type Index struct {
// NewIndex returns a new instance of Index.
func NewIndex(path, name string) (*Index, error) {
err := ValidateName(name)
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
@ -61,7 +62,7 @@ func NewIndex(path, name string) (*Index, error) {
return &Index{
path: path,
name: name,
frames: make(map[string]*Frame),
fields: make(map[string]*Field),
remoteMaxSlice: 0,
@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name }
// Path returns the path the index was initialized with.
func (i *Index) Path() string { return i.path }
// Keys returns true if the index uses string keys.
func (i *Index) Keys() bool { return i.keys }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore }
@ -106,8 +110,8 @@ func (i *Index) Open() error {
return errors.Wrap(err, "loading meta file")
}
if err := i.openFrames(); err != nil {
return errors.Wrap(err, "opening frames")
if err := i.openFields(); err != nil {
return errors.Wrap(err, "opening fields")
}
if err := i.columnAttrStore.Open(); err != nil {
@ -117,8 +121,8 @@ func (i *Index) Open() error {
return nil
}
// openFrames opens and initializes the frames inside the index.
func (i *Index) openFrames() error {
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields() error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -135,14 +139,14 @@ func (i *Index) openFrames() error {
continue
}
fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := fr.Open(); err != nil {
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
i.frames[fr.Name()] = fr
i.fields[fld.Name()] = fld
}
return nil
}
@ -164,18 +168,17 @@ func (i *Index) loadMeta() error {
}
// Copy metadata fields.
i.keys = pb.Keys
return nil
}
// NOTE: Until we introduce new attributes to store in the index .meta file,
// we don't need to actually write the file. The code related to index.options
// and the index meta file are left in place for future use.
/*
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{})
buf, err := proto.Marshal(&internal.IndexMeta{
Keys: i.keys,
})
if err != nil {
return errors.Wrap(err, "marshalling")
}
@ -187,9 +190,8 @@ func (i *Index) saveMeta() error {
return nil
}
*/
// Close closes the index and its frames.
// Close closes the index and its fields.
func (i *Index) Close() error {
i.mu.Lock()
defer i.mu.Unlock()
@ -197,13 +199,13 @@ func (i *Index) Close() error {
// Close the attribute store.
i.columnAttrStore.Close()
// Close all frames.
for _, f := range i.frames {
// Close all fields.
for _, f := range i.fields {
if err := f.Close(); err != nil {
return errors.Wrap(err, "closing frame")
return errors.Wrap(err, "closing field")
}
}
i.frames = make(map[string]*Frame)
i.fields = make(map[string]*Field)
return nil
}
@ -217,7 +219,7 @@ func (i *Index) MaxSlice() uint64 {
defer i.mu.RUnlock()
max := i.remoteMaxSlice
for _, f := range i.frames {
for _, f := range i.fields {
if slice := f.MaxSlice(); slice > max {
max = slice
}
@ -234,154 +236,139 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.remoteMaxSlice = newmax
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
// FieldPath returns the path to a field in the index.
func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) }
// Frame returns a frame in the index by name.
func (i *Index) Frame(name string) *Frame {
// Field returns a field in the index by name.
func (i *Index) Field(name string) *Field {
i.mu.RLock()
defer i.mu.RUnlock()
return i.frame(name)
return i.field(name)
}
func (i *Index) frame(name string) *Frame { return i.frames[name] }
func (i *Index) field(name string) *Field { return i.fields[name] }
// Frames returns a list of all frames in the index.
func (i *Index) Frames() []*Frame {
// Fields returns a list of all fields in the index.
func (i *Index) Fields() []*Field {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*Frame, 0, len(i.frames))
for _, f := range i.frames {
a := make([]*Field, 0, len(i.fields))
for _, f := range i.fields {
a = append(a, f)
}
sort.Sort(frameSlice(a))
sort.Sort(fieldSlice(a))
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
// RecalculateCaches recalculates caches on every field in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
frame.RecalculateCaches()
for _, field := range i.Fields() {
field.RecalculateCaches()
}
}
// CreateFrame creates a frame.
func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
// CreateField creates a field.
func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Ensure frame doesn't already exist.
if i.frames[name] != nil {
return nil, ErrFrameExists
// Ensure field doesn't already exist.
if i.fields[name] != nil {
return nil, ErrFieldExists
}
return i.createFrame(name, opt)
return i.createField(name, opt)
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find frame in cache first.
if f := i.frames[name]; f != nil {
// Find field in cache first.
if f := i.fields[name]; f != nil {
return f, nil
}
return i.createFrame(name, opt)
return i.createField(name, opt)
}
func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
if name == "" {
return nil, errors.New("frame name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Validate fields.
for _, field := range opt.Fields {
if err := ValidateField(field); err != nil {
return nil, err
}
// Validate options.
if err := opt.Validate(); err != nil {
return nil, errors.Wrap(err, "validating options")
}
// Initialize frame.
f, err := i.newFrame(i.FramePath(name), name)
// Initialize field.
f, err := i.newField(i.FieldPath(name), name)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
// Open frame.
// Open field.
if err := f.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
}
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
// Apply field options.
if err := f.applyOptions(opt); err != nil {
f.Close()
return nil, errors.Wrap(err, "setting time quantum")
return nil, errors.Wrap(err, "applying options")
}
// Set cache type.
if opt.CacheType == "" {
opt.CacheType = DefaultCacheType
}
f.cacheType = opt.CacheType
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
// Set fields.
f.fields = opt.Fields
if err := f.saveMeta(); err != nil {
f.Close()
return nil, errors.Wrap(err, "saving meta")
}
// Add to index's frame lookup.
i.frames[name] = f
// Add to index's field lookup.
i.fields[name] = f
return f, nil
}
func (i *Index) newFrame(path, name string) (*Frame, error) {
f, err := NewFrame(path, i.name, name)
func (i *Index) newField(path, name string) (*Field, error) {
f, err := NewField(path, i.name, name)
if err != nil {
return nil, err
}
f.Logger = i.Logger
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
return f, nil
}
// DeleteFrame removes a frame from the index.
func (i *Index) DeleteFrame(name string) error {
// DeleteField removes a field from the index.
func (i *Index) DeleteField(name string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if frame doesn't exist.
f := i.frame(name)
// Ignore if field doesn't exist.
f := i.field(name)
if f == nil {
return nil
}
// Close frame.
// Close field.
if err := f.Close(); err != nil {
return errors.Wrap(err, "closing")
}
// Delete frame directory.
if err := os.RemoveAll(i.FramePath(name)); err != nil {
// Delete field directory.
if err := os.RemoveAll(i.FieldPath(name)); err != nil {
return errors.Wrap(err, "removing directory")
}
// Remove reference.
delete(i.frames, name)
delete(i.fields, name)
return nil
}
@ -395,7 +382,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
Frames []*FrameInfo `json:"frames"`
Fields []*FieldInfo `json:"fields"`
}
type indexInfoSlice []*IndexInfo
@ -417,16 +404,20 @@ func EncodeIndexes(a []*Index) []*internal.Index {
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Frames: encodeFrames(d.Frames()),
Fields: encodeFields(d.Fields()),
}
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct{}
type IndexOptions struct {
Keys bool `json:"keys"`
}
// Encode converts i into its internal representation.
func (i *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{}
return &internal.IndexMeta{
Keys: i.Keys,
}
}
// hasTime returns true if a contains a non-nil time.

44
filesystem.go → index_internal_test.go
View file

@ -15,28 +15,32 @@
package pilosa
import (
"fmt"
"net/http"
"io/ioutil"
)
// Ensure nopFileSystem implements interface.
var _ FileSystem = &nopFileSystem{}
// FileSystem represents an interface for a WebUI file system.
type FileSystem interface {
New() (http.FileSystem, error)
// mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func mustOpenIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, "i")
if err != nil {
panic(err)
}
if err := index.Open(); err != nil {
panic(err)
}
return index
}
func init() {
NopFileSystem = &nopFileSystem{}
}
// NopFileSystem represents a FileSystem that returns an error if called.
var NopFileSystem FileSystem
type nopFileSystem struct{}
// New is a no-op implementation of FileSystem New method.
func (n *nopFileSystem) New() (http.FileSystem, error) {
return nil, fmt.Errorf("file system not implemented")
// reopen closes the index and reopens it.
func (i *Index) reopen() error {
if err := i.Close(); err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}

215
index_test.go
View file

@ -23,181 +23,184 @@ import (
"github.com/pilosa/pilosa/test"
)
// Ensure index can open and retrieve a frame.
func TestIndex_CreateFrameIfNotExists(t *testing.T) {
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// Ensure index can open and retrieve a field.
func TestIndex_CreateFieldIfNotExists(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
// Create field.
f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
} else if f == nil {
t.Fatal("expected frame")
t.Fatal("expected field")
}
// Retrieve existing frame.
other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
// Retrieve existing field.
other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
} else if f.Frame != other.Frame {
t.Fatal("frame mismatch")
} else if f.Field != other.Field {
t.Fatal("field mismatch")
}
if f.Frame != index.Frame("f") {
t.Fatal("frame mismatch")
if f.Field != index.Field("f") {
t.Fatal("field mismatch")
}
}
func TestIndex_CreateFrame(t *testing.T) {
// Ensure time quantum can be set appropriately on a new frame.
func TestIndex_CreateField(t *testing.T) {
// Ensure time quantum can be set appropriately on a new field.
t.Run("TimeQuantum", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with explicit quantum.
f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
// Create field with explicit quantum.
f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected frame time quantum: %s", q)
t.Fatalf("unexpected field time quantum: %s", q)
}
})
})
// Ensure frame can include range columns.
// Ensure field can include range columns.
t.Run("BSIFields", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
// Create field with schema and verify it exists.
if f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 20,
}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected fields: %#v", f.Fields())
} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
t.Fatalf("unexpected type: %#v", f.Type())
}
// Reopen the index & verify the fields are loaded.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
} else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
t.Fatalf("unexpected type after reopen: %#v", f.Type())
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// TODO: These errors don't apply here. Instead, we need these tests
// on field creation FieldOptions validation.
/*
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
})
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateField("f", pilosa.FieldOptions{
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrFieldFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != nil {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFieldNameRequired", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrFieldNameRequired", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFieldNameRequired {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFieldNameRequired {
t.Fatal(err)
}
})
t.Run("ErrInvalidFieldType", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrInvalidFieldType", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
}); err != pilosa.ErrInvalidFieldType {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
}); err != pilosa.ErrInvalidFieldType {
t.Fatal(err)
}
})
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
}); err != pilosa.ErrInvalidFieldRange {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
}); err != pilosa.ErrInvalidBSIGroupRange {
t.Fatal(err)
}
})
*/
})
}
// Ensure index can delete a frame.
func TestIndex_DeleteFrame(t *testing.T) {
// Ensure index can delete a field.
func TestIndex_DeleteField(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Delete frame & verify it's gone.
if err := index.DeleteFrame("f"); err != nil {
// Delete field & verify it's gone.
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
} else if index.Frame("f") != nil {
t.Fatal("expected nil frame")
} else if index.Field("f") != nil {
t.Fatal("expected nil field")
}
// Delete again to make sure it doesn't error.
if err := index.DeleteFrame("f"); err != nil {
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
}
}
// Ensure index can delete a frame.
// Ensure index can delete a field.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {

215
inmem/translator.go Normal file
View file

@ -0,0 +1,215 @@
package inmem
import (
"context"
"io"
"sync"
"github.com/pilosa/pilosa"
)
// Ensure type implements interface.
var _ pilosa.TranslateStore = &TranslateStore{}
// TranslateStore is an in-memory storage engine for translating string-to-uint64 values.
type TranslateStore struct {
mu sync.RWMutex
cols map[string]*translateIndex
rows map[frameKey]*translateIndex
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore() *TranslateStore {
return &TranslateStore{
cols: make(map[string]*translateIndex),
rows: make(map[frameKey]*translateIndex),
}
}
// Reader returns an error because it is not supported by the inmem store.
func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) {
return nil, pilosa.ErrReplicationNotSupported
}
// TranslateColumnsToUint64 converts value to a uint64 id.
// If value does not have an associated id then one is created.
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
ret := make([]uint64, len(values))
// Read value under read lock.
s.mu.RLock()
if idx := s.cols[index]; idx != nil {
var writeRequired bool
for i := range values {
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
}
ret[i] = v
}
if !writeRequired {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
// If any values not found then recheck and then add under a write lock.
s.mu.Lock()
defer s.mu.Unlock()
// Recheck if value was created between the read lock and write lock.
idx := s.cols[index]
if idx != nil {
var writeRequired bool
for i := range values {
if ret[i] != 0 {
continue
}
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
continue
}
ret[i] = v
}
if !writeRequired {
return ret, nil
}
}
// Create index map if it doesn't exists.
if idx == nil {
idx = newTranslateIndex()
s.cols[index] = idx
}
// Add new identifiers.
for i := range values {
if ret[i] != 0 {
continue
}
idx.seq++
v := idx.seq
ret[i] = v
idx.lookup[values[i]] = v
idx.reverse[v] = values[i]
}
return ret, nil
}
// TranslateColumnToString converts a uint64 id to its associated string value.
// If the id is not associated with a string value then a blank string is returned.
func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.cols[index]; idx != nil {
if ret, ok := idx.reverse[value]; ok {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
return "", nil
}
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
key := frameKey{index, frame}
ret := make([]uint64, len(values))
// Read value under read lock.
s.mu.RLock()
if idx := s.rows[key]; idx != nil {
var writeRequired bool
for i := range values {
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
}
ret[i] = v
}
if !writeRequired {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
// If any values not found then recheck and then add under a write lock.
s.mu.Lock()
defer s.mu.Unlock()
// Recheck if value was created between the read lock and write lock.
idx := s.rows[key]
if idx != nil {
var writeRequired bool
for i := range values {
if ret[i] != 0 {
continue
}
v, ok := idx.lookup[values[i]]
if !ok {
writeRequired = true
continue
}
ret[i] = v
}
if !writeRequired {
return ret, nil
}
}
// Create map if it doesn't exists.
if idx == nil {
idx = newTranslateIndex()
s.rows[key] = idx
}
// Add new identifiers.
for i := range values {
if ret[i] != 0 {
continue
}
idx.seq++
v := idx.seq
ret[i] = v
idx.lookup[values[i]] = v
idx.reverse[v] = values[i]
}
return ret, nil
}
func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
s.mu.RLock()
if idx := s.rows[frameKey{index, frame}]; idx != nil {
if ret, ok := idx.reverse[value]; ok {
s.mu.RUnlock()
return ret, nil
}
}
s.mu.RUnlock()
return "", nil
}
type frameKey struct {
index string
frame string
}
type translateIndex struct {
seq uint64
lookup map[string]uint64
reverse map[uint64]string
}
func newTranslateIndex() *translateIndex {
return &translateIndex{
lookup: make(map[string]uint64),
reverse: make(map[uint64]string),
}
}

132
inmem/translator_test.go Normal file
View file

@ -0,0 +1,132 @@
package inmem_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
"github.com/pilosa/pilosa/inmem"
)
func TestTranslateStore_TranslateColumn(t *testing.T) {
s := inmem.NewTranslateStore()
// First translation should start id at zero.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
}
func TestTranslateStore_TranslateRow(t *testing.T) {
s := inmem.NewTranslateStore()
// First translation should start id at zero.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Next translation on the same index should move to one.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{2}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different index restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Translation on a different frame restarts at 0.
if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(ids, []uint64{1}) {
t.Fatalf("unexpected id: %#v", ids)
}
// Ensure that string values can be looked up by ID.
if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil {
t.Fatal(err)
} else if value != "bar" {
t.Fatalf("unexpected value: %s", value)
}
}
func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) {
const batchSize = 1000
s := inmem.NewTranslateStore()
// Generate keys before benchmark begins
keySets := make([][]string, b.N/1000)
for i := range keySets {
keySets[i] = make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i)
}
}
b.ResetTimer()
for _, keySet := range keySets {
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) {
const batchSize = 1000
s := inmem.NewTranslateStore()
// Generate keys before benchmark begins
for i := 0; i < b.N; i += batchSize {
keySet := make([]string, batchSize)
for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) {
keySet[j] = fmt.Sprintf("%08d%08d", jv, i)
}
if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil {
b.Fatal(err)
}
}
// Generate random key access.
perm := rand.New(rand.NewSource(0)).Perm(b.N)
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil {
b.Fatal(err)
}
}
}

1138
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

42
internal/private.proto
View file

@ -3,13 +3,17 @@ syntax = "proto3";
package internal;
message IndexMeta {
bool Keys = 3;
}
message FrameMeta {
message FieldOptions {
string Type = 8;
string CacheType = 3;
uint32 CacheSize = 4;
int64 Min = 9;
int64 Max = 10;
string TimeQuantum = 5;
repeated Field Fields = 7;
bool Keys = 11;
}
message ImportResponse {
@ -18,7 +22,7 @@ message ImportResponse {
message BlockDataRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 5;
uint64 Slice = 4;
uint64 Block = 3;
@ -51,32 +55,20 @@ message CreateIndexMessage {
IndexMeta Meta = 2;
}
message CreateFrameMessage {
string Index = 1;
string Frame = 2;
FrameMeta Meta = 3;
}
message DeleteFrameMessage {
string Index = 1;
string Frame = 2;
}
message CreateFieldMessage {
string Index = 1;
string Frame = 2;
Field Field = 3;
string Field = 2;
FieldOptions Meta = 3;
}
message DeleteFieldMessage {
string Index = 1;
string Frame = 2;
string Field = 3;
string Field = 2;
}
message Frame {
message Field {
string Name = 1;
FrameMeta Meta = 2;
FieldOptions Meta = 2;
repeated string Views = 3;
}
@ -86,7 +78,7 @@ message Schema {
message Index {
string Name = 1;
repeated Frame Frames = 4;
repeated Field Fields = 4;
}
message URI {
@ -123,7 +115,7 @@ message ClusterStatus {
repeated Node Nodes = 3;
}
message Field {
message BSIGroup {
string Name = 1;
string Type = 2;
int64 Min = 3;
@ -132,13 +124,13 @@ message Field {
message CreateViewMessage {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 3;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 3;
}
@ -154,7 +146,7 @@ message ResizeInstruction {
message ResizeSource {
Node Node = 1;
string Index = 2;
string Frame = 3;
string Field = 3;
string View = 4;
uint64 Slice = 5;
}

214
internal/public.pb.go
View file

@ -27,7 +27,7 @@ import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import binary "encoding/binary"
import encoding_binary "encoding/binary"
import io "io"
@ -412,7 +412,7 @@ func (m *QueryResult) GetChanged() bool {
type ImportRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
@ -433,9 +433,9 @@ func (m *ImportRequest) GetIndex() string {
return ""
}
func (m *ImportRequest) GetFrame() string {
func (m *ImportRequest) GetField() string {
if m != nil {
return m.Frame
return m.Field
}
return ""
}
@ -484,9 +484,8 @@ func (m *ImportRequest) GetTimestamps() []int64 {
type ImportValueRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"`
Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
@ -504,9 +503,9 @@ func (m *ImportValueRequest) GetIndex() string {
return ""
}
func (m *ImportValueRequest) GetFrame() string {
func (m *ImportValueRequest) GetField() string {
if m != nil {
return m.Frame
return m.Field
}
return ""
}
@ -518,13 +517,6 @@ func (m *ImportValueRequest) GetSlice() uint64 {
return 0
}
func (m *ImportValueRequest) GetField() string {
if m != nil {
return m.Field
}
return ""
}
func (m *ImportValueRequest) GetColumnIDs() []uint64 {
if m != nil {
return m.ColumnIDs
@ -808,7 +800,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
i += 8
}
return i, nil
@ -1064,11 +1056,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
if len(m.Field) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
dAtA[i] = 0x18
@ -1181,23 +1173,17 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
if len(m.Field) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
}
if len(m.Field) > 0 {
dAtA[i] = 0x22
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if len(m.ColumnIDs) > 0 {
dAtA14 := make([]byte, len(m.ColumnIDs)*10)
var j13 int
@ -1472,7 +1458,7 @@ func (m *ImportRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Frame)
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
@ -1522,17 +1508,13 @@ func (m *ImportValueRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Frame)
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPublic(uint64(m.Slice))
}
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if len(m.ColumnIDs) > 0 {
l = 0
for _, e := range m.ColumnIDs {
@ -2335,7 +2317,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if (iNdEx + 8) > l {
return io.ErrUnexpectedEOF
}
v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
m.FloatValue = float64(math.Float64frombits(v))
default:
@ -3067,7 +3049,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -3092,7 +3074,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Frame = string(dAtA[iNdEx:postIndex])
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
@ -3437,54 +3419,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Frame = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
@ -3513,6 +3447,25 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType == 0 {
var v uint64
@ -3795,50 +3748,49 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 709 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c,
0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4,
0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd,
0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64,
0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8,
0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82,
0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69,
0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f,
0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd,
0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e,
0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e,
0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62,
0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf,
0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8,
0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77,
0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e,
0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35,
0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88,
0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59,
0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7,
0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c,
0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd,
0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85,
0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3,
0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18,
0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3,
0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4,
0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96,
0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60,
0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec,
0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd,
0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e,
0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c,
0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41,
0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62,
0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9,
0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b,
0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82,
0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8,
0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd,
0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7,
0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7,
0xa4, 0x5a, 0x06, 0x00, 0x00,
// 699 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e,
0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54,
0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d,
0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73,
0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26,
0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0,
0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe,
0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46,
0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee,
0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3,
0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5,
0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12,
0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b,
0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98,
0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c,
0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf,
0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6,
0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda,
0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa,
0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3,
0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57,
0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2,
0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd,
0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10,
0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3,
0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95,
0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1,
0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59,
0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5,
0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab,
0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb,
0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94,
0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82,
0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9,
0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9,
0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7,
0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1,
0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f,
0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d,
0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66,
0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90,
0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00,
0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00,
}

5
internal/public.proto
View file

@ -70,7 +70,7 @@ message QueryResult {
message ImportRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
uint64 Slice = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
@ -81,9 +81,8 @@ message ImportRequest {
message ImportValueRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
uint64 Slice = 3;
string Field = 4;
repeated uint64 ColumnIDs = 5;
repeated string ColumnKeys = 7;
repeated int64 Values = 6;

190
internal/test/querygenerator.go Normal file
View file

@ -0,0 +1,190 @@
package test
import (
"fmt"
"strconv"
"strings"
"github.com/pilosa/pilosa/pql"
)
type Args map[string]interface{}
type Calls []*pql.Call
func PQL(calls ...*pql.Call) *pql.Query {
return &pql.Query{Calls: calls}
}
func Row(frame string, row int) *pql.Call {
return &pql.Call{
Name: "Row",
Args: Args{
"frame": frame,
"row": row,
},
}
}
func mutationArgs(args ...interface{}) Args {
rargs := make(Args)
for _, arg := range args {
switch v := arg.(type) {
case int:
rargs["column"] = v
case string:
if strings.Contains(v, "=") {
parts := strings.Split(v, "=")
rargs["frame"] = parts[0]
i, _ := strconv.ParseInt(parts[1], 10, 64)
rargs["value"] = i
} else {
rargs["timestamp"] = v
}
default:
fmt.Printf("wat %T!\n", v)
}
}
return rargs
}
func Set(args ...interface{}) *pql.Call {
return &pql.Call{Name: "Set", Args: mutationArgs(args...)}
}
func Clear(args ...interface{}) *pql.Call {
return &pql.Call{Name: "Clear", Args: mutationArgs(args...)}
}
func magic(args ...interface{}) (Args, Calls) {
var (
rargs Args
calls Calls
)
for _, arg := range args {
switch v := arg.(type) {
case Args:
rargs = v
case []*pql.Call:
calls = append(calls, v...)
default:
fmt.Printf("wat %T!\n", v)
}
}
return rargs, calls
}
func Count(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Count", Args: kvargs, Children: children}
}
func Union(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Union", Args: kvargs, Children: children}
}
func Intersect(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Intersect", Args: kvargs, Children: children}
}
func Difference(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Difference", Args: kvargs, Children: children}
}
func Xor(args ...*pql.Call) *pql.Call {
kvargs, children := magic(args)
return &pql.Call{Name: "Xor", Args: kvargs, Children: children}
}
func Between(frame string, min, max int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.BETWEEN,
"Value": []int{min, max},
},
}
}
func Lt(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.LT,
"Value": column,
},
}
}
func Lte(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.LTE,
"Value": column,
},
}
}
func Gt(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.GT,
"Value": column,
},
}
}
func Gte(frame string, column int) *pql.Call {
return &pql.Call{
Name: "Range",
Args: Args{
"Op": pql.GTE,
"Value": column,
},
}
}
func CompareCall(a, b *pql.Call) bool {
if a.Name != b.Name {
return false
}
for k, i := range a.Args {
switch v := i.(type) {
case []int:
bside := b.Args[k]
for j := range v {
if v[j] != bside.([]int)[j] {
return false
}
}
default:
if b.Args[k] != i {
return false
}
}
}
if len(a.Children) == len(b.Children) {
for i := range a.Children {
if !CompareCall(a.Children[i], b.Children[i]) {
return false
}
}
} else {
return false
}
return true
}
func Compare(a, b *pql.Query) bool {
for i := range a.Calls {
if !CompareCall(a.Calls[i], b.Calls[i]) {
return false
}
}
return true
}

219
internal/test/querygenerator_test.go Normal file
View file

@ -0,0 +1,219 @@
package test
import (
"testing"
"github.com/pilosa/pilosa/pql"
)
func TestPQL_Generator(t *testing.T) {
t.Run("pql.Query generator", func(t *testing.T) {
for _, u := range []struct {
pql string
calc *pql.Query
exp *pql.Query
}{
{
pql: "Union(Row(aaa=10),Row(bbb=9))",
calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Union",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Intersect(Row(aaa=10),Row(bbb=9))",
calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Intersect",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Difference(Row(aaa=10),Row(bbb=9))",
calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Difference",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
},
},
},
{
pql: "Range(bbb > 20)",
calc: PQL(Gt("bbb", 20)),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Range",
Args: map[string]interface{}{
"Op": pql.GT,
"Value": 20,
},
},
},
},
},
{
pql: "Range(10 < bbb < 20)",
calc: PQL(Between("bbb", 10, 20)),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Range",
Args: map[string]interface{}{
"Op": pql.BETWEEN,
"Value": []int{10, 20},
},
},
},
},
},
{
pql: "Set(10, aaa=9)",
calc: PQL(Set(10, "aaa=9")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Set",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
},
},
},
},
},
{
pql: "Clear(10, aaa=10)",
calc: PQL(Clear(10, "aaa=9")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Clear",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
},
},
},
},
},
{
pql: `Set(10, aaa=10, "2017-03-02T03:00")`,
calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Set",
Args: map[string]interface{}{
"frame": "aaa",
"value": int64(9),
"column": 10,
"timestamp": "2017-03-02T03:00",
},
},
},
},
},
{
pql: `Count(Row(aaa=10))`,
calc: PQL(Count(Row("aaa", 10))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Count",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
},
},
},
},
},
{
pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))",
calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))),
exp: &pql.Query{
Calls: []*pql.Call{
{
Name: "Intersect",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Union",
Args: map[string]interface{}{},
Children: []*pql.Call{
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 10},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "bbb", "row": 9},
},
},
},
{
Name: "Row",
Args: map[string]interface{}{"frame": "aaa", "row": 12},
},
},
},
},
},
},
} {
if !Compare(u.calc, u.exp) {
t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql)
}
}
})
}

14
mock/mock.go Normal file
View file

@ -0,0 +1,14 @@
package mock
type ReadCloser struct {
ReadFunc func(p []byte) (int, error)
CloseFunc func() error
}
func (rc *ReadCloser) Read(p []byte) (int, error) {
return rc.ReadFunc(p)
}
func (rc *ReadCloser) Close() error {
return rc.CloseFunc()
}

38
mock/translator.go Normal file
View file

@ -0,0 +1,38 @@
package mock
import (
"context"
"io"
"github.com/pilosa/pilosa"
)
var _ pilosa.TranslateStore = (*TranslateStore)(nil)
type TranslateStore struct {
TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error)
TranslateColumnToStringFunc func(index string, values uint64) (string, error)
TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error)
TranslateRowToStringFunc func(index, frame string, values uint64) (string, error)
ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error)
}
func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) {
return s.TranslateColumnsToUint64Func(index, values)
}
func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) {
return s.TranslateColumnToStringFunc(index, values)
}
func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) {
return s.TranslateRowsToUint64Func(index, frame, values)
}
func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) {
return s.TranslateRowToStringFunc(index, frame, value)
}
func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) {
return s.ReaderFunc(ctx, off)
}

113
pilosa.go
View file

@ -16,9 +16,7 @@ package pilosa
import (
"errors"
"net"
"regexp"
"strings"
"github.com/pilosa/pilosa/internal"
)
@ -31,26 +29,26 @@ var (
ErrIndexExists = errors.New("index already exists")
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
// ErrFieldRequired is returned when no field is specified.
ErrFieldRequired = errors.New("field required")
ErrFieldExists = errors.New("field already exists")
ErrFieldNotFound = errors.New("field not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrBSIGroupNotFound = errors.New("bsigroup not found")
ErrBSIGroupExists = errors.New("bsigroup already exists")
ErrBSIGroupNameRequired = errors.New("bsigroup name required")
ErrInvalidBSIGroupType = errors.New("invalid bsigroup type")
ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range")
ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type")
ErrBSIGroupValueTooLow = errors.New("bsigroup value too low")
ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]")
ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]")
ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
// ErrFragmentNotFound is returned when a fragment does not exist.
@ -63,6 +61,8 @@ var (
ErrNodeIDNotExists = errors.New("node with provided ID does not exist")
ErrNodeNotCoordinator = errors.New("node is not the coordinator")
ErrResizeNotRunning = errors.New("no resize job currently running")
ErrNotImplemented = errors.New("not implemented")
)
// ApiMethodNotAllowedError wraps an error value indicating that a particular
@ -78,27 +78,28 @@ type BadRequestError struct {
error
}
// Regular expression to validate index and frame names.
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
Key string `json:"key,omitempty"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
// encodeColumnAttrSets converts a into its internal representation.
func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
// EncodeColumnAttrSets converts a into its internal representation.
func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
other := make([]*internal.ColumnAttrSet, len(a))
for i := range a {
other[i] = encodeColumnAttrSet(a[i])
other[i] = EncodeColumnAttrSet(a[i])
}
return other
}
// encodeColumnAttrSet converts set into its internal representation.
func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
// EncodeColumnAttrSet converts set into its internal representation.
func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
return &internal.ColumnAttrSet{
ID: set.ID,
Attrs: encodeAttrs(set.Attrs),
@ -108,26 +109,16 @@ func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// ValidateName ensures that the name is a valid format.
func ValidateName(name string) error {
// validateName ensures that the name is a valid format.
func validateName(name string) error {
if !nameRegexp.Match([]byte(name)) {
return ErrName
}
return nil
}
// StringInSlice checks for substring a in the slice.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
// StringSlicesAreEqual determines if two string slices are equal.
func StringSlicesAreEqual(a, b []string) bool {
// stringSlicesAreEqual determines if two string slices are equal.
func stringSlicesAreEqual(a, b []string) bool {
if a == nil && b == nil {
return true
@ -150,54 +141,6 @@ func StringSlicesAreEqual(a, b []string) bool {
return true
}
// SliceDiff returns the difference between two uint64 slices.
func SliceDiff(a, b []uint64) []uint64 {
m := make(map[uint64]uint64)
for _, y := range b {
m[y]++
}
var ret []uint64
for _, x := range a {
if m[x] > 0 {
m[x]--
continue
}
ret = append(ret, x)
}
return ret
}
// ContainsSubstring checks to see if substring a is contained in any string in the slice.
func ContainsSubstring(a string, list []string) bool {
for _, b := range list {
if strings.Contains(b, a) {
return true
}
}
return false
}
// HostToIP converts host to an IP4 address based on net.LookupIP().
func HostToIP(host string) string {
// if host is not an IP addr, check net.LookupIP()
if net.ParseIP(host) == nil {
hosts, err := net.LookupIP(host)
if err != nil {
return host
}
for _, h := range hosts {
// this restricts pilosa to IP4
if h.To4() != nil {
return h.String()
}
}
}
return host
}
// AddressWithDefaults converts addr into a valid address,
// using defaults when necessary.
func AddressWithDefaults(addr string) (*URI, error) {

43
pilosa_internal_test.go Normal file
View file

@ -0,0 +1,43 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"testing"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if validateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if validateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}

48
pilosa_test.go
View file

@ -22,54 +22,6 @@ import (
_ "github.com/pilosa/pilosa/test"
)
func TestValidateName(t *testing.T) {
names := []string{
"a", "ab", "ab1", "b-c", "d_e",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
}
for _, name := range names {
if pilosa.ValidateName(name) != nil {
t.Fatalf("Should be valid index name: %s", name)
}
}
}
func TestValidateNameInvalid(t *testing.T) {
names := []string{
"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
}
for _, name := range names {
if pilosa.ValidateName(name) == nil {
t.Fatalf("Should be invalid index name: %s", name)
}
}
}
func TestStringInSlice(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "localhost:10101"
if !pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s in %v", substr, list)
}
substr = "10101"
if pilosa.StringInSlice(substr, list) {
t.Fatalf("Expected substring %s not in %v", substr, list)
}
}
func TestContainsSubstring(t *testing.T) {
list := []string{"localhost:10101", "localhost:10102", "localhost:10103"}
substr := "10101"
if !pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s contained in %v", substr, list)
}
substr = "4000"
if pilosa.ContainsSubstring(substr, list) {
t.Fatalf("Expected substring %s in not contained in %v", substr, list)
}
}
func TestAddressWithDefaults(t *testing.T) {
tests := []struct {
addr string

33
pql/ast.go
View file

@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int {
return n
}
// HasKeys returns true if any call in the query uses keys and requires translation to ids.
func (q *Query) HasKeys() bool {
for _, call := range q.Calls {
if call.Args["col"] != nil {
if _, ok := call.Args["col"].(string); ok {
return true
}
}
if call.Args["row"] != nil {
if _, ok := call.Args["row"].(string); ok {
return true
}
}
}
return false
}
// String returns a string representation of the query.
func (q *Query) String() string {
a := make([]string, len(q.Calls))
@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
}
}
// StringArg is for reading the value at key from call.Args as a string. If the
// key is not in Call.Args, the value of the returned bool will be false, and
// the error will be nil. An error is returned if the value is not a string.
func (c *Call) StringArg(key string) (string, bool, error) {
val, ok := c.Args[key]
if !ok {
return "", false, nil
}
switch tval := val.(type) {
case string:
return tval, true, nil
default:
return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval)
}
}
// Keys returns a list of argument keys in sorted order.
func (c *Call) Keys() []string {
a := make([]string, 0, len(c.Args))

4
pql/ast_test.go
View file

@ -33,11 +33,11 @@ func TestCall_String(t *testing.T) {
c := &pql.Call{
Name: "Range",
Args: map[string]interface{}{
"frame": "f",
"other": "f",
"field0": &pql.Condition{Op: pql.GTE, Value: 10},
},
}
if s := c.String(); s != `Range(field0 >= 10, frame="f")` {
if s := c.String(); s != `Range(field0 >= 10, other="f")` {
t.Fatalf("unexpected string: %s", s)
}
})

10
pql/parser_test.go
View file

@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with both child calls and arguments.
t.Run("ChildrenAndArguments", func(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`)
q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -143,9 +143,9 @@ func TestParser_Parse(t *testing.T) {
Name: "TopN",
Children: []*pql.Call{{
Name: "Bitmap",
Args: map[string]interface{}{"id": int64(100), "frame": "other"},
Args: map[string]interface{}{"id": int64(100), "field": "other"},
}},
Args: map[string]interface{}{"n": int64(3), "frame": "f"},
Args: map[string]interface{}{"n": int64(3), "field": "f"},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
@ -154,14 +154,14 @@ func TestParser_Parse(t *testing.T) {
// Parse a list argument.
t.Run("ListArgument", func(t *testing.T) {
q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`)
q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Args: map[string]interface{}{
"frame": "f",
"field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
},
},

18
row.go
View file

@ -27,6 +27,9 @@ import (
type Row struct {
segments []RowSegment
// String keys translated to/from segment columns.
Keys []string
// Attributes associated with the row.
Attrs map[string]interface{}
}
@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) {
return s.ClearBit(i)
}
// Segments returns a list of all segments in the row.
func (r *Row) Segments() []RowSegment {
return r.segments
}
// segment returns a segment for a given slice.
// Returns nil if segment does not exist.
func (r *Row) segment(slice uint64) *RowSegment {
@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
Keys []string `json:"keys,omitempty"`
}
o.Columns = r.Columns()
o.Keys = r.Keys
o.Attrs = r.Attrs
if o.Attrs == nil {
@ -261,8 +271,8 @@ func (r *Row) Columns() []uint64 {
return a
}
// encodeRow converts r into its internal representation.
func encodeRow(r *Row) *internal.Row {
// EncodeRow converts r into its internal representation.
func EncodeRow(r *Row) *internal.Row {
if r == nil {
return nil
}
@ -273,8 +283,8 @@ func encodeRow(r *Row) *internal.Row {
}
}
// decodeRow converts r from its internal representation.
func decodeRow(pr *internal.Row) *Row {
// DecodeRow converts r from its internal representation.
func DecodeRow(pr *internal.Row) *Row {
if pr == nil {
return nil
}

145
server.go
View file

@ -19,7 +19,6 @@ import (
"fmt"
"log"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
@ -53,17 +52,16 @@ type Server struct {
closing chan struct{}
// Internal
Holder *Holder
Cluster *Cluster
diagnostics *DiagnosticsCollector
executor *Executor
Holder *Holder
Cluster *Cluster
TranslateFile *TranslateFile
diagnostics *DiagnosticsCollector
executor *Executor
// External
handler *Handler
handler Handler
Broadcaster Broadcaster
BroadcastReceiver BroadcastReceiver
Gossiper Gossiper
remoteClient *http.Client
systemInfo SystemInfo
gcNotifier GCNotifier
NewAttrStore func(string) AttrStore
@ -77,6 +75,8 @@ type Server struct {
diagnosticInterval time.Duration
maxWritesPerRequest int
primaryTranslateStore TranslateStore
defaultClient InternalClient
dataDir string
}
@ -127,7 +127,7 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption {
}
}
func OptServerHandler(h *Handler) ServerOption {
func OptServerHandler(h Handler) ServerOption {
return func(s *Server) error {
s.handler = h
return nil
@ -162,12 +162,18 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption {
}
}
func OptServerRemoteClient(c *http.Client) ServerOption {
func OptServerInternalClient(c InternalClient) ServerOption {
return func(s *Server) error {
s.executor = NewExecutor(c)
s.remoteClient = c
s.defaultClient = NewInternalHTTPClientFromURI(nil, c)
s.Cluster.RemoteClient = c
s.executor = NewExecutor(OptExecutorInternalQueryClient(c))
s.defaultClient = c
s.Cluster.InternalClient = c
return nil
}
}
func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
return func(s *Server) error {
s.primaryTranslateStore = store
return nil
}
}
@ -203,15 +209,10 @@ func OptServerURI(uri *URI) ServerOption {
// NewServer returns a new instance of Server.
func NewServer(opts ...ServerOption) (*Server, error) {
handler, err := NewHandler()
if err != nil {
return nil, errors.Wrap(err, "initializing handler")
}
s := &Server{
closing: make(chan struct{}),
Cluster: NewCluster(),
Holder: NewHolder(),
handler: handler,
Broadcaster: NopBroadcaster,
BroadcastReceiver: NopBroadcastReceiver,
diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer),
@ -249,6 +250,14 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.Cluster.Logger = s.logger
s.Cluster.Holder = s.Holder
// Initialize translation database.
s.TranslateFile = NewTranslateFile()
s.TranslateFile.Path = filepath.Join(path, "keys")
s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore
if err := s.TranslateFile.Open(); err != nil {
return nil, err
}
// update URI port with actual listener port. TODO this should probably be done outside of here.
if s.URI.Port() == 0 {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
@ -267,8 +276,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.Holder = s.Holder
s.executor.Node = node
s.executor.Cluster = s.Cluster
s.executor.TranslateStore = s.TranslateFile
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.handler.API.Executor = s.executor
s.handler.GetAPI().Executor = s.executor
s.handler.GetAPI().TranslateStore = s.TranslateFile
return s, nil
}
@ -300,27 +311,18 @@ func (s *Server) Open() error {
s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest
// Initialize HTTP handler.
s.handler.API.Holder = s.Holder
s.handler.API.Broadcaster = s.Broadcaster
s.handler.API.BroadcastHandler = s
s.handler.API.StatusHandler = s
s.handler.API.Cluster = s.Cluster
api := s.handler.GetAPI()
api.Holder = s.Holder
api.Broadcaster = s.Broadcaster
api.BroadcastHandler = s
api.StatusHandler = s
api.Cluster = s.Cluster
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
// Serve HTTP.
go func() {
server := &http.Server{Handler: s.handler}
go func() {
<-s.closing
server.Close()
}()
err := server.Serve(s.ln)
if err != nil && err.Error() != "http: Server closed" {
s.logger.Printf("HTTP handler terminated with error: %s\n", err)
}
}()
// Serve handler.
go s.handler.Serve(s.ln, s.closing)
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
@ -328,7 +330,7 @@ func (s *Server) Open() error {
}
// Open Cluster management.
if err := s.Cluster.Open(); err != nil {
if err := s.Cluster.open(); err != nil {
return fmt.Errorf("opening Cluster: %v", err)
}
@ -336,7 +338,7 @@ func (s *Server) Open() error {
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
if err := s.Cluster.SetNodeState(NodeStateReady); err != nil {
if err := s.Cluster.setNodeState(NodeStateReady); err != nil {
return fmt.Errorf("setting nodeState: %v", err)
}
@ -345,7 +347,7 @@ func (s *Server) Open() error {
// the cluster without waiting for data to load on the coordinator. Before
// this starts, the joins are queued up in the Cluster.joiningLeavingNodes
// buffered channel.
s.Cluster.ListenForJoins()
s.Cluster.listenForJoins()
// Start background monitoring.
s.wg.Add(3)
@ -366,11 +368,14 @@ func (s *Server) Close() error {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.Close()
s.Cluster.close()
}
if s.Holder != nil {
s.Holder.Close()
}
if s.TranslateFile != nil {
s.TranslateFile.Close()
}
return nil
}
@ -420,7 +425,6 @@ func (s *Server) monitorAntiEntropy() {
syncer.Node = s.Cluster.Node
syncer.Cluster = s.Cluster
syncer.Closing = s.closing
syncer.RemoteClient = s.remoteClient
syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer")
// Sync holders.
@ -455,71 +459,60 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err := s.Holder.DeleteIndex(obj.Index); err != nil {
return err
}
case *internal.CreateFrameMessage:
case *internal.CreateFieldMessage:
idx := s.Holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := decodeFrameOptions(obj.Meta)
_, err := idx.CreateFrame(obj.Frame, *opt)
opt := decodeFieldOptions(obj.Meta)
_, err := idx.CreateField(obj.Field, *opt)
if err != nil {
return err
}
case *internal.DeleteFrameMessage:
idx := s.Holder.Index(obj.Index)
if err := idx.DeleteFrame(obj.Frame); err != nil {
return err
}
case *internal.CreateFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
field := decodeField(obj.Field)
if err := f.CreateField(field); err != nil {
return err
}
case *internal.DeleteFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if err := f.DeleteField(obj.Field); err != nil {
idx := s.Holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
f := s.Holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
return fmt.Errorf("Local Field not found: %s", obj.Field)
}
_, _, err := f.createViewIfNotExistsBase(obj.View)
if err != nil {
return err
}
case *internal.DeleteViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
f := s.Holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
return fmt.Errorf("Local Field not found: %s", obj.Field)
}
err := f.DeleteView(obj.View)
if err != nil {
return err
}
case *internal.ClusterStatus:
err := s.Cluster.MergeClusterStatus(obj)
err := s.Cluster.mergeClusterStatus(obj)
if err != nil {
return err
}
case *internal.ResizeInstruction:
err := s.Cluster.FollowResizeInstruction(obj)
err := s.Cluster.followResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
err := s.Cluster.MarkResizeInstructionComplete(obj)
err := s.Cluster.markResizeInstructionComplete(obj)
if err != nil {
return err
}
case *internal.SetCoordinatorMessage:
s.Cluster.SetCoordinator(DecodeNode(obj.New))
s.Cluster.setCoordinator(DecodeNode(obj.New))
case *internal.UpdateCoordinatorMessage:
s.Cluster.UpdateCoordinator(DecodeNode(obj.New))
s.Cluster.updateCoordinator(DecodeNode(obj.New))
case *internal.NodeStateMessage:
err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State)
err := s.Cluster.receiveNodeState(obj.NodeID, obj.State)
if err != nil {
return err
}
@ -553,7 +546,7 @@ func (s *Server) SendSync(pb proto.Message) error {
// SendAsync represents an implementation of Broadcaster.
func (s *Server) SendAsync(pb proto.Message) error {
return s.Gossiper.SendAsync(pb)
return ErrNotImplemented
}
// SendTo represents an implementation of Broadcaster.
@ -657,7 +650,7 @@ func (s *Server) monitorDiagnostics() {
s.diagnostics.Logger = s.logger
s.diagnostics.SetVersion(Version)
s.diagnostics.Set("Host", s.URI.host)
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ","))
s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ","))
s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes))
s.diagnostics.Set("NumCPU", runtime.NumCPU())
s.diagnostics.Set("NodeID", s.NodeID)
@ -666,7 +659,7 @@ func (s *Server) monitorDiagnostics() {
// Flush the diagnostics metrics at startup, then on each tick interval
flush := func() {
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
if err == nil {
s.diagnostics.Set("OpenFiles", openFiles)
}
@ -723,7 +716,7 @@ func (s *Server) monitorRuntime() {
// Record the number of go routines.
s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
openFiles, err := CountOpenFiles()
openFiles, err := countOpenFiles()
// Open File handles.
if err == nil {
s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0)
@ -739,8 +732,8 @@ func (s *Server) monitorRuntime() {
}
}
// CountOpenFiles on operating systems that support lsof.
func CountOpenFiles() (int, error) {
// countOpenFiles on operating systems that support lsof.
func countOpenFiles() (int, error) {
switch runtime.GOOS {
case "darwin", "linux", "unix", "freebsd":
// -b option avoid kernel blocks
@ -754,9 +747,9 @@ func CountOpenFiles() (int, error) {
return len(lines), nil
case "windows":
// TODO: count open file handles on windows
return 0, errors.New("CountOpenFiles() on Windows is not supported")
return 0, errors.New("countOpenFiles() on Windows is not supported")
default:
return 0, errors.New("CountOpenFiles() on this OS is not supported")
return 0, errors.New("countOpenFiles() on this OS is not supported")
}
}

48
server/cluster_test.go
View file

@ -42,7 +42,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Expected indexes and Frames
// Expected indexes and Fields
expected := map[string][]string{
"i": []string{"f"},
}
@ -51,14 +51,14 @@ func TestMain_SendReceiveMessage(t *testing.T) {
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Make sure node0 knows about the index and frame created.
// Make sure node0 knows about the index and field created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
@ -66,15 +66,15 @@ func TestMain_SendReceiveMessage(t *testing.T) {
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
for _, field := range idx.Fields {
received0[idx.Name] = append(received0[idx.Name], field.Name)
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
}
// Make sure node1 knows about the index and frame created.
// Make sure node1 knows about the index and field created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
@ -82,8 +82,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
for _, field := range idx.Fields {
received1[idx.Name] = append(received1[idx.Name], field.Name)
}
}
if !reflect.DeepEqual(received1, expected) {
@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -206,10 +206,10 @@ func TestClusterResize_AddNode(t *testing.T) {
// Create a client for each node.
client0 := m0.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -250,17 +250,17 @@ func TestClusterResize_AddNode(t *testing.T) {
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=1300000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=1300000)
`); err != nil {
t.Fatal(err)
}
@ -302,17 +302,17 @@ func TestClusterResize_AddNode(t *testing.T) {
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -455,10 +455,10 @@ func TestClusterResize_RemoveNode(t *testing.T) {
t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) {
client0 := m0.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setColumns := ""
for i := 0; i < 20; i++ {
setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setColumns); err != nil {

7
server/config.go
View file

@ -61,7 +61,7 @@ type Config struct {
Handler struct {
// CORS Allowed Origins
AllowedOrigins []string `toml:"allowed-origins"`
}
} `toml:"handler"`
// TLS
TLS TLSConfig `toml:"tls"`
@ -78,6 +78,11 @@ type Config struct {
// Gossip config is based around memberlist.Config.
Gossip gossip.Config `toml:"gossip"`
// Translation config supports translation store replication.
Translation struct {
PrimaryURL string `toml:"primary-url"`
}
AntiEntropy struct {
Interval toml.Duration `toml:"interval"`
} `toml:"anti-entropy"`

52
server/server.go
View file

@ -25,7 +25,6 @@ import (
"log"
"math/rand"
"net"
"net/http"
"os"
"os/signal"
"strconv"
@ -39,7 +38,7 @@ import (
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/statsd"
"github.com/pkg/errors"
)
@ -165,14 +164,17 @@ func (m *Command) SetupServer() error {
}
m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
handler, err := pilosa.NewHandler(pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins))
api := pilosa.NewAPI()
api.Logger = m.logger
handler, err := http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(api),
http.OptHandlerLogger(m.logger),
)
if err != nil {
return errors.Wrap(err, "wrapping handler")
}
handler.Logger = m.logger
handler.FileSystem = &statik.FileSystem{}
handler.API = pilosa.NewAPI()
handler.API.Logger = m.logger
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
@ -213,8 +215,13 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "getting listener")
}
c := GetHTTPClient(TLSConfig)
handler.API.RemoteClient = c
c := http.GetHTTPClient(TLSConfig)
// Setup connection to primary store if this is a replica.
var primaryTranslateStore pilosa.TranslateStore
if m.Config.Translation.PrimaryURL != "" {
primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL)
}
m.Server, err = pilosa.NewServer(
pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)),
@ -233,32 +240,13 @@ func (m *Command) SetupServer() error {
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerRemoteClient(c),
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
)
return errors.Wrap(err, "new server")
}
func GetHTTPClient(t *tls.Config) *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 1000,
MaxIdleConnsPerHost: 200,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
if t != nil {
transport.TLSClientConfig = t
}
return &http.Client{Transport: transport}
}
// SetupNetworking sets up internode communication based on the configuration.
func (m *Command) SetupNetworking() error {
@ -280,7 +268,6 @@ func (m *Command) SetupNetworking() error {
m.Server.Broadcaster = pilosa.NopBroadcaster
m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes)
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
m.Server.Gossiper = pilosa.NopGossiper
return nil
}
@ -325,7 +312,6 @@ func (m *Command) SetupNetworking() error {
m.Server.Cluster.MemberSet = gossipMemberSet
m.Server.Broadcaster = m.Server
m.Server.BroadcastReceiver = gossipMemberSet
m.Server.Gossiper = gossipMemberSet
return nil
}
@ -355,7 +341,7 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) {
case "nop", "none":
return pilosa.NopStatsClient, nil
default:
return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].")
return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].", name)
}
}

123
server/server_test.go
View file

@ -21,14 +21,14 @@ import (
"io/ioutil"
"math/rand"
"reflect"
"runtime"
"sort"
"strings"
"testing"
"testing/quick"
"github.com/BurntSushi/toml"
"github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
)
@ -44,27 +44,27 @@ func TestMain_Set_Quick(t *testing.T) {
defer m.Close()
// Create client.
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
t.Fatal(err)
}
// Execute SetBit() commands.
for _, cmd := range cmds {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
}
if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") {
if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists {
t.Fatal(err)
}
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil {
t.Fatal(err)
}
}
// Validate data.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, columnIDs := range frameSet {
for field, fieldSet := range SetCommands(cmds).Fields() {
for id, columnIDs := range fieldSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -86,8 +86,8 @@ func TestMain_Set_Quick(t *testing.T) {
}
// Validate data after reopening.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, columnIDs := range frameSet {
for field, fieldSet := range SetCommands(cmds).Fields() {
for id, columnIDs := range fieldSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -119,49 +119,49 @@ func TestMain_SetRowAttrs(t *testing.T) {
m := test.MustRunMain()
defer m.Close()
// Create frames.
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set columns on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
// Set columns on different rows in different fields.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -196,18 +196,18 @@ func TestMain_SetColumnAttrs(t *testing.T) {
m := test.MustRunMain()
defer m.Close()
// Create frames.
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil {
t.Fatal(err)
}
@ -217,7 +217,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
@ -262,21 +262,6 @@ func tempMkdir(t *testing.T) string {
return dir
}
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported CountOpenFiles test on Windows.")
}
count, err := pilosa.CountOpenFiles()
if err != nil {
t.Errorf("CountOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("CountOpenFiles returned invalid value 0.")
}
}
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)
@ -286,15 +271,15 @@ func TestMain_RecalculateHashes(t *testing.T) {
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create frame:", err)
if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create field:", err)
}
// Set some columns
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID))
data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID))
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
@ -311,7 +296,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(frame="f")`)
res, err := m.Query("i", "", `TopN(field="f")`)
if err != nil {
t.Fatal(err)
}
@ -325,37 +310,37 @@ func TestMain_RecalculateHashes(t *testing.T) {
// SetCommand represents a command to set a column.
type SetCommand struct {
ID uint64
Frame string
Field string
ColumnID uint64
}
type SetCommands []SetCommand
// Frames returns the set of column ids for each frame/row.
func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
// Fields returns the set of column ids for each field/row.
func (a SetCommands) Fields() map[string]map[uint64][]uint64 {
// Create a set of unique commands.
m := make(map[SetCommand]struct{})
for _, cmd := range a {
m[cmd] = struct{}{}
}
// Build unique ids for each frame & row.
frames := make(map[string]map[uint64][]uint64)
// Build unique ids for each field & row.
fields := make(map[string]map[uint64][]uint64)
for cmd := range m {
if frames[cmd.Frame] == nil {
frames[cmd.Frame] = make(map[uint64][]uint64)
if fields[cmd.Field] == nil {
fields[cmd.Field] = make(map[uint64][]uint64)
}
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID)
fields[cmd.Field][cmd.ID] = append(fields[cmd.Field][cmd.ID], cmd.ColumnID)
}
// Sort each set of column ids.
for _, frame := range frames {
for id := range frame {
sort.Sort(uint64Slice(frame[id]))
for _, field := range fields {
for id := range field {
sort.Sort(uint64Slice(field[id]))
}
}
return frames
return fields
}
// GenerateSetCommands generates random SetCommand objects.
@ -364,7 +349,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
for i := range cmds {
cmds[i] = SetCommand{
ID: uint64(rand.Intn(1000)),
Frame: "x",
Field: "x",
ColumnID: uint64(rand.Intn(10)),
}
}
@ -374,7 +359,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
// ParseConfig parses s into a Config.
func ParseConfig(s string) (server.Config, error) {
var c server.Config
_, err := toml.Decode(s, &c)
err := toml.Unmarshal([]byte(s), &c)
return c, err
}

30
test/client.go → server_internal_test.go
View file

@ -12,24 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package test
package pilosa
import (
"net/http"
"github.com/pilosa/pilosa"
"runtime"
"testing"
)
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*pilosa.InternalHTTPClient
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string, h *http.Client) *Client {
c, err := pilosa.NewInternalHTTPClient(host, h)
if err != nil {
panic(err)
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported countOpenFiles test on Windows.")
}
count, err := countOpenFiles()
if err != nil {
t.Errorf("countOpenFiles failed: %s", err)
}
if count == 0 {
t.Error("countOpenFiles returned invalid value 0.")
}
return &Client{InternalHTTPClient: c}
}

18
server_test.go
View file

@ -1,3 +1,17 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
@ -19,9 +33,9 @@ func TestMonitorAntiEntropy(t *testing.T) {
if err != nil {
t.Fatalf("creating index: %v", err)
}
err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{})
err = client.CreateField(context.Background(), "balh", "fralh", pilosa.FieldOptions{})
if err != nil {
t.Fatalf("creating frame: %v", err)
t.Fatalf("creating field: %v", err)
}
time.Sleep(time.Millisecond * 2)

1
statik/.gitignore vendored
View file

@ -1 +0,0 @@
/statik.go

37
statik/filesystem.go
View file

@ -1,37 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//go:generate statik -src=../webui -dest=..
//
// Package statik contains static assets for the Web UI. `go generate` or
// `make generate-statik` will produce statik.go, which is ignored by git.
package statik
import (
"net/http"
"github.com/pilosa/pilosa"
"github.com/rakyll/statik/fs"
)
// Ensure nopFileSystem implements interface.
var _ pilosa.FileSystem = &FileSystem{}
// FileSystem represents a static FileSystem.
type FileSystem struct{}
// New is a statik implementation of FileSystem New method.
func (s *FileSystem) New() (http.FileSystem, error) {
return fs.New()
}

10
statik/statik.go Normal file
View file

File diff suppressed because one or more lines are too long

6
stats.go
View file

@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient {
return &ExpvarStatsClient{
m: m,
tags: UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
}
}
@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error {
return nil
}
// UnionStringSlice returns a sorted set of tags which combine a & b.
func UnionStringSlice(a, b []string) []string {
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)

84
stats_test.go
View file

@ -36,45 +36,45 @@ func TestMultiStatClient_Expvar(t *testing.T) {
ms[0] = c
hldr.Stats = ms
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
hldr.ClearBit("d", "f", 0, 1)
if pilosa.Expvar.String() != `{"index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"})
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Gauge creates a unique key, subsequent Gauge calls will overwrite
hldr.Stats.Gauge("g", 5, 1.0)
hldr.Stats.Gauge("g", 8, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Set creates a unique key, subsequent sets will overwrite
hldr.Stats.Set("s", "4", 1.0)
hldr.Stats.Set("s", "7", 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Record timing duration and a uniquely Set key/value
dur, _ := time.ParseDuration("123us")
hldr.Stats.Timing("tt", dur, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Expvar histogram is implemented as a gauge
hldr.Stats.Histogram("hh", 3, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
@ -88,14 +88,14 @@ func TestStatsCount_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, SliceWidth)
hldr.SetBit("d", "f", 0, SliceWidth+2)
// Execute query.
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "TopN" {
@ -109,7 +109,7 @@ func TestStatsCount_TopN(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -121,10 +121,10 @@ func TestStatsCount_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
@ -138,7 +138,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -150,17 +150,17 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
frame := e.Holder.Frame("d", "f")
if frame == nil {
t.Fatal("frame not found")
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
field := e.Holder.Field("d", "f")
if field == nil {
t.Fatal("field not found")
}
frame.Stats = &MockStats{
field.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "SetRowAttrs" {
t.Errorf("Expected SetRowAttrs, Results %s", name)
@ -168,7 +168,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -180,11 +180,11 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
hldr.SetBit("d", "f", 10, 0)
hldr.SetBit("d", "f", 10, 1)
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1))
idx := e.Holder.Index("d")
if idx == nil {
t.Fatal("idex not found")
@ -199,7 +199,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -257,7 +257,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) {
}
}
func TestStatsCount_CreateFrame(t *testing.T) {
func TestStatsCount_CreateField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -272,8 +272,8 @@ func TestStatsCount_CreateFrame(t *testing.T) {
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createFrame" {
t.Errorf("Expected createFrame, Results %s", name)
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
@ -282,13 +282,13 @@ func TestStatsCount_CreateFrame(t *testing.T) {
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteFrame(t *testing.T) {
func TestStatsCount_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -298,13 +298,13 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
called := false
// Create index.
indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil {
if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteFrame" {
t.Errorf("Expected deleteFrame, Results %s", name)
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
@ -313,7 +313,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}

38
statsd/statsd.go
View file

@ -15,6 +15,7 @@
package statsd
import (
"sort"
"time"
"github.com/DataDog/datadog-go/statsd"
@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string {
func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient {
return &StatsClient{
client: c.client,
tags: pilosa.UnionStringSlice(c.tags, tags),
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
}
}
@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) {
func (c *StatsClient) SetLogger(logger pilosa.Logger) {
c.logger = logger
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}

90
test/attr.go
View file

@ -1,90 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"io/ioutil"
"os"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
)
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore(string) pilosa.AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{boltdb.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() pilosa.AttrStore {
s := NewAttrStore("")
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

402
test/cluster.go
View file

@ -15,17 +15,10 @@
package test
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Hasher = newModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
ID: fmt.Sprintf("node%d", i),
URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)),
URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)),
})
}
@ -55,396 +48,19 @@ func NewCluster(n int) *pilosa.Cluster {
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// modHasher represents a simple, mod-based hashing.
type modHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
// newModHasher returns a new instance of ModHasher with n buckets.
func newModHasher() *modHasher { return &modHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
func (*modHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// NewURI is a test URI creator that intentionally swallows errors.
func NewURI(scheme, host string, port uint16) pilosa.URI {
// newURI is a test URI creator that intentionally swallows errors.
func newURI(scheme, host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetScheme(scheme)
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
func NewURIFromHostPort(host string, port uint16) pilosa.URI {
uri := pilosa.DefaultURI()
uri.SetHost(host)
uri.SetPort(port)
return *uri
}
// TestCluster represents a cluster of test nodes, each of which
// has a pilosa.Cluster.
type TestCluster struct {
Clusters []*pilosa.Cluster
common *commonClusterSettings
mu sync.RWMutex
resizing bool
resizeDone chan struct{}
}
type commonClusterSettings struct {
Nodes []*pilosa.Node
}
func (t *TestCluster) CreateIndex(name string) error {
for _, c := range t.Clusters {
if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / pilosa.SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
return c
}
}
return nil
}
// AddNode adds a node to the cluster and (potentially) starts a resize job.
func (t *TestCluster) AddNode(saveTopology bool) error {
id := len(t.Clusters)
c, err := t.addCluster(id, saveTopology)
if err != nil {
return err
}
// Send NodeJoin event to coordinator.
if id > 0 {
coord := t.Clusters[0]
ev := &pilosa.NodeEvent{
Event: pilosa.NodeJoin,
Node: c.Node,
}
if err := coord.ReceiveEvent(ev); err != nil {
return err
}
// Wait for the AddNode job to finish.
if c.State() != pilosa.ClusterStateNormal {
t.resizeDone = make(chan struct{})
t.mu.Lock()
t.resizing = true
t.mu.Unlock()
<-t.resizeDone
}
}
return nil
}
// WriteTopology writes the given topology to disk.
func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error {
if buf, err := proto.Marshal(top.Encode()); err != nil {
return err
} else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil {
return err
}
return nil
}
func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &pilosa.Node{
ID: id,
URI: uri,
}
// add URI to common
//t.common.NodeIDs = append(t.common.NodeIDs, id)
//sort.Sort(t.common.NodeIDs)
// add node to common
t.common.Nodes = append(t.common.Nodes, node)
// create node-specific temp directory
path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i))
if err != nil {
return nil, err
}
// holder
h := pilosa.NewHolder()
h.Path = path
// cluster
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
c.Path = path
c.Topology = pilosa.NewTopology()
c.Holder = h
c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes)
c.Node = node
c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator
c.Broadcaster = t
// add nodes
if saveTopology {
for _, n := range t.common.Nodes {
c.AddNode(n)
}
}
// Add this node to the TestCluster.
t.Clusters = append(t.Clusters, c)
return c, nil
}
// NewTestCluster returns a new instance of test.Cluster.
func NewTestCluster(n int) *TestCluster {
tc := &TestCluster{
common: &commonClusterSettings{},
}
// add clusters
for i := 0; i < n; i++ {
_, err := tc.addCluster(i, true)
if err != nil {
panic(err)
}
}
return tc
}
// SetState sets the state of the cluster on each node.
func (t *TestCluster) SetState(state string) {
for _, c := range t.Clusters {
c.SetState(state)
}
}
// Open opens all clusters in the test cluster.
func (t *TestCluster) Open() error {
for _, c := range t.Clusters {
if err := c.Open(); err != nil {
return err
}
if err := c.Holder.Open(); err != nil {
return err
}
if err := c.SetNodeState(pilosa.NodeStateReady); err != nil {
return err
}
}
// Start the listener on the coordinator.
if len(t.Clusters) == 0 {
return nil
}
t.Clusters[0].ListenForJoins()
return nil
}
// Close closes all clusters in the test cluster.
func (t *TestCluster) Close() error {
for _, c := range t.Clusters {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// TestCluster implements Broadcaster interface.
// SendSync is a test implemenetation of Broadcaster SendSync method.
func (t *TestCluster) SendSync(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ClusterStatus:
// Apply the send message to all nodes (except the coordinator).
for _, c := range t.Clusters {
c.MergeClusterStatus(obj)
}
t.mu.RLock()
if obj.State == pilosa.ClusterStateNormal && t.resizing {
close(t.resizeDone)
}
t.mu.RUnlock()
}
return nil
}
// SendAsync is a test implemenetation of Broadcaster SendAsync method.
func (t *TestCluster) SendAsync(pb proto.Message) error {
return nil
}
// SendTo is a test implemenetation of Broadcaster SendTo method.
func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error {
switch obj := pb.(type) {
case *internal.ResizeInstruction:
err := t.FollowResizeInstruction(obj)
if err != nil {
return err
}
case *internal.ResizeInstructionComplete:
coord := t.clusterByID(to.ID)
go coord.MarkResizeInstructionComplete(obj)
}
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error {
// Prepare the return message.
complete := &internal.ResizeInstructionComplete{
JobID: instr.JobID,
Node: instr.Node,
Error: "",
}
// Stop processing on any error.
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
// Sync the schema received in the resize instruction.
if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil {
return err
}
for _, src := range instr.Sources {
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)
if err != nil {
return err
}
}
buf := bytes.NewBuffer(nil)
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
}
}
return nil
}(); err != nil {
complete.Error = err.Error()
}
node := pilosa.DecodeNode(instr.Coordinator)
if err := t.SendTo(node, complete); err != nil {
return err
}
return nil
}

13
test/executor.go
View file

@ -15,12 +15,13 @@
package test
import (
"net/http"
gohttp "net/http"
"strings"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/inmem"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/server"
)
// Executor represents a test wrapper for pilosa.Executor.
@ -28,19 +29,21 @@ type Executor struct {
*pilosa.Executor
}
var remoteClient *http.Client
var remoteClient *gohttp.Client
func init() {
remoteClient = server.GetHTTPClient(nil)
remoteClient = http.GetHTTPClient(nil)
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the uri of the first cluster node.
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
executor := pilosa.NewExecutor(remoteClient)
client := http.NewInternalClientFromURI(nil, remoteClient)
executor := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(client))
e := &Executor{Executor: executor}
e.Holder = holder
e.Cluster = cluster
e.TranslateStore = inmem.NewTranslateStore()
e.Node = cluster.Nodes[0]
return e
}

113
test/fragment.go
View file

@ -15,9 +15,6 @@
package test
import (
"io/ioutil"
"os"
"github.com/pilosa/pilosa"
)
@ -29,113 +26,3 @@ type Fragment struct {
*pilosa.Fragment
RowAttrStore pilosa.AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
}
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore
return f
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
if cacheType == "" {
cacheType = pilosa.DefaultCacheType
}
f := NewFragment(index, frame, view, slice, cacheType)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the fragment and removes all underlying data.
func (f *Fragment) Close() error {
defer os.Remove(f.Path())
defer os.Remove(f.CachePath())
defer f.RowAttrStore.Close()
return f.Fragment.Close()
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *Fragment) Reopen() error {
cacheType := f.Fragment.CacheType
path := f.Path()
if err := f.Fragment.Close(); err != nil {
return err
}
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearColumns clears columns on a row. Panic on error.
func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// RowAttrStore provides simple storage for attributes.
type RowAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewRowAttrStore returns a new instance of RowAttrStore.
func NewRowAttrStore() *RowAttrStore {
return &RowAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// RowAttrs returns the attributes set to a row id.
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetRowAttrs assigns a set of attributes to a row id.
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {
if i%100 >= ipct {
continue
}
rowIDs = append(rowIDs, uint64(i%SliceWidth))
columnIDs = append(columnIDs, uint64(i/SliceWidth))
}
return
}

52
test/frame.go
View file

@ -23,48 +23,48 @@ import (
"github.com/pilosa/pilosa"
)
// Frame represents a test wrapper for pilosa.Frame.
type Frame struct {
*pilosa.Frame
// Field represents a test wrapper for pilosa.Field.
type Field struct {
*pilosa.Field
}
// NewFrame returns a new instance of Frame d/0.
func NewFrame() *Frame {
path, err := ioutil.TempDir("", "pilosa-frame-")
// NewField returns a new instance of Field d/0.
func NewField(opt ...pilosa.FieldOption) *Field {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", "f")
field, err := pilosa.NewField(path, "i", "f", opt...)
if err != nil {
panic(err)
}
return &Frame{Frame: frame}
return &Field{Field: field}
}
// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error.
func MustOpenFrame() *Frame {
f := NewFrame()
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(opt ...pilosa.FieldOption) *Field {
f := NewField(opt...)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the frame and removes the underlying data.
func (f *Frame) Close() error {
// Close closes the field and removes the underlying data.
func (f *Field) Close() error {
defer os.RemoveAll(f.Path())
return f.Frame.Close()
return f.Field.Close()
}
// Reopen closes the index and reopens it.
func (f *Frame) Reopen() error {
func (f *Field) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
if err := f.Field.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Frame, err = pilosa.NewFrame(path, index, name)
f.Field, err = pilosa.NewField(path, index, name)
if err != nil {
return err
}
@ -75,8 +75,8 @@ func (f *Frame) Reopen() error {
return nil
}
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
// MustSetBit sets a bit on the field. Panic on error.
func (f *Field) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {
panic(err)
@ -84,23 +84,23 @@ func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (c
return changed
}
// Ensure frame can set its cache
func TestFrame_SetCacheSize(t *testing.T) {
f := MustOpenFrame()
// Ensure field can set its cache
func TestField_SetCacheSize(t *testing.T) {
f := MustOpenField()
defer f.Close()
cacheSize := uint32(100)
// Set & retrieve frame cache size.
// Set & retrieve field cache size.
if err := f.SetCacheSize(cacheSize); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size: %d", q)
t.Fatalf("unexpected field cache size: %d", q)
}
// Reload frame and verify that it is persisted.
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size (reopen): %d", q)
t.Fatalf("unexpected field cache size (reopen): %d", q)
}
}

15
test/handler.go
View file

@ -19,25 +19,26 @@ import (
"encoding/json"
"io"
"io/ioutil"
"net/http"
gohttp "net/http"
"net/http/httptest"
"net/url"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
)
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
*http.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) {
handler, err := pilosa.NewHandler(opts...)
func NewHandler(opts ...http.HandlerOption) (*Handler, error) {
handler, err := http.NewHandler(opts...)
if err != nil {
return nil, err
}
@ -55,7 +56,7 @@ func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) {
}
// MustNewHandler returns a new instance of Handler.
func MustNewHandler(opts ...pilosa.HandlerOption) *Handler {
func MustNewHandler(opts ...http.HandlerOption) *Handler {
h, err := NewHandler(opts...)
if err != nil {
panic(err)
@ -145,8 +146,8 @@ func MustParseURLHost(rawurl string) string {
}
// MustNewHTTPRequest creates a new HTTP request. Panic on error.
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
req, err := http.NewRequest(method, urlStr, body)
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *gohttp.Request {
req, err := gohttp.NewRequest(method, urlStr, body)
if err != nil {
panic(err)
}

68
test/holder.go
View file

@ -80,19 +80,19 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
return &Index{Index: idx}
}
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
// MustCreateFieldIfNotExists returns a given field. Panic on error.
func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
return f
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
panic(err)
}
@ -107,20 +107,58 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice
return &Fragment{Fragment: frag}
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
// Row returns a Row for a given field.
func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
row, err := f.Row(rowID)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
return row
}
// ViewRow returns a Row for a given field and view.
func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
row, err := f.ViewRow(view, rowID)
if err != nil {
panic(err)
}
return row
}
// SetBit clears a bit on the given field.
func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
f.SetBit(pilosa.ViewStandard, rowID, columnID, nil)
}
// ClearBit clears a bit on the given field.
func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
f.ClearBit(pilosa.ViewStandard, rowID, columnID, nil)
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
h.SetBit(index, field, rowID, columnID)
}
}

16
test/index.go
View file

@ -73,20 +73,20 @@ func (i *Index) Reopen() error {
return nil
}
// CreateFrame creates a frame with the given options.
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrame(name, opt)
// CreateField creates a field with the given options.
func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateField(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
return &Field{Field: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrameIfNotExists(name, opt)
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateFieldIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
return &Field{Field: f}, nil
}

16
test/pilosa.go
View file

@ -19,15 +19,15 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
gohttp "net/http"
"os"
"strings"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/boltdb"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
@ -238,8 +238,8 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.InternalHTTPClient {
client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil))
func (m *Main) Client() *http.InternalClient {
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
panic(err)
}
@ -249,7 +249,7 @@ func (m *Main) Client() *pilosa.InternalHTTPClient {
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
if resp.StatusCode != gohttp.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}
return resp.Body, nil
@ -267,11 +267,11 @@ func (m *Main) RecalculateCaches() error {
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.
func MustDo(method, urlStr string, body string) *httpResponse {
req, err := http.NewRequest(method, urlStr, strings.NewReader(body))
req, err := gohttp.NewRequest(method, urlStr, strings.NewReader(body))
if err != nil {
panic(err)
}
resp, err := http.DefaultClient.Do(req)
resp, err := gohttp.DefaultClient.Do(req)
if err != nil {
panic(err)
}
@ -287,6 +287,6 @@ func MustDo(method, urlStr string, body string) *httpResponse {
// httpResponse is a wrapper for http.Response that holds the Body as a string.
type httpResponse struct {
*http.Response
*gohttp.Response
Body string
}

28
time.go
View file

@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) {
return q, nil
}
// ViewByTimeUnit returns the view name for time with a given quantum unit.
func ViewByTimeUnit(name string, t time.Time, unit rune) string {
// viewByTimeUnit returns the view name for time with a given quantum unit.
func viewByTimeUnit(name string, t time.Time, unit rune) string {
switch unit {
case 'Y':
return fmt.Sprintf("%s_%s", name, t.Format("2006"))
@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string {
}
}
// ViewsByTime returns a list of views for a given timestamp.
func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
// viewsByTime returns a list of views for a given timestamp.
func viewsByTime(name string, t time.Time, q TimeQuantum) []string {
a := make([]string, 0, len(q))
for _, unit := range q {
view := ViewByTimeUnit(name, t, unit)
view := viewByTimeUnit(name, t, unit)
if view == "" {
continue
}
@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string {
return a
}
// ViewsByTimeRange returns a list of views to traverse to query a time range.
func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
// viewsByTimeRange returns a list of views to traverse to query a time range.
func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string {
t := start
// Save flags for performance.
@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextDayGTE(t, end) {
break
} else if t.Hour() != 0 {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
continue
}
@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextMonthGTE(t, end) {
break
} else if t.Day() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
continue
}
@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
if !nextYearGTE(t, end) {
break
} else if t.Month() != 1 {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
continue
}
@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string
// Walk back down from largest units to smallest units.
for t.Before(end) {
if hasYear && nextYearGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'Y'))
results = append(results, viewByTimeUnit(name, t, 'Y'))
t = t.AddDate(1, 0, 0)
} else if hasMonth && nextMonthGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'M'))
results = append(results, viewByTimeUnit(name, t, 'M'))
t = t.AddDate(0, 1, 0)
} else if hasDay && nextDayGTE(t, end) {
results = append(results, ViewByTimeUnit(name, t, 'D'))
results = append(results, viewByTimeUnit(name, t, 'D'))
t = t.AddDate(0, 0, 1)
} else if hasHour {
results = append(results, ViewByTimeUnit(name, t, 'H'))
results = append(results, viewByTimeUnit(name, t, 'H'))
t = t.Add(time.Hour)
} else {
break

82
time_test.go → time_internal_test.go
View file

@ -12,28 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
package pilosa
import (
"reflect"
"testing"
"time"
"github.com/pilosa/pilosa"
)
// Ensure string can be parsed into time quantum.
func TestParseTimeQuantum(t *testing.T) {
t.Run("OK", func(t *testing.T) {
if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil {
if q, err := ParseTimeQuantum("YMDH"); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if q != pilosa.TimeQuantum("YMDH") {
} else if q != TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %#v", q)
}
})
t.Run("ErrInvalidTimeQuantum", func(t *testing.T) {
if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum {
if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum {
t.Fatalf("unexpected error: %s", err)
}
})
@ -44,125 +42,125 @@ func TestViewByTimeUnit(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("Y", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("M", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" {
if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("D", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" {
t.Fatalf("unexpected name: %s", s)
}
})
t.Run("H", func(t *testing.T) {
if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" {
t.Fatalf("unexpected name: %s", s)
}
})
}
// Ensure all applicable frame names can be generated when mutating a time bit.
// Ensure all applicable field names can be generated when mutating a time bit.
func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH"))
a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) {
t.Fatalf("unexpected names: %+v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D"))
a := viewsByTime("F", ts, mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000102"}) {
t.Fatalf("unexpected names: %+v", a)
}
})
}
// Ensure sets of frames can be returned for a given time range.
// Ensure sets of fields can be returned for a given time range.
func TestViewsByTimeRange(t *testing.T) {
t.Run("Y", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YM", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM"))
if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD"))
if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("M", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M"))
if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD"))
if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH"))
if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("DH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH"))
if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("H", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H"))
if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
}
// DefaultTimeLayout is the time layout used by the tests.
const DefaultTimeLayout = "2006-01-02 15:04"
// defaultTimeLayout is the time layout used by the tests.
const defaultTimeLayout = "2006-01-02 15:04"
// MustParseTime parses value using DefaultTimeLayout. Panic on error.
func MustParseTime(value string) time.Time {
v, err := time.Parse(DefaultTimeLayout, value)
// mustParseTime parses value using DefaultTimeLayout. Panic on error.
func mustParseTime(value string) time.Time {
v, err := time.Parse(defaultTimeLayout, value)
if err != nil {
panic(err)
}
return v
}
// MustParseTimeQuantum parses v into a time quantum. Panic on error.
func MustParseTimeQuantum(v string) pilosa.TimeQuantum {
q, err := pilosa.ParseTimeQuantum(v)
// mustParseTimeQuantum parses v into a time quantum. Panic on error.
func mustParseTimeQuantum(v string) TimeQuantum {
q, err := ParseTimeQuantum(v)
if err != nil {
panic(err)
}

1006
translate.go Normal file
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more