scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 23:11:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into slicecount-optimization

Browse source
This commit is contained in:
tgruben 2018-06-07 10:55:09 -05:00 • committed by GitHub
commit 813cd06bee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
84 changed files with 3171 additions and 5531 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.travis.yml
View file

@ -13,7 +13,7 @@ env:
- GOARCH=amd64
- GOARCH=amd64 ENTERPRISE=1
install:
- make install-dep install-statik vendor generate-statik
- make install-dep vendor
script:
- make test
before_deploy:
@ -27,9 +27,16 @@ deploy:
go: "1.10"
condition: $GOARCH = amd64
matrix:
# Excluding or allowing failures on non-primary matrix configurations due to long running times.
fast_finish: true
allow_failures:
- go: master
fast_finish: true
- go: 1.9
exclude:
- go: 1.9
env: GOARCH=386
- go: 1.9
env: GOARCH=amd64
notifications:
slack:
secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

8
CONTRIBUTING.md
View file

@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks:
make install
```
- Install build dependencies (dep, statik, and protoc):
- Install build dependencies (dep and protoc):
```sh
make install-build-deps
@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks:
make release
```
- Generate static assets for the WebUI:
```sh
make generate-statik
```
- Regenerate protocol buffer files in `internal/`:
```sh

2
Dockerfile
View file

@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder
COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
&& CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
&& CGO_ENABLED=0 make install-dep install FLAGS="-a"
FROM scratch

16
Gopkg.lock generated
View file

@ -1,12 +1,6 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"
[[projects]]
branch = "master"
name = "github.com/CAFxX/gcnotifier"
@ -212,14 +206,22 @@
[[projects]]
name = "github.com/shirou/gopsutil"
packages = [
"cpu",
"host",
"internal/common",
"mem",
"net",
"process"
]
revision = "bfe3c2e8f406bf352bc8df81f98c752224867349"
version = "v2.17.11"
[[projects]]
branch = "master"
name = "github.com/shirou/w32"
packages = ["."]
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
branch = "master"
name = "github.com/spf13/afero"
@ -302,6 +304,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba"
inputs-digest = "325d0fb217ec7f1509186ff947e184f6c8e65941f06000eb110180e65816b1a4"
solver-name = "gps-cdcl"
solver-version = 1

16
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@ -88,16 +88,12 @@ install: vendor
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
# `go generate` statik assets (WebUI)
generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa
# `go generate` all needed packages
generate: generate-protoc generate-statik generate-stringer
generate: generate-protoc generate-stringer
# Create Docker image from Dockerfile
docker:
@ -126,23 +122,17 @@ endef
require-dep:
$(call require,dep)
require-statik:
$(call require,statik)
require-protoc-gen-gofast:
$(call require,protoc-gen-gofast)
require-protoc:
$(call require,protoc)
install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer
install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer
install-dep:
go get -u github.com/golang/dep/cmd/dep
install-statik:
go get -u github.com/rakyll/statik
install-stringer:
go get -u golang.org/x/tools/cmd/stringer

231
api.go
View file

@ -217,9 +217,9 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
return nil
}
// CreateFrame makes the named frame in the named index with the given options.
func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) {
if err := api.validate(apiCreateFrame); err != nil {
// CreateField makes the named field in the named index with the given options.
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, options FieldOptions) (*Field, error) {
if err := api.validate(apiCreateField); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
@ -229,32 +229,32 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str
return nil, ErrIndexNotFound
}
// Create frame.
frame, err := index.CreateFrame(frameName, options)
// Create field.
field, err := index.CreateField(fieldName, options)
if err != nil {
return nil, errors.Wrap(err, "creating frame")
return nil, errors.Wrap(err, "creating field")
}
// Send the create frame message to all nodes.
// Send the create field message to all nodes.
err = api.Broadcaster.SendSync(
&internal.CreateFrameMessage{
&internal.CreateFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
Meta: options.Encode(),
})
if err != nil {
api.Logger.Printf("problem sending CreateFrame message: %s", err)
return nil, errors.Wrap(err, "sending CreateFrame message")
api.Logger.Printf("problem sending CreateField message: %s", err)
return nil, errors.Wrap(err, "sending CreateField message")
}
api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return frame, nil
api.Holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return field, nil
}
// DeleteFrame removes the named frame from the named index. If the index is not
// found, an error is returned. If the frame is not found, it is ignored and no
// DeleteField removes the named field from the named index. If the index is not
// found, an error is returned. If the field is not found, it is ignored and no
// action is taken.
func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error {
if err := api.validate(apiDeleteFrame); err != nil {
func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error {
if err := api.validate(apiDeleteField); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -264,28 +264,28 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str
return ErrIndexNotFound
}
// Delete frame from the index.
if err := index.DeleteFrame(frameName); err != nil {
return errors.Wrap(err, "deleting frame")
// Delete field from the index.
if err := index.DeleteField(fieldName); err != nil {
return errors.Wrap(err, "deleting field")
}
// Send the delete frame message to all nodes.
// Send the delete field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.DeleteFrameMessage{
&internal.DeleteFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
})
if err != nil {
api.Logger.Printf("problem sending DeleteFrame message: %s", err)
return errors.Wrap(err, "sending DeleteFrame message")
api.Logger.Printf("problem sending DeleteField message: %s", err)
return errors.Wrap(err, "sending DeleteField message")
}
api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
api.Holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return nil
}
// ExportCSV encodes the fragment designated by the index,frame,slice as
// ExportCSV encodes the fragment designated by the index,field,slice as
// CSV of the form <row>,<col>
func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error {
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error {
if err := api.validate(apiExportCSV); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -297,7 +297,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
}
// Find the fragment.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return ErrFragmentNotFound
}
@ -333,13 +333,13 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64)
// MarshalFragment returns an object which can write the specified fragment's data
// to an io.Writer. The serialized data can be read back into a fragment with
// the UnmarshalFragment API call.
func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) {
func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) {
if err := api.validate(apiMarshalFragment); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -349,15 +349,15 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName
// UnmarshalFragment creates a new fragment (if necessary) and reads data from a
// Reader which was previously written by MarshalFragment to populate the
// fragment's data.
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error {
func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error {
if err := api.validate(apiUnmarshalFragment); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
// Retrieve field.
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Retrieve view.
@ -366,7 +366,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa
return errors.Wrap(err, "creating view")
}
// Retrieve fragment from frame.
// Retrieve fragment from field.
frag, err := view.CreateFragmentIfNotExists(slice)
if err != nil {
return errors.Wrap(err, "creating fragment")
@ -397,7 +397,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice)
f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -415,13 +415,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte,
}
// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) {
func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) {
if err := api.validate(apiFragmentBlocks); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve fragment from holder.
f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice)
f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice)
if f == nil {
return nil, ErrFragmentNotFound
}
@ -482,101 +482,22 @@ func (api *API) LocalID() string {
return api.Cluster.Node.ID
}
// Schema returns information about each index in Pilosa including which frames
// Schema returns information about each index in Pilosa including which fields
// and views they contain.
func (api *API) Schema(ctx context.Context) []*IndexInfo {
return api.Holder.Schema()
}
// CreateField creates a new BSI field in the given index and frame.
func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error {
if err := api.validate(apiCreateField); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame by name.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Create new field.
if err := f.CreateField(field); err != nil {
return errors.Wrap(err, "creating field")
}
// Send the create field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.CreateFieldMessage{
Index: indexName,
Frame: frameName,
Field: encodeField(field),
})
if err != nil {
api.Logger.Printf("problem sending CreateField message: %s", err)
}
return errors.Wrap(err, "sending CreateField message")
}
// DeleteField deletes the given field.
func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error {
if err := api.validate(apiDeleteField); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame by name.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Delete field.
if err := f.DeleteField(fieldName); err != nil {
return errors.Wrap(err, "deleting field")
}
// Send the delete field message to all nodes.
err := api.Broadcaster.SendSync(
&internal.DeleteFieldMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
})
if err != nil {
api.Logger.Printf("problem sending DeleteField message: %s", err)
}
return errors.Wrap(err, "sending DeleteField message")
}
// Fields returns the fields in the given frame.
func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) {
if err := api.validate(apiFields); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
index := api.Holder.index(indexName)
if index == nil {
return nil, ErrIndexNotFound
}
frame := index.frame(frameName)
if frame == nil {
return nil, ErrFrameNotFound
}
return frame.GetFields()
}
// Views returns the views in the given frame.
func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) {
// Views returns the views in the given field.
func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*View, error) {
if err := api.validate(apiViews); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve views.
f := api.Holder.Frame(indexName, frameName)
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
// Fetch views.
@ -585,15 +506,15 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) (
}
// DeleteView removes the given view.
func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error {
func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error {
if err := api.validate(apiDeleteView); err != nil {
return errors.Wrap(err, "validating api method")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
// Retrieve field.
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Delete the view.
@ -608,7 +529,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri
err := api.Broadcaster.SendSync(
&internal.DeleteViewMessage{
Index: indexName,
Frame: frameName,
Field: fieldName,
View: viewName,
})
if err != nil {
@ -653,15 +574,15 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At
return attrs, nil
}
func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
if err := api.validate(apiFrameAttrDiff); err != nil {
func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) {
if err := api.validate(apiFieldAttrDiff); err != nil {
return nil, errors.Wrap(err, "validating api method")
}
// Retrieve index from holder.
f := api.Holder.Frame(indexName, frameName)
f := api.Holder.Field(indexName, fieldName)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
// Retrieve local blocks.
@ -687,15 +608,15 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s
return attrs, nil
}
// Import bulk imports data into a particular index,frame,slice.
// Import bulk imports data into a particular index,field,slice.
func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
if err := api.validate(apiImport); err != nil {
return errors.Wrap(err, "validating api method")
}
_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
if err != nil {
return errors.Wrap(err, "getting frame")
return errors.Wrap(err, "getting field")
}
// Convert timestamps to time.Time.
@ -709,9 +630,9 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
}
// Import into fragment.
err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps)
err = field.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -722,15 +643,15 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
return errors.Wrap(err, "validating api method")
}
_, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice)
_, field, err := api.indexField(req.Index, req.Field, req.Slice)
if err != nil {
return errors.Wrap(err, "getting frame")
return errors.Wrap(err, "getting field")
}
// Import into fragment.
err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values)
err = field.ImportValue(req.ColumnIDs, req.Values)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -758,7 +679,7 @@ func (api *API) LongQueryTime() time.Duration {
return api.Cluster.LongQueryTime
}
func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) {
func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) {
// Validate that this handler owns the slice.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName)
@ -766,20 +687,20 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I
}
// Find the Index.
api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice)
api.Logger.Printf("importing: %v %v %v", indexName, fieldName, slice)
index := api.Holder.Index(indexName)
if index == nil {
api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error())
api.Logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error())
return nil, nil, ErrIndexNotFound
}
// Retrieve frame.
frame := index.Frame(frameName)
if frame == nil {
api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error())
return nil, nil, ErrFrameNotFound
// Retrieve field.
field := index.Field(fieldName)
if field == nil {
api.Logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error())
return nil, nil, ErrFieldNotFound
}
return index, frame, nil
return index, field, nil
}
// SetCoordinator makes a new Node the cluster coordinator.
@ -870,17 +791,14 @@ type apiMethod int
const (
apiClusterMessage apiMethod = iota
apiCreateField
apiCreateFrame
apiCreateIndex
apiDeleteField
apiDeleteFrame
apiDeleteIndex
apiDeleteView
apiExportCSV
apiFields
apiFragmentBlockData
apiFragmentBlocks
apiFrameAttrDiff
apiFieldAttrDiff
//apiHosts // not implemented
apiImport
apiImportValue
@ -916,17 +834,14 @@ var methodsResizing = map[apiMethod]struct{}{
var methodsNormal = map[apiMethod]struct{}{
apiCreateField: struct{}{},
apiCreateFrame: struct{}{},
apiCreateIndex: struct{}{},
apiDeleteField: struct{}{},
apiDeleteFrame: struct{}{},
apiDeleteIndex: struct{}{},
apiDeleteView: struct{}{},
apiExportCSV: struct{}{},
apiFields: struct{}{},
apiFragmentBlockData: struct{}{},
apiFragmentBlocks: struct{}{},
apiFrameAttrDiff: struct{}{},
apiFieldAttrDiff: struct{}{},
apiImport: struct{}{},
apiImportValue: struct{}{},
apiIndex: struct{}{},

8
apimethod_string.go
View file

@ -2,15 +2,15 @@
package pilosa
import "strconv"
import "fmt"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews"
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381}
var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329}
func (i apiMethod) String() string {
if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
return fmt.Sprintf("apiMethod(%d)", i)
}
return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
}

53
attr.go
View file

@ -221,3 +221,56 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) {
}
return decodeAttrs(pb.GetAttrs()), nil
}
func newMemAttrStore() AttrStore {
return &memAttrStore{
store: make(map[uint64]map[string]interface{}),
}
}
// memAttrStore represents an in-memory implementation of the AttrStore interface.
type memAttrStore struct {
store map[uint64]map[string]interface{}
}
// Path is an in-memory implementation of AttrStore Path method.
func (s *memAttrStore) Path() string { return "" }
// Open is an in-memory implementation of AttrStore Open method.
func (s *memAttrStore) Open() error {
return nil
}
// Close is an in-memory implementation of AttrStore Close method.
func (s *memAttrStore) Close() error {
return nil
}
// Attrs returns a set of attributes by ID.
func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
return s.store[id], nil
}
// SetAttrs sets attribute values for a given ID.
func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
s.store[id] = m
return nil
}
// SetBulkAttrs sets attribute values for a set of ids.
func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
for id, v := range m {
s.store[id] = v
}
return nil
}
// Blocks is an in-memory implementation of AttrStore Blocks method.
func (s *memAttrStore) Blocks() ([]AttrBlock, error) {
return nil, nil
}
// BlockData is an in-memory implementation of AttrStore BlockData method.
func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
return nil, nil
}

30
broadcast.go
View file

@ -123,12 +123,10 @@ const (
MessageTypeCreateSlice = iota
MessageTypeCreateIndex
MessageTypeDeleteIndex
MessageTypeCreateFrame
MessageTypeDeleteFrame
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeCreateField
MessageTypeDeleteField
MessageTypeCreateView
MessageTypeDeleteView
MessageTypeClusterStatus
MessageTypeResizeInstruction
MessageTypeResizeInstructionComplete
@ -149,18 +147,14 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
typ = MessageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = MessageTypeDeleteIndex
case *internal.CreateFrameMessage:
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
typ = MessageTypeDeleteFrame
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.CreateFieldMessage:
typ = MessageTypeCreateField
case *internal.DeleteFieldMessage:
typ = MessageTypeDeleteField
case *internal.CreateViewMessage:
typ = MessageTypeCreateView
case *internal.DeleteViewMessage:
typ = MessageTypeDeleteView
case *internal.ClusterStatus:
typ = MessageTypeClusterStatus
case *internal.ResizeInstruction:
@ -199,18 +193,14 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
m = &internal.CreateIndexMessage{}
case MessageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
case MessageTypeCreateFrame:
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:
m = &internal.DeleteFrameMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeCreateField:
m = &internal.CreateFieldMessage{}
case MessageTypeDeleteField:
m = &internal.DeleteFieldMessage{}
case MessageTypeCreateView:
m = &internal.CreateViewMessage{}
case MessageTypeDeleteView:
m = &internal.DeleteViewMessage{}
case MessageTypeClusterStatus:
m = &internal.ClusterStatus{}
case MessageTypeResizeInstruction:

113
client.go
View file

@ -107,7 +107,7 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]ui
return rsp.Standard, nil
}
// Schema returns all index and frame schema information.
// Schema returns all index and field schema information.
func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Execute request against the host.
u := c.defaultURI.Path("/schema")
@ -269,14 +269,14 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri
}
// Import bulk imports bits for a single slice to a host.
func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
func (c *InternalHTTPClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if field == "" {
return ErrFieldRequired
}
buf, err := marshalImportPayload(index, frame, slice, bits)
buf, err := marshalImportPayload(index, field, slice, bits)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -298,14 +298,14 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
}
// ImportK bulk imports bits specified by string keys to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error {
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, field string, columns []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if field == "" {
return ErrFieldRequired
}
buf, err := marshalImportPayloadK(index, frame, columns)
buf, err := marshalImportPayloadK(index, field, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -330,16 +330,16 @@ func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, optio
return err
}
func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error {
err := c.CreateFrame(ctx, indexName, frameName, options)
if err == nil || err == ErrFrameExists {
func (c *InternalHTTPClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error {
err := c.CreateField(ctx, indexName, fieldName, options)
if err == nil || err == ErrFieldExists {
return nil
}
return err
}
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
func marshalImportPayload(index, field string, slice uint64, bits []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
@ -348,7 +348,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
Field: field,
Slice: slice,
RowIDs: rowIDs,
ColumnIDs: columnIDs,
@ -361,7 +361,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
}
// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) {
func marshalImportPayloadK(index, field string, bits []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowKeys := Bits(bits).RowKeys()
columnKeys := Bits(bits).ColumnKeys()
@ -370,7 +370,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) {
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
Field: field,
RowKeys: rowKeys,
ColumnKeys: columnKeys,
Timestamps: timestamps,
@ -420,14 +420,14 @@ func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []b
}
// ImportValue bulk imports field values for a single slice to a host.
func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error {
func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if field == "" {
return ErrFieldRequired
}
buf, err := marshalImportValuePayload(index, frame, field, slice, vals)
buf, err := marshalImportValuePayload(index, field, slice, vals)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -449,7 +449,7 @@ func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, fiel
}
// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice.
func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) {
func marshalImportValuePayload(index, field string, slice uint64, vals []FieldValue) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
@ -457,9 +457,8 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals []
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportValueRequest{
Index: index,
Frame: frame,
Slice: slice,
Field: field,
Slice: slice,
ColumnIDs: columnIDs,
Values: values,
})
@ -508,11 +507,11 @@ func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, bu
}
// ExportCSV bulk exports data for a single slice from a host to CSV format.
func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error {
func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if field == "" {
return ErrFieldRequired
}
// Retrieve a list of nodes that own the slice.
@ -526,7 +525,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string,
for _, i := range rand.Perm(len(nodes)) {
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil {
if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err)
continue
} else {
@ -538,12 +537,12 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string,
}
// exportNode copies a CSV export from a node to w.
func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error {
func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, field string, slice uint64, w io.Writer) error {
// Create URL.
u := nodePathToURL(node, "/export")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -575,18 +574,18 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
return nil
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) {
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
}
return c.backupSliceNode(ctx, index, frame, slice, node)
return c.backupSliceNode(ctx, index, field, slice, node)
}
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) {
func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *Node) (io.ReadCloser, error) {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -616,14 +615,14 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame s
return resp.Body, nil
}
// CreateFrame creates a new frame on the server.
func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
// CreateField creates a new field on the server.
func (c *InternalHTTPClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error {
if index == "" {
return ErrIndexRequired
}
// Encode query request.
buf, err := json.Marshal(&postFrameRequest{
buf, err := json.Marshal(&postFieldRequest{
Options: opt,
})
if err != nil {
@ -631,7 +630,7 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
}
// Create URL & HTTP request.
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame))
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field))
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
@ -659,7 +658,7 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
case http.StatusOK:
return nil // ok
case http.StatusConflict:
return ErrFrameExists
return ErrFieldExists
default:
return errors.New(string(body))
}
@ -667,11 +666,11 @@ func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame strin
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) {
func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error) {
u := uriPathToURL(c.defaultURI, "/fragment/blocks")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"field": {field},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
@ -708,10 +707,10 @@ func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame st
}
// BlockData returns row/column id pairs for a block.
func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) {
func (c *InternalHTTPClient) BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
Index: index,
Frame: frame,
Field: field,
Slice: slice,
Block: uint64(block),
})
@ -795,11 +794,11 @@ func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, b
}
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame))
func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field))
// Encode request.
buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks})
buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, errors.Wrap(err, "marshaling")
}
@ -823,13 +822,13 @@ func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame strin
switch resp.StatusCode {
case http.StatusOK: // ok
case http.StatusNotFound:
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
default:
return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
}
// Decode response object.
var rsp postFrameAttrDiffResponse
var rsp postFieldAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, errors.Wrap(err, "decoding")
}
@ -962,7 +961,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit {
}
// FieldValues represents the value for a column within a
// range-encoded frame.
// range-encoded field.
type FieldValue struct {
ColumnID uint64
Value int64
@ -1054,16 +1053,16 @@ type InternalClient interface {
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error
ImportK(ctx context.Context, index, frame string, bits []Bit) error
Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error
ImportK(ctx context.Context, index, field string, bits []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error)
EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error
ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, index, field string, slice uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, index, field string, slice uint64, block int) ([]uint64, []uint64, error)
ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
RowAttrDiff(ctx context.Context, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, pb proto.Message) error
}

21
client_test.go
View file

@ -145,7 +145,7 @@ func TestClient_MultiNode(t *testing.T) {
topN := 4
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN),
Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN),
Remote: false,
}
result, err := client[0].Query(context.Background(), "i", queryRequest)
@ -244,8 +244,9 @@ func TestClient_ImportValue(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
fld := pilosa.Field{
Name: "fld",
fldName := "f"
fo := pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
@ -253,7 +254,7 @@ func TestClient_ImportValue(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}})
field, err := index.CreateFieldIfNotExists(fldName, fo)
if err != nil {
t.Fatal(err)
}
@ -266,7 +267,7 @@ func TestClient_ImportValue(t *testing.T) {
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
{ColumnID: 3, Value: 40},
@ -275,7 +276,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Sum.
sum, cnt, err := frame.FieldSum(nil, fld.Name)
sum, cnt, err := field.Sum(nil, fldName)
if err != nil {
t.Fatal(err)
}
@ -284,7 +285,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Min.
min, cnt, err := frame.FieldMin(nil, fld.Name)
min, cnt, err := field.Min(nil, fldName)
if err != nil {
t.Fatal(err)
}
@ -293,11 +294,11 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Min with Filter.
filter, err := frame.FieldRange(fld.Name, pql.GT, 40)
filter, err := field.Range(fldName, pql.GT, 40)
if err != nil {
t.Fatal(err)
}
min, cnt, err = frame.FieldMin(filter, fld.Name)
min, cnt, err = field.Min(filter, fldName)
if err != nil {
t.Fatal(err)
}
@ -306,7 +307,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify Max.
max, cnt, err := frame.FieldMax(nil, fld.Name)
max, cnt, err := field.Max(nil, fldName)
if err != nil {
t.Fatal(err)
}

42
cluster.go
View file

@ -584,7 +584,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool {
// frag is a struct of basic fragment information.
type frag struct {
frame string
field string
view string
slice uint64
}
@ -617,36 +617,36 @@ func (a fragsByHost) add(b fragsByHost) fragsByHost {
return a
}
type viewsByFrame map[string][]string
type viewsByField map[string][]string
func (a viewsByFrame) addView(frame, view string) {
a[frame] = append(a[frame], view)
func (a viewsByField) addView(field, view string) {
a[field] = append(a[field], view)
}
func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
// frameViews is a map of frame to slice of views.
frameViews := make(viewsByFrame)
// fieldViews is a map of field to slice of views.
fieldViews := make(viewsByField)
for _, frame := range idx.Frames() {
for _, view := range frame.Views() {
frameViews.addView(frame.Name(), view.Name())
for _, field := range idx.Fields() {
for _, view := range field.Views() {
fieldViews.addView(field.Name(), view.Name())
}
}
return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews)
}
// fragCombos returns a map (by uri) of lists of fragments for a given index
// by creating every combination of frame/view specified in `frameViews` up to maxSlice.
func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost {
// by creating every combination of field/view specified in `fieldViews` up to maxSlice.
func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost {
t := make(fragsByHost)
for i := uint64(0); i <= maxSlice; i++ {
nodes := c.SliceNodes(idx, i)
for _, n := range nodes {
// for each frame/view combination:
for frame, views := range frameViews {
// for each field/view combination:
for field, views := range fieldViews {
for _, view := range views {
t[n.ID] = append(t[n.ID], frag{frame, view, i})
t[n.ID] = append(t[n.ID], frag{field, view, i})
}
}
}
@ -770,7 +770,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R
src := &internal.ResizeSource{
Node: EncodeNode(c.nodeByID(srcNodeID)),
Index: idx.Name(),
Frame: frag.frame,
Field: frag.field,
View: frag.view,
Slice: frag.slice,
}
@ -1239,10 +1239,10 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
srcURI := decodeURI(src.Node.URI)
// Retrieve frame.
f := c.Holder.Frame(src.Index, src.Frame)
// Retrieve field.
f := c.Holder.Field(src.Index, src.Field)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Create view.
@ -1259,7 +1259,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
// Stream slice from remote node.
c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI)
rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI)
rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI)
if err != nil {
// For now it is an acceptable error if the fragment is not found
// on the remote node. This occurs when a slice has been skipped and
@ -1275,7 +1275,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err
return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI)
}
// Write to local frame and always close reader.
// Write to local field and always close reader.
if err := func() error {
defer rd.Close()
_, err := frag.ReadFrom(rd)

18
cluster_internal_test.go
View file

@ -45,13 +45,13 @@ func TestFragCombos(t *testing.T) {
tests := []struct {
idx string
maxSlice uint64
frameViews viewsByFrame
fieldViews viewsByField
expected fragsByHost
}{
{
idx: "i",
maxSlice: uint64(2),
frameViews: viewsByFrame{"f": []string{"v1", "v2"}},
fieldViews: viewsByField{"f": []string{"v1", "v2"}},
expected: fragsByHost{
"node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}},
"node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}},
@ -60,7 +60,7 @@ func TestFragCombos(t *testing.T) {
{
idx: "foo",
maxSlice: uint64(3),
frameViews: viewsByFrame{"f": []string{"v0"}},
fieldViews: viewsByField{"f": []string{"v0"}},
expected: fragsByHost{
"node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}},
"node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}},
@ -69,7 +69,7 @@ func TestFragCombos(t *testing.T) {
}
for _, test := range tests {
actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews)
actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf("expected: %v, but got: %v", test.expected, actual)
}
@ -145,23 +145,23 @@ func TestFragSources(t *testing.T) {
c5.addNodeBasicSorted(node3)
idx := newIndexWithTempPath("i")
frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{})
field, err := idx.CreateFieldIfNotExists("f", FieldOptions{})
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 101, nil)
_, err = field.SetBit("standard", 1, 101, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 1300000, nil)
_, err = field.SetBit("standard", 1, 1300000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 2600000, nil)
_, err = field.SetBit("standard", 1, 2600000, nil)
if err != nil {
t.Fatal(err)
}
_, err = frame.SetBit("standard", 1, 3900000, nil)
_, err = field.SetBit("standard", 1, 3900000, nil)
if err != nil {
t.Fatal(err)
}

44
cluster_test.go
View file

@ -416,42 +416,19 @@ func TestCluster_ResizeStates(t *testing.T) {
}
// Add Bit Data to node0.
if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil {
if err := tc.CreateField("i", "f", FieldOptions{}); err != nil {
t.Fatal(err)
}
tc.SetBit("i", "f", "standard", 1, 101, nil)
tc.SetBit("i", "f", "standard", 1, 1300000, nil)
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
Fields: []*Field{
{
Name: "fld0",
Type: FieldTypeInt,
Min: -100,
Max: 100,
},
},
}); err != nil {
t.Fatal(err)
}
tc.SetFieldValue("i", "fields", 1, "fld0", -10)
tc.SetFieldValue("i", "fields", 1, "fld0", 10)
tc.SetFieldValue("i", "fields", 1300000, "fld0", -99)
tc.SetFieldValue("i", "fields", 1300000, "fld0", 99)
// Before starting the resize, get the CheckSum to use for
// comparison later.
node0Frame := node0.Holder.Frame("i", "f")
node0View := node0Frame.View("standard")
node0Field := node0.Holder.Field("i", "f")
node0View := node0Field.View("standard")
node0Fragment := node0View.Fragment(1)
node0Checksum := node0Fragment.Checksum()
node0Frame = node0.Holder.Frame("i", "fields")
node0View = node0Frame.View("field_fld0")
node0Fragment = node0View.Fragment(1)
node0ChecksumFld := node0Fragment.Checksum()
// AddNode needs to block until the resize process has completed.
tc.AddNode(false)
node1 := tc.Clusters[1]
@ -476,8 +453,8 @@ func TestCluster_ResizeStates(t *testing.T) {
// Bits
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")
node1Field := node1.Holder.Field("i", "f")
node1View := node1Field.View("standard")
node1Fragment := node1View.Fragment(1)
// Ensure checksums are the same.
@ -485,17 +462,6 @@ func TestCluster_ResizeStates(t *testing.T) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Values
// Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0.
node1Frame = node1.Holder.Frame("i", "fields")
node1View = node1Frame.View("field_fld0")
node1Fragment = node1View.Fragment(1)
// Ensure checksums are the same.
if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) {
t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)

57
cmd/bench.go
View file

@ -1,57 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
)
var Bencher *ctl.BenchCommand
func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr)
benchCmd := &cobra.Command{
Use: "bench",
Short: "Benchmark operations.",
Long: `
Executes a benchmark for a given operation against the index.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Bencher.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := benchCmd.Flags()
flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")
ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify)
return benchCmd
}
func init() {
subcommandFns["bench"] = NewBenchCommand
}

54
cmd/bench_test.go
View file

@ -1,54 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestBenchHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "bench", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa bench") || err != nil {
t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestBenchConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"bench", "--operation", "set-bit"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Bencher.Host, "localhost:12345")
v.Check(cmd.Bencher.Index, "myindex")
v.Check(cmd.Bencher.Frame, "f1")
v.Check(cmd.Bencher.Op, "set-bit")
v.Check(cmd.Bencher.N, 0)
return v.Error()
},
},
}
executeDry(t, tests)
}

2
cmd/export.go
View file

@ -52,7 +52,7 @@ The file does not contain any headers.
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)

4
cmd/export_test.go
View file

@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) {
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
field = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.Field, "f1")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()
},

11
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and frame. The data
Long: `Bulk imports one or more CSV files to a host's index and field. The data
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -54,15 +54,14 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags := importCmd.Flags()
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.")
flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.")
flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field")
flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)
return importCmd

4
cmd/import_test.go
View file

@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) {
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
field = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Importer.Host, "localhost:12345")
v.Check(cmd.Importer.Index, "myindex")
v.Check(cmd.Importer.Frame, "f1")
v.Check(cmd.Importer.Field, "f1")
return v.Error()
},
},

116
ctl/bench.go
View file

@ -1,116 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"fmt"
"io"
"math/rand"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BenchCommand represents a command for benchmarking index operations.
type BenchCommand struct {
// Destination host and port.
Host string
// Name of the index & frame to execute against.
Index string
Frame string
// Type of operation and number to execute.
Op string
N int
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewBenchCommand returns a new instance of BenchCommand.
func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand {
return &BenchCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the bench command.
func (cmd *BenchCommand) Run(ctx context.Context) error {
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
switch cmd.Op {
case "set-bit":
return cmd.runSetBit(ctx, client)
case "":
return errors.New("op required")
default:
return fmt.Errorf("unknown bench op: %q", cmd.Op)
}
}
// runSetBit executes a benchmark of random SetBit() operations.
func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error {
if cmd.N == 0 {
return errors.New("operation count required")
} else if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
}
const maxRowID = 1000
const maxColumnID = 100000
startTime := time.Now()
// Execute operation continuously.
for i := 0; i < cmd.N; i++ {
rowID := rand.Intn(maxRowID)
columnID := rand.Intn(maxColumnID)
queryRequest := &internal.QueryRequest{
Query: fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID),
Remote: false,
}
if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil {
return err
}
}
// Print results.
elapsed := time.Since(startTime)
fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds())
return nil
}
func (cmd *BenchCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

98
ctl/bench_test.go
View file

@ -1,98 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"os"
"testing"
"github.com/pilosa/pilosa"
"github.com/pkg/errors"
)
func TestBenchCommand_InvalidOption(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewBenchCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
if errors.Cause(err) != pilosa.ErrHostRequired {
t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err)
}
cm.Host = "localhost:10101"
err = cm.Run(context.Background())
if err.Error() != "op required" {
t.Fatalf("Expect err: %s, actual err: %s", "op required", err)
}
cm.Op = "test"
err = cm.Run(context.Background())
if err.Error() != "unknown bench op: \"test\"" {
t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err)
}
}
func TestBenchCommand_Run(t *testing.T) {
rder := []byte{}
stdin := bytes.NewReader(rder)
r, w, _ := os.Pipe()
cm := NewBenchCommand(stdin, w, w)
cm.Op = "set-bit"
cm.Host = "localhost:10101"
err := cm.Run(context.Background())
if err.Error() != "operation count required" {
t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err)
}
cm.N = 1
err = cm.Run(context.Background())
if err != pilosa.ErrIndexRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err)
}
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err)
}
cm.Frame = "f"
err = cm.Run(context.Background())
w.Close()
var buf bytes.Buffer
io.Copy(&buf, r)
fmt.Println(buf.String())
if err != nil {
fmt.Println(buf.String())
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

10
ctl/export.go
View file

@ -30,9 +30,9 @@ type ExportCommand struct {
// Remote host and port.
Host string
// Name of the index & frame to export from.
// Name of the index & field to export from.
Index string
Frame string
Field string
// Filename to export to.
Path string
@ -57,8 +57,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if cmd.Field == "" {
return pilosa.ErrFieldRequired
}
// Use output file, if specified.
@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

8
ctl/export_test.go
View file

@ -38,8 +38,8 @@ func TestExportCommand_Validation(t *testing.T) {
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
if err != pilosa.ErrFieldRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
}
}
@ -59,10 +59,10 @@ func TestExportCommand_Run(t *testing.T) {
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader("")))
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}

33
ctl/generate_config.go
View file

@ -18,9 +18,11 @@ import (
"context"
"fmt"
"io"
"strings"
"github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// GenerateConfigCommand represents a command for printing a default config.
@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera
// Run prints out the default config.
func (cmd *GenerateConfigCommand) Run(ctx context.Context) error {
fmt.Fprintln(cmd.Stdout, strings.TrimSpace(`
data-dir = "~/.pilosa"
bind = "localhost:10101"
max-writes-per-request = 5000
[cluster]
replicas = 1
hosts = [
"localhost:10101",
]
[anti-entropy]
interval = "10m0s"
[profile]
cpu = ""
cpu-time = "30s"
[metric]
service = "statsd"
host = "127.0.0.1:8125"
poll-interval = "0m15s"
`)+"\n")
conf := server.NewConfig()
ret, err := toml.Marshal(*conf)
if err != nil {
return errors.Wrap(err, "unmarshaling default config")
}
fmt.Fprintf(cmd.Stdout, "%s\n", ret)
return nil
}

2
ctl/generate_config_test.go
View file

@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) {
io.Copy(&buf, r)
if err != nil {
t.Fatalf("Config Run doesn't work: %s", err)
} else if !strings.Contains(buf.String(), "localhost:10101") {
} else if !strings.Contains(buf.String(), ":10101") {
t.Fatalf("Unexpected config: %s", buf.String())
}
}

71
ctl/import.go
View file

@ -35,20 +35,17 @@ type ImportCommand struct {
// Destination host and port.
Host string `json:"host"`
// Name of the index & frame to import into.
// Name of the index & field to import into.
Index string `json:"index"`
Frame string `json:"frame"`
Field string `json:"field"`
// Options for index & frame to be created if they don't exist
// Options for index & field to be created if they don't exist
IndexOptions pilosa.IndexOptions
FrameOptions pilosa.FrameOptions
FieldOptions pilosa.FieldOptions
// CreateSchema ensures the schema exists before import
CreateSchema bool
// For Range-Encoded fields, name of the Field to import into.
Field string `json:"field"`
// Indicates that the payload should be treated as string keys.
StringKeys bool `json:"StringKeys"`
@ -83,11 +80,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Validate arguments.
// Index and frame are validated early before the files are parsed.
// Index and field are validated early before the files are parsed.
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if cmd.Field == "" {
return pilosa.ErrFieldRequired
} else if len(cmd.Paths) == 0 {
return errors.New("path required")
}
@ -105,10 +102,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
}
}
// Determine the field type in order to correctly handle the input data.
fieldType := pilosa.DefaultFieldType
schema, err := cmd.Client.Schema(ctx)
if err != nil {
return errors.Wrap(err, "getting schema")
}
for _, index := range schema {
if index.Name == cmd.Index {
for _, field := range index.Fields {
if field.Name == cmd.Field {
fieldType = field.Options.Type
}
}
}
}
// Import each path and import by slice.
for _, path := range cmd.Paths {
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, path); err != nil {
if err := cmd.importPath(ctx, fieldType, path); err != nil {
return err
}
}
@ -121,18 +134,18 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
if err != nil {
return fmt.Errorf("Error Creating Index: %s", err)
}
err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions)
err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field, cmd.FieldOptions)
if err != nil {
return fmt.Errorf("Error Creating Frame: %s", err)
return fmt.Errorf("Error Creating Field: %s", err)
}
return nil
}
// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
// If a field is provided, treat the import data as values to be range-encoded.
if cmd.Field != "" {
return cmd.bufferFieldValues(ctx, path)
func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error {
// If fieldType is `int`, treat the import data as values to be range-encoded.
if fieldType == pilosa.FieldTypeInt {
return cmd.bufferValues(ctx, path)
} else {
if cmd.StringKeys {
return cmd.bufferBitsK(ctx, path)
@ -241,7 +254,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
}
logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil {
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}
@ -338,15 +351,15 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er
// TODO: does it help to sort the rowKeys?
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil {
return errors.Wrap(err, "importing keys")
}
return nil
}
// bufferFieldValues buffers slices of fieldValues to be imported as a batch.
func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error {
// bufferValues buffers slices of FieldValues to be imported as a batch.
func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error {
a := make([]pilosa.FieldValue, 0, cmd.BufferSize)
var r *csv.Reader
@ -394,7 +407,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
}
val.ColumnID = columnID
// Parse field value.
// Parse FieldValue.
value, err := strconv.ParseInt(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
@ -403,9 +416,9 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
a = append(a, val)
// If we've reached the buffer size then import field values.
// If we've reached the buffer size then import FieldValues.
if len(a) == cmd.BufferSize {
if err := cmd.importFieldValues(ctx, a); err != nil {
if err := cmd.importValues(ctx, a); err != nil {
return err
}
a = a[:0]
@ -413,29 +426,29 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
}
// If there are still values in the buffer then flush them.
if err := cmd.importFieldValues(ctx, a); err != nil {
if err := cmd.importValues(ctx, a); err != nil {
return err
}
return nil
}
// importFieldValues sends batches of fieldValues to the server.
func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error {
// importValues sends batches of FieldValues to the server.
func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Group vals by slice.
logger.Printf("grouping %d vals", len(vals))
valsBySlice := pilosa.FieldValues(vals).GroupBySlice()
// Parse path into field values.
// Parse path into FieldValues.
for slice, vals := range valsBySlice {
if cmd.Sort {
sort.Sort(pilosa.FieldValues(vals))
}
logger.Printf("importing slice: %d, n=%d", slice, len(vals))
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil {
if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil {
return errors.Wrap(err, "importing values")
}
}

39
ctl/import_test.go
View file

@ -15,6 +15,7 @@
package ctl
import (
"bufio"
"bytes"
"context"
"io"
@ -38,11 +39,11 @@ func TestImportCommand_Validation(t *testing.T) {
cm.Index = "i"
err = cm.Run(context.Background())
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
if err != pilosa.ErrFieldRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err)
}
cm.Frame = "f"
cm.Field = "f"
err = cm.Run(context.Background())
if err.Error() != "path required" {
t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err)
@ -72,7 +73,7 @@ func TestImportCommand_Run(t *testing.T) {
cm.Host = s.Host()
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
cm.CreateSchema = true
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
@ -81,8 +82,7 @@ func TestImportCommand_Run(t *testing.T) {
}
}
// Ensure that the ImportValue path runs (note: we have specified a value
// for cm.Field.)
// Ensure that the ImportValue path runs.
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
@ -106,11 +106,10 @@ func TestImportCommand_RunValue(t *testing.T) {
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
cm.Index = "i"
cm.Frame = "f"
cm.Field = "foo"
cm.Field = "f"
cm.Paths = []string{file.Name()}
err = cm.Run(ctx)
if err != nil {
@ -120,12 +119,21 @@ func TestImportCommand_RunValue(t *testing.T) {
func TestImportCommand_InvalidFile(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = "anyhost"
cm.Host = s.Host()
cm.Index = "i"
cm.Frame = "f"
cm.Field = "f"
file, err := ioutil.TempFile("", "import.csv")
file.Write([]byte("a,2\n3,5\n5,6"))
if err != nil {
@ -180,3 +188,12 @@ func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
}
return req
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

14
diagnostics.go
View file

@ -217,7 +217,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() {
// EnrichWithSchemaProperties adds schema info to the diagnostics payload.
func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
var numSlices uint64
numFrames := 0
numFields := 0
numIndexes := 0
bsiFieldCount := 0
timeQuantumEnabled := false
@ -225,19 +225,19 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() {
for _, index := range d.server.Holder.Indexes() {
numSlices += index.MaxSlice() + 1
numIndexes += 1
for _, frame := range index.Frames() {
numFrames += 1
if fields, err := frame.GetFields(); err == nil {
bsiFieldCount += len(fields)
for _, field := range index.Fields() {
numFields += 1
if field.Type() == FieldTypeInt {
bsiFieldCount += 1
}
if frame.TimeQuantum() != "" {
if field.TimeQuantum() != "" {
timeQuantumEnabled = true
}
}
}
d.Set("NumIndexes", numIndexes)
d.Set("NumFrames", numFrames)
d.Set("NumFields", numFields)
d.Set("NumSlices", numSlices)
d.Set("BSIFieldCount", bsiFieldCount)
d.Set("TimeQuantumEnabled", timeQuantumEnabled)

4
docs/getting-started.md
View file

@ -112,10 +112,6 @@ Note that both the user IDs and the repository IDs were remapped to sequential i
#### Make Some Queries
<div class="note">
<p>Note the Pilosa server comes with a <a href="../webui/">WebUI</a> for constructing queries in a browser. In local development, it is available at <a href="http://localhost:10101">localhost:10101</a>.</p>
</div>
Which repositories did user 14 star:
``` request
curl localhost:10101/index/repository/query \

11
docs/installation.md
View file

@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -144,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -173,7 +170,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -262,7 +258,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.
@ -298,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
git clone https://github.com/pilosa/pilosa.git
```
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
3. Build the Pilosa repo:
```
cd $GOPATH/src/github.com/pilosa/pilosa
make install-build-deps
make generate-statik
make install
```
@ -327,7 +321,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
export Export data from pilosa.

22
docs/webui.md
View file

@ -9,13 +9,27 @@ nav = [
## WebUI
The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101).
A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status.
This can be used for constructing queries and viewing the cluster status.
### Installation
Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac.
Installing on a Mac with Homebrew is simple; just run:
```
brew install pilosa-webui
```
You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running:
```
make install
```
### Console
The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown.
Each query's result will be displayed in the Output section along with the query time.
@ -39,4 +53,4 @@ Frame creation also supports options like `timeQuantum`. When creating a new fra
### Cluster Admin
Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.
Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames.

390
executor.go
View file

@ -26,9 +26,9 @@ import (
"github.com/pkg/errors"
)
// DefaultFrame is the frame used if one is not specified.
// DefaultField is the field used if one is not specified.
const (
DefaultFrame = "general"
DefaultField = "general"
// MinThreshold is the lowest count to use in a Top-N operation when
// looking for additional id/count pairs.
@ -127,10 +127,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeSum(ctx, index, c, slices, opt)
case "Min":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMin(ctx, index, c, slices, opt)
return e.executeMin(ctx, index, c, slices, opt)
case "Max":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeFieldMax(ctx, index, c, slices, opt)
return e.executeMax(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -138,8 +138,8 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return e.executeCount(ctx, index, c, slices, opt)
case "SetBit":
return e.executeSetBit(ctx, index, c, opt)
case "SetFieldValue":
return nil, e.executeSetFieldValue(ctx, index, c, opt)
case "SetValue":
return nil, e.executeSetValue(ctx, index, c, opt)
case "SetRowAttrs":
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
@ -174,9 +174,7 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
// executeSum executes a Sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Sum(): frame required")
} else if field := c.Args["field"]; field == "" {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Sum(): field required")
}
@ -207,11 +205,9 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl
return other, nil
}
// executeFieldMin executes a Min() call.
func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Min(): frame required")
} else if field := c.Args["field"]; field == "" {
// executeMin executes a Min() call.
func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Min(): field required")
}
@ -221,7 +217,7 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMinSlice(ctx, index, c, slice)
return e.executeMinSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
@ -242,11 +238,9 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal
return other, nil
}
// executeFieldMax executes a Max() call.
func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if frame := c.Args["frame"]; frame == "" {
return ValCount{}, errors.New("Max(): frame required")
} else if field := c.Args["field"]; field == "" {
// executeMax executes a Max() call.
func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) {
if field := c.Args["field"]; field == "" {
return ValCount{}, errors.New("Max(): field required")
}
@ -256,7 +250,7 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeFieldMaxSlice(ctx, index, c, slice)
return e.executeMaxSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
@ -318,8 +312,8 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
} else if err != nil {
return nil, err
} else {
frame, _ := c.Args["frame"].(string)
if fr := idx.Frame(frame); fr != nil {
field, _ := c.Args["field"].(string)
if fr := idx.Field(field); fr != nil {
rowID, _, err := c.UintArg(rowLabel)
if err != nil {
return nil, errors.Wrap(err, "getting row")
@ -362,7 +356,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
}
}
// executeSumCountSlice calculates the sum and count for fields on a slice.
// executeSumCountSlice calculates the sum and count for bsiGroups on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
@ -373,36 +367,35 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq
filter = row
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
field := e.Holder.Field(index, fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth())
vsum, vcount, err := fragment.Sum(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, errors.Wrap(err, "computing sum")
}
return ValCount{
Val: int64(vsum) + (int64(vcount) * field.Min),
Val: int64(vsum) + (int64(vcount) * bsig.Min),
Count: int64(vcount),
}, nil
}
// executeFieldMinSlice calculates the min for fields on a slice.
func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
// executeMinSlice calculates the min for bsiGroups on a slice.
func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
@ -412,36 +405,35 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq
filter = row
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
field := e.Holder.Field(index, fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth())
fmin, fcount, err := fragment.Min(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmin) + field.Min,
Val: int64(fmin) + bsig.Min,
Count: int64(fcount),
}, nil
}
// executeFieldMaxSlice calculates the max for fields on a slice.
func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
// executeMaxSlice calculates the max for bsiGroups on a slice.
func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) {
var filter *Row
if len(c.Children) == 1 {
row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
@ -451,30 +443,29 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq
filter = row
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ValCount{}, nil
}
field := frame.Field(fieldName)
field := e.Holder.Field(index, fieldName)
if field == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return ValCount{}, nil
}
fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if fragment == nil {
return ValCount{}, nil
}
fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth())
fmax, fcount, err := fragment.Max(filter, bsig.BitDepth())
if err != nil {
return ValCount{}, err
}
return ValCount{
Val: int64(fmax) + field.Min,
Val: int64(fmax) + bsig.Min,
Count: int64(fcount),
}, nil
}
@ -547,12 +538,12 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
field, _ := c.Args["field"].(string)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
}
field, _ := c.Args["field"].(string)
attrName, _ := c.Args["attrName"].(string)
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -561,7 +552,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
}
filters, _ := c.Args["filters"].([]interface{})
attrValues, _ := c.Args["attrValues"].([]interface{})
tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -579,12 +570,12 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
return nil, errors.New("TopN() can only have one input bitmap")
}
// Set default frame.
if frame == "" {
frame = DefaultFrame
// Set default field.
if field == "" {
field = DefaultField
}
f := e.Holder.Fragment(index, frame, ViewStandard, slice)
f := e.Holder.Fragment(index, field, ViewStandard, slice)
if f == nil {
return nil, nil
}
@ -600,8 +591,8 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
N: int(n),
Src: src,
RowIDs: rowIDs,
FilterField: field,
FilterValues: filters,
FilterName: attrName,
FilterValues: attrValues,
MinThreshold: minThreshold,
TanimotoThreshold: tanimotoThreshold,
})
@ -636,14 +627,14 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, ErrIndexNotFound
}
// Fetch frame & row label based on argument.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
// Fetch field & row label based on argument.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
}
f := e.Holder.Frame(index, frame)
f := e.Holder.Field(index, field)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
rowID, rowOK, rowErr := c.UintArg(rowLabel)
@ -654,7 +645,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
}
frag := e.Holder.Fragment(index, frame, ViewStandard, slice)
frag := e.Holder.Fragment(index, field, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
@ -685,15 +676,15 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// Handle field ranges differently.
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
return e.executeFieldRangeSlice(ctx, index, c, slice)
return e.executeBSIGroupRangeSlice(ctx, index, c, slice)
}
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
// Parse field, use default if unset.
field, _ := c.Args["field"].(string)
if field == "" {
field = DefaultField
}
// Retrieve column label.
@ -702,10 +693,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return nil, ErrIndexNotFound
}
// Retrieve base frame.
f := idx.Frame(frame)
// Retrieve base field.
f := idx.Field(field)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
// Read row & column id.
@ -743,10 +734,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return &Row{}, nil
}
// Union bitmaps across all time-based subframes.
// Union bitmaps across all time-based views.
row := &Row{}
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, frame, view, slice)
f := e.Holder.Fragment(index, field, view, slice)
if f == nil {
continue
}
@ -756,33 +747,19 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
return row, nil
}
// executeFieldRangeSlice executes a range(field) call for a local slice.
func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
// Remove frame field.
args := pql.CopyArgs(c.Args)
delete(args, "frame")
// Only one conditional field should remain.
if len(args) == 0 {
// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice.
func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) {
// Only one conditional should be present.
if len(c.Args) == 0 {
return nil, errors.New("Range(): condition required")
} else if len(args) > 1 {
} else if len(c.Args) > 1 {
return nil, errors.New("Range(): too many arguments")
}
// Extract condition field.
// Extract conditional.
var fieldName string
var cond *pql.Condition
for k, v := range args {
for k, v := range c.Args {
vv, ok := v.(*pql.Condition)
if !ok {
return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v)
@ -790,28 +767,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
fieldName, cond = k, vv
}
// EQ null (not implemented: flip frag.FieldNotNull with max ColumnID)
// NEQ null frag.FieldNotNull()
// BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween()
// BETWEEN a,b(out) BETWEEN/frag.FieldNotNull()
// EQ <int> frag.FieldRange
// NEQ <int> frag.FieldRange
f := e.Holder.Field(index, fieldName)
if f == nil {
return nil, ErrFieldNotFound
}
// EQ null (not implemented: flip frag.NotNull with max ColumnID)
// NEQ null frag.NotNull()
// BETWEEN a,b(in) BETWEEN/frag.RangeBetween()
// BETWEEN a,b(out) BETWEEN/frag.NotNull()
// EQ <int> frag.RangeOp
// NEQ <int> frag.RangeOp
// Handle `!= null`.
if cond.Op == pql.NEQ && cond.Value == nil {
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
// Find bsiGroup.
bsig := f.bsiGroup(fieldName)
if bsig == nil {
return nil, ErrBSIGroupNotFound
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if frag == nil {
return NewRow(), nil
}
return frag.FieldNotNull(field.BitDepth())
return frag.NotNull(bsig.BitDepth())
} else if cond.Op == pql.BETWEEN {
@ -826,33 +808,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
}
// The reason we don't just call:
// return f.FieldRangeBetween(fieldName, predicates[0], predicates[1])
// return f.RangeBetween(fieldName, predicates[0], predicates[1])
// here is because we need the call to be slice-specific.
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
// Find bsiGroup.
bsig := f.bsiGroup(fieldName)
if bsig == nil {
return nil, ErrBSIGroupNotFound
}
baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1])
baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1])
if outOfRange {
return NewRow(), nil
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if frag == nil {
return NewRow(), nil
}
// If the query is asking for the entire valid range, just return
// the not-null bitmap for the field.
if predicates[0] <= field.Min && predicates[1] >= field.Max {
return frag.FieldNotNull(field.BitDepth())
// the not-null bitmap for the bsiGroup.
if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max {
return frag.NotNull(bsig.BitDepth())
}
return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax)
return frag.RangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax)
} else {
@ -862,36 +844,36 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *
return nil, errors.New("Range(): conditions only support integer values")
}
// Find field.
field := f.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
// Find bsiGroup.
bsig := f.bsiGroup(fieldName)
if bsig == nil {
return nil, ErrBSIGroupNotFound
}
baseValue, outOfRange := field.BaseValue(cond.Op, value)
baseValue, outOfRange := bsig.baseValue(cond.Op, value)
if outOfRange && cond.Op != pql.NEQ {
return NewRow(), nil
}
// Retrieve fragment.
frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice)
frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice)
if frag == nil {
return NewRow(), nil
}
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range.
if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) ||
(cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) {
return frag.FieldNotNull(field.BitDepth())
// LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range.
if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) ||
(cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) {
return frag.NotNull(bsig.BitDepth())
}
// outOfRange for NEQ should return all not-null.
if outOfRange && cond.Op == pql.NEQ {
return frag.FieldNotNull(field.BitDepth())
return frag.NotNull(bsig.BitDepth())
}
f.Stats.Count("range:field", 1, 1.0)
return frag.FieldRange(cond.Op, field.BitDepth(), baseValue)
f.Stats.Count("range:bsigroup", 1, 1.0)
return frag.RangeOp(cond.Op, bsig.BitDepth(), baseValue)
}
}
@ -967,19 +949,19 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
frame, ok := c.Args["frame"].(string)
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("ClearBit() frame required")
return false, errors.New("ClearBit() field required")
}
// Retrieve frame.
// Retrieve field.
idx := e.Holder.Index(index)
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Frame(frame)
f := idx.Field(field)
if f == nil {
return false, ErrFrameNotFound
return false, ErrFieldNotFound
}
// Read fields using labels.
@ -1001,7 +983,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
}
// executeClearBitView executes a ClearBit() call for a single view.
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(index, slice) {
@ -1032,19 +1014,19 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
frame, ok := c.Args["frame"].(string)
field, ok := c.Args["field"].(string)
if !ok {
return false, errors.New("SetBit() field required: frame")
return false, errors.New("SetBit() field required: field")
}
// Retrieve frame.
// Retrieve field.
idx := e.Holder.Index(index)
if idx == nil {
return false, ErrIndexNotFound
}
f := idx.Frame(frame)
f := idx.Field(field)
if f == nil {
return false, ErrFrameNotFound
return false, ErrFieldNotFound
}
// Read fields using labels.
@ -1076,7 +1058,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
}
// executeSetBitView executes a SetBit() call for a specific view.
func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
@ -1107,46 +1089,40 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C
return ret, nil
}
// executeSetFieldValue executes a SetFieldValue() call.
func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
frameName, ok := c.Args["frame"].(string)
if !ok {
return errors.New("SetFieldValue() frame required")
}
// Retrieve frame.
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ErrFrameNotFound
}
// executeSetValue executes a SetValue() call.
func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
// Parse labels.
columnID, ok, err := c.UintArg(columnLabel)
if err != nil {
return fmt.Errorf("reading SetFieldValue() column: %v", err)
return fmt.Errorf("reading SetValue() column: %v", err)
} else if !ok {
return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel)
return fmt.Errorf("SetValue() column field '%v' required", columnLabel)
}
// Copy args and remove reserved fields.
args := pql.CopyArgs(c.Args)
delete(args, "frame")
// While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion.
// Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility.
// While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion.
// Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility.
delete(args, columnLabel)
// Set values.
for name, value := range args {
// Retrieve field.
field := e.Holder.Field(index, name)
if field == nil {
return ErrFieldNotFound
}
switch value := value.(type) {
case int64:
if _, err := frame.SetFieldValue(columnID, name, value); err != nil {
if _, err := field.SetValue(columnID, value); err != nil {
return err
}
default:
return ErrInvalidFieldValueType
return ErrInvalidBSIGroupValueType
}
field.Stats.Count("SetValue", 1, 1.0)
}
frame.Stats.Count("SetFieldValue", 1, 1.0)
// Do not forward call if this is already being forwarded.
if opt.Remote {
@ -1175,15 +1151,15 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
frameName, ok := c.Args["frame"].(string)
fieldName, ok := c.Args["field"].(string)
if !ok {
return errors.New("SetRowAttrs() frame required")
return errors.New("SetRowAttrs() field required")
}
// Retrieve frame.
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ErrFrameNotFound
// Retrieve field.
field := e.Holder.Field(index, fieldName)
if field == nil {
return ErrFieldNotFound
}
// Parse labels.
@ -1196,14 +1172,14 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "field")
delete(attrs, rowLabel)
// Set attributes.
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
frame.Stats.Count("SetRowAttrs", 1, 1.0)
field.Stats.Count("SetRowAttrs", 1, 1.0)
// Do not forward call if this is already being forwarded.
if opt.Remote {
@ -1232,18 +1208,18 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
// Collect attributes by frame/id.
// Collect attributes by field/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
frame, ok := c.Args["frame"].(string)
field, ok := c.Args["field"].(string)
if !ok {
return nil, errors.New("SetRowAttrs() frame required")
return nil, errors.New("SetRowAttrs() field required")
}
// Retrieve frame.
f := e.Holder.Frame(index, frame)
// Retrieve field.
f := e.Holder.Field(index, field)
if f == nil {
return nil, ErrFrameNotFound
return nil, ErrFieldNotFound
}
rowID, ok, err := c.UintArg(rowLabel)
@ -1255,20 +1231,20 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "field")
delete(attrs, rowLabel)
// Create frame group, if not exists.
frameMap := m[frame]
if frameMap == nil {
frameMap = make(map[uint64]map[string]interface{})
m[frame] = frameMap
// Create field group, if not exists.
fieldMap := m[field]
if fieldMap == nil {
fieldMap = make(map[uint64]map[string]interface{})
m[field] = fieldMap
}
// Set or merge attributes.
attr := frameMap[rowID]
attr := fieldMap[rowID]
if attr == nil {
frameMap[rowID] = cloneAttrs(attrs)
fieldMap[rowID] = cloneAttrs(attrs)
} else {
for k, v := range attrs {
attr[k] = v
@ -1276,19 +1252,19 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
}
}
// Bulk insert attributes by frame.
for name, frameMap := range m {
// Retrieve frame.
frame := e.Holder.Frame(index, name)
if frame == nil {
return nil, ErrFrameNotFound
// Bulk insert attributes by field.
for name, fieldMap := range m {
// Retrieve field.
field := e.Holder.Field(index, name)
if field == nil {
return nil, ErrFieldNotFound
}
// Set attributes.
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
if err := field.RowAttrStore().SetBulkAttrs(fieldMap); err != nil {
return nil, err
}
frame.Stats.Count("SetRowAttrs", 1, 1.0)
field.Stats.Count("SetRowAttrs", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
@ -1333,7 +1309,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, columnLabel)
delete(attrs, "frame")
delete(attrs, "field")
// Set attributes.
if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil {

432
executor_test.go
View file

@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
f, err := index.CreateField("f", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
@ -42,9 +42,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
@ -61,7 +61,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
}
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -91,9 +91,9 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
@ -225,7 +225,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -243,7 +243,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -254,7 +254,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -263,35 +263,34 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
}
}
// Ensure a SetFieldValue() query can be executed.
func TestExecutor_Execute_SetFieldValue(t *testing.T) {
// Ensure a SetValue() query can be executed.
func TestExecutor_Execute_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frames.
// Create felds.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2},
},
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 50,
}); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set field values.
// Set bsiGroup values.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil {
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil {
t.Fatal(err)
}
f := hldr.Frame("i", "f")
if value, exists, err := f.FieldValue(10, "field0"); err != nil {
f := hldr.Field("i", "f")
if value, exists, err := f.Value(10); err != nil {
t.Fatal(err)
} else if !exists {
t.Fatal("expected value to exist")
@ -299,15 +298,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
t.Fatalf("unexpected value: %v", value)
}
if value, exists, err := f.FieldValue(10, "field1"); err != nil {
t.Fatal(err)
} else if !exists {
t.Fatal("expected value to exist")
} else if value != 2 {
t.Fatalf("unexpected value: %v", value)
}
if value, exists, err := f.FieldValue(100, "field0"); err != nil {
if value, exists, err := f.Value(100); err != nil {
t.Fatal(err)
} else if !exists {
t.Fatal("expected value to exist")
@ -320,38 +311,31 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100},
},
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100,
}); err != nil {
t.Fatal(err)
}
t.Run("ErrFrameRequired", func(t *testing.T) {
t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldRequired", func(t *testing.T) {
t.Run("ErrColumnBSIGroupValue", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrColumnFieldValue", func(t *testing.T) {
t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrInvalidFieldValueType", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType {
t.Fatalf("unexpected error: %s", err)
}
})
@ -363,31 +347,31 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frames.
// Create fields.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
} else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
// Set two attrs on f/10.
// Also set attrs on other bitmaps and fields to test isolation.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
f := hldr.Frame("i", "f")
f := hldr.Field("i", "f")
if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
@ -404,20 +388,20 @@ func TestExecutor_Execute_TopN(t *testing.T) {
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=1)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(frame=f, row=10, col=0)
SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(frame=other, row=0, col=0)
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=1)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetBit(field=f, row=10, col=0)
SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`)
SetBit(field=other, row=0, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -427,7 +411,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
t.Run("Standard", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: 0, Count: 5},
@ -452,7 +436,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
@ -486,7 +470,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -521,7 +505,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
// Execute query.
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -541,11 +525,11 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -564,11 +548,11 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -588,29 +572,33 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -10,
Max: 100,
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=3)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(frame=f, row=1, col=1)
SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=3)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(field=x, row=1, col=1)
SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=0)
SetFieldValue(frame=f, foo=-5, col=1)
SetFieldValue(frame=f, foo=-5, col=2)
SetFieldValue(frame=f, foo=10, col=3)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(f=20, col=0)
SetValue(f=-5, col=1)
SetValue(f=-5, col=2)
SetValue(f=10, col=3)
SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`)
`), nil, nil); err != nil {
t.Fatal(err)
}
@ -622,16 +610,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: -5, cnt: 2},
{filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
{filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Min(frame=f, field=foo)`
pql = `Min(field=f)`
} else {
pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter)
pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
@ -648,16 +636,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
cnt int64
}{
{filter: ``, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1},
{filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1},
{filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1},
{filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1},
}
for i, tt := range tests {
var pql string
if tt.filter == "" {
pql = `Max(frame=f, field=foo)`
pql = `Max(field=f)`
} else {
pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter)
pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter)
}
if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil {
t.Fatal(err)
@ -679,39 +667,51 @@ func TestExecutor_Execute_Sum(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("foo", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 100,
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
if _, err := idx.CreateField("bar", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100000,
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("other", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(field=x, row=0, col=0)
SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, col=0)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
SetValue(foo=20, col=0)
SetValue(bar=2000, col=0)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -719,7 +719,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -728,7 +728,7 @@ func TestExecutor_Execute_Sum(t *testing.T) {
}
// Ensure a range query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -736,8 +736,9 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
}); err != nil {
t.Fatal(err)
@ -745,32 +746,31 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00")
SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00")
SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00")
SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00")
SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00")
SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00")
SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00")
SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00")
SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00")
SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00")
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
}
// Ensure a Range(field) query can be executed.
func TestExecutor_Execute_FieldRange(t *testing.T) {
// Ensure a Range(bsiGroup) query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
@ -780,49 +780,61 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("foo", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 100,
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
if _, err := idx.CreateField("bar", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 100000,
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100},
},
if _, err := idx.CreateField("other", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateField("edge", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 100,
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetBit(field=f, row=0, col=0)
SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, col=50)
SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, col=0)
SetFieldValue(frame=edge, foo=100, col=0)
SetFieldValue(frame=edge, foo=-100, col=1)
SetValue(foo=20, col=50)
SetValue(bar=2000, col=50)
SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`)
SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`)
SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`)
SetValue(other=1000, col=0)
SetValue(edge=100, col=0)
SetValue(edge=-100, col=1)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -831,19 +843,19 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("NEQ", func(t *testing.T) {
// NEQ null
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != null)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != -20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -852,7 +864,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -860,7 +872,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -868,7 +880,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -876,7 +888,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -884,16 +896,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
// Ensure that the FieldNotNull code path gets run.
t.Run("FieldNotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil {
// Ensure that the NotNull code path gets run.
t.Run("NotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -901,7 +913,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 0)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -909,7 +921,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("AboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
@ -917,7 +929,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("LTAboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge < 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
@ -925,21 +937,15 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
})
t.Run("GTBelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge > -200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
t.Run("ErrFrameNotFound", func(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound {
t.Fatal(err)
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound {
t.Fatal(err)
}
})
@ -963,7 +969,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(frame="f", row=10)` {
} else if query.String() != `Bitmap(field="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
@ -986,7 +992,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
@ -1021,7 +1027,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -1049,7 +1055,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, frame="f", row=10)` {
} else if query.String() != `SetBit(col=2, field="f", row=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1061,13 +1067,13 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
// Create field.
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1101,7 +1107,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` {
} else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -1113,15 +1119,15 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
// Create frame.
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
// Create field.
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := f.SetTimeQuantum("Y"); err != nil {
t.Fatal(err)
}
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -1162,11 +1168,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// slices and a second time to get the counts for a set of bitmaps.
switch remoteExecN {
case 0:
if query.String() != `TopN(frame="f", n=3)` {
if query.String() != `TopN(field="f", n=3)` {
t.Fatalf("unexpected query(0): %s", query.String())
}
case 1:
if query.String() != `TopN(frame="f", ids=[0,10,30], n=3)` {
if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` {
t.Fatalf("unexpected query(1): %s", query.String())
}
default:
@ -1190,7 +1196,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -1212,23 +1218,23 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
}
}
// Ensure SetColumnAttrs doesn't save `frame` as an attribute
func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
// Ensure SetColumnAttrs doesn't save `field` as an attribute
func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index.CreateFrame("f", pilosa.FrameOptions{})
index.CreateField("f", pilosa.FieldOptions{})
targetAttrs := map[string]interface{}{
"foo": "bar",
}
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// SetColumnAttrs call should exclude the frame attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil)
// SetColumnAttrs call should exclude the field attribute
_, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil)
if err != nil {
t.Fatal(err)
}
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil)
_, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1240,8 +1246,8 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
t.Fatalf("%#v != %#v", targetAttrs, attrs)
}
// SetColumnAttrs call should not break if frame is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil)
// SetColumnAttrs call should not break if field is not specified
_, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil)
if err != nil {
t.Fatal(err)
}

950
frame.go → field.go
View file

File diff suppressed because it is too large Load diff

149
field_internal_test.go Normal file
View file

@ -0,0 +1,149 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"reflect"
"testing"
"github.com/pilosa/pilosa/pql"
)
// Ensure a bsiGroup can adjust to its baseValue.
func TestBSIGroup_BaseValue(t *testing.T) {
b0 := &bsiGroup{
Name: "b0",
Type: bsiGroupTypeInt,
Min: -100,
Max: 900,
}
b1 := &bsiGroup{
Name: "b1",
Type: bsiGroupTypeInt,
Min: 0,
Max: 1000,
}
b2 := &bsiGroup{
Name: "b2",
Type: bsiGroupTypeInt,
Min: 100,
Max: 1100,
}
t.Run("Normal Condition", func(t *testing.T) {
for _, tt := range []struct {
f *bsiGroup
op pql.Token
val int64
expBaseValue uint64
expOutOfRange bool
}{
// LT
{b0, pql.LT, 5, 105, false},
{b0, pql.LT, -8, 92, false},
{b0, pql.LT, -108, 0, true},
{b0, pql.LT, 1005, 1000, false},
{b0, pql.LT, 0, 100, false},
{b1, pql.LT, 5, 5, false},
{b1, pql.LT, -8, 0, true},
{b1, pql.LT, 1005, 1000, false},
{b1, pql.LT, 0, 0, false},
{b2, pql.LT, 5, 0, true},
{b2, pql.LT, -8, 0, true},
{b2, pql.LT, 105, 5, false},
{b2, pql.LT, 1105, 1000, false},
// GT
{b0, pql.GT, -105, 0, false},
{b0, pql.GT, 5, 105, false},
{b0, pql.GT, 905, 0, true},
{b0, pql.GT, 0, 100, false},
{b1, pql.GT, 5, 5, false},
{b1, pql.GT, -8, 0, false},
{b1, pql.GT, 1005, 0, true},
{b1, pql.GT, 0, 0, false},
{b2, pql.GT, 5, 0, false},
{b2, pql.GT, -8, 0, false},
{b2, pql.GT, 105, 5, false},
{b2, pql.GT, 1105, 0, true},
// EQ
{b0, pql.EQ, -105, 0, true},
{b0, pql.EQ, 5, 105, false},
{b0, pql.EQ, 905, 0, true},
{b0, pql.EQ, 0, 100, false},
{b1, pql.EQ, 5, 5, false},
{b1, pql.EQ, -8, 0, true},
{b1, pql.EQ, 1005, 0, true},
{b1, pql.EQ, 0, 0, false},
{b2, pql.EQ, 5, 0, true},
{b2, pql.EQ, -8, 0, true},
{b2, pql.EQ, 105, 5, false},
{b2, pql.EQ, 1105, 0, true},
} {
bv, oor := tt.f.baseValue(tt.op, tt.val)
if oor != tt.expOutOfRange {
t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
}
}
})
t.Run("Betwween Condition", func(t *testing.T) {
for _, tt := range []struct {
f *bsiGroup
predMin int64
predMax int64
expBaseValueMin uint64
expBaseValueMax uint64
expOutOfRange bool
}{
{b0, -205, -105, 0, 0, true},
{b0, -105, 80, 0, 180, false},
{b0, 5, 20, 105, 120, false},
{b0, 20, 1005, 120, 1000, false},
{b0, 1005, 2000, 0, 0, true},
{b1, -105, -5, 0, 0, true},
{b1, -5, 20, 0, 20, false},
{b1, 5, 20, 5, 20, false},
{b1, 20, 1005, 20, 1000, false},
{b1, 1005, 2000, 0, 0, true},
{b2, 5, 95, 0, 0, true},
{b2, 95, 120, 0, 20, false},
{b2, 105, 120, 5, 20, false},
{b2, 120, 1105, 20, 1000, false},
{b2, 1105, 2000, 0, 0, true},
} {
min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax)
if oor != tt.expOutOfRange {
t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
}
}
})
}

287
field_test.go Normal file
View file

@ -0,0 +1,287 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure field can open and retrieve a view.
func TestField_CreateViewIfNotExists(t *testing.T) {
f := test.MustOpenField()
defer f.Close()
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
// Ensure field can set its time quantum.
func TestField_SetTimeQuantum(t *testing.T) {
fo := pilosa.FieldOptions{
Type: "time",
}
f := test.MustOpenField(pilosa.OptFieldFieldOptions(fo))
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value on field.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100); err != nil {
t.Fatal(err)
} else if value != 21 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
// Setting value should return no change.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
}
})
t.Run("Overwrite", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if changed, err := f.SetValue(100, 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set different value.
if changed, err := f.SetValue(100, 23); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.Value(100); err != nil {
t.Fatal(err)
} else if value != 23 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
})
t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeSet,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 20,
Max: 30,
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestField_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
field, err := pilosa.NewField(path, "i", ".meta")
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
}
// Ensure that field name validation is consistent.
func TestField_NameValidation(t *testing.T) {
validFieldNames := []string{
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
}
invalidFieldNames := []string{
"",
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"camelCase",
"UPPERCASE",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
for _, name := range validFieldNames {
_, err := pilosa.NewField(path, "i", name)
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
_, err := pilosa.NewField(path, "i", name)
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}
}
}
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := test.MustOpenField()
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in field")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}

92
fragment.go
View file

@ -66,13 +66,13 @@ const (
DefaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a frame and slice in an index.
// Fragment represents the intersection of a field and slice in an index.
type Fragment struct {
mu sync.RWMutex
// Composite identifiers
index string
frame string
field string
view string
slice uint64
@ -84,7 +84,7 @@ type Fragment struct {
opN int // number of ops since snapshot
// Cache for row counts.
CacheType string // passed in by frame
CacheType string // passed in by field
cache Cache
CacheSize uint32
@ -106,18 +106,18 @@ type Fragment struct {
Logger Logger
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
// This is set by the parent field unless overridden for testing.
RowAttrStore AttrStore
stats StatsClient
}
// NewFragment returns a new instance of Fragment.
func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
func NewFragment(path, index, field, view string, slice uint64) *Fragment {
return &Fragment{
path: path,
index: index,
frame: frame,
field: field,
view: view,
slice: slice,
CacheType: DefaultCacheType,
@ -139,8 +139,8 @@ func (f *Fragment) CachePath() string { return f.path + CacheExt }
// Index returns the index that the fragment was initialized with.
func (f *Fragment) Index() string { return f.index }
// Frame returns the frame the fragment was initialized with.
func (f *Fragment) Frame() string { return f.frame }
// Field returns the field the fragment was initialized with.
func (f *Fragment) Field() string { return f.field }
// View returns the view the fragment was initialized with.
func (f *Fragment) View() string { return f.view }
@ -247,7 +247,7 @@ func (f *Fragment) openStorage() error {
// openCache initializes the cache from row ids persisted to disk.
func (f *Fragment) openCache() error {
// Determine cache type from frame name.
// Determine cache type from field name.
switch f.CacheType {
case CacheTypeRanked:
f.cache = NewRankCache(f.CacheSize)
@ -486,8 +486,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
return f.storage.Contains(pos), nil
}
// FieldValue uses a column of bits to read a multi-bit value.
func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// Value uses a column of bits to read a multi-bit value.
func (f *Fragment) Value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -510,8 +510,8 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi
return value, true, nil
}
// SetFieldValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// SetValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
@ -541,8 +541,8 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
return changed, nil
}
// importSetFieldValue is a more efficient SetFieldValue just for imports.
func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// importSetValue is a more efficient SetValue just for imports.
func (f *Fragment) importSetValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
for i := uint(0); i < bitDepth; i++ {
if value&(1<<i) != 0 {
@ -582,9 +582,9 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
return changed, nil
}
// FieldSum returns the sum of a given field as well as the number of columns involved.
// Sum returns the sum of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
func (f *Fragment) Sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence row.
row := f.Row(uint64(bitDepth))
if filter != nil {
@ -614,9 +614,9 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err
return sum, count, nil
}
// FieldMin returns the min of a given field as well as the number of columns involved.
// Min returns the min of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) {
func (f *Fragment) Min(filter *Row, bitDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
@ -647,9 +647,9 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
return min, count, nil
}
// FieldMax returns the max of a given field as well as the number of columns involved.
// Max returns the max of a given bsiGroup as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) {
func (f *Fragment) Max(filter *Row, bitDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
if filter != nil {
@ -678,23 +678,23 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
return max, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
// RangeOp returns bitmaps with a bsiGroup value encoding matching the predicate.
func (f *Fragment) RangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
switch op {
case pql.EQ:
return f.fieldRangeEQ(bitDepth, predicate)
return f.rangeEQ(bitDepth, predicate)
case pql.NEQ:
return f.fieldRangeNEQ(bitDepth, predicate)
return f.rangeNEQ(bitDepth, predicate)
case pql.LT, pql.LTE:
return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
return f.rangeLT(bitDepth, predicate, op == pql.LTE)
case pql.GT, pql.GTE:
return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE)
return f.rangeGT(bitDepth, predicate, op == pql.GTE)
default:
return nil, ErrInvalidRangeOperation
}
}
func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
@ -713,12 +713,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
return b, nil
}
func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
// Get the equal bitmap.
eq, err := f.fieldRangeEQ(bitDepth, predicate)
eq, err := f.rangeEQ(bitDepth, predicate)
if err != nil {
return nil, err
}
@ -729,7 +729,7 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error)
return b, nil
}
func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
keep := NewRow()
// Start with set of columns with values set.
@ -777,7 +777,7 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.Row(uint64(bitDepth))
keep := NewRow()
@ -812,13 +812,13 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
// FieldNotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) {
// NotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) NotNull(bitDepth uint) (*Row, error) {
return f.Row(uint64(bitDepth)), nil
}
// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
// RangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) RangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.Row(uint64(bitDepth))
keep1 := NewRow() // GTE
keep2 := NewRow() // LTE
@ -900,7 +900,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Create a fast lookup of filter values.
var filters map[interface{}]struct{}
if opt.FilterField != "" && len(opt.FilterValues) > 0 {
if opt.FilterName != "" && len(opt.FilterValues) > 0 {
filters = make(map[interface{}]struct{})
for _, v := range opt.FilterValues {
filters[v] = struct{}{}
@ -948,7 +948,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
return nil, errors.Wrap(err, "getting attrs")
} else if attr == nil {
continue
} else if attrValue := attr[opt.FilterField]; attrValue == nil {
} else if attrValue := attr[opt.FilterName]; attrValue == nil {
continue
} else if _, ok := filters[attrValue]; !ok {
continue
@ -1074,7 +1074,7 @@ type TopOptions struct {
MinThreshold uint64
// Filter field name & values.
FilterField string
FilterName string
FilterValues []interface{}
TanimotoThreshold uint64
}
@ -1408,7 +1408,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error
for i := range columnIDs {
columnID, value := columnIDs[i], values[i]
_, err := f.importSetFieldValue(columnID, bitDepth, value)
_, err := f.importSetValue(columnID, bitDepth, value)
if err != nil {
return errors.Wrap(err, "setting")
}
@ -1452,8 +1452,8 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) {
}
func (f *Fragment) snapshot() error {
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice)
start := time.Now()
defer track(start, completeMessage, f.stats, f.Logger)
@ -1783,7 +1783,7 @@ func (s *FragmentSyncer) SyncFragment() error {
// Retrieve remote blocks.
client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient)
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice())
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Field(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return errors.Wrap(err, "getting blocks")
}
@ -1862,7 +1862,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
clients = append(clients, client)
// Only sync the standard block.
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id)
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Field(), f.Slice(), id)
if err != nil {
return errors.Wrap(err, "getting block")
}
@ -1904,11 +1904,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Only sync the standard block.
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.Field(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
count++
}
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.Field(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
count++
}

487
fragment_test.go → fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

423
frame_test.go
View file

@ -1,423 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa_test
import (
"io/ioutil"
"reflect"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
// Ensure frame can open and retrieve a view.
func TestFrame_CreateViewIfNotExists(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
// Create view.
view, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
// Retrieve existing view.
view2, err := f.CreateViewIfNotExists("v")
if err != nil {
t.Fatal(err)
} else if view != view2 {
t.Fatal("view mismatch")
}
if view != f.View("v") {
t.Fatal("view mismatch")
}
}
// Ensure frame can set its time quantum.
func TestFrame_SetTimeQuantum(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
// Set & retrieve time quantum.
if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload frame and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Ensure a frame can set & read a field value.
func TestFrame_SetFieldValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25},
},
})
if err != nil {
t.Fatal(err)
}
// Set value on first field.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set value on same column but different field.
if changed, err := f.SetFieldValue(100, "field1", 25); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.FieldValue(100, "field0"); err != nil {
t.Fatal(err)
} else if value != 21 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
// Setting value should return no change.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if changed {
t.Fatal("expected no change")
}
})
t.Run("Overwrite", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if changed, err := f.SetFieldValue(100, "field0", 21); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Set different value.
if changed, err := f.SetFieldValue(100, "field0", 23); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.FieldValue(100, "field0"); err != nil {
t.Fatal(err)
} else if value != 23 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected value to exist")
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldValueTooLow", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFieldValueTooHigh", func(t *testing.T) {
idx := test.MustOpenIndex()
defer idx.Close()
f, err := idx.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30},
},
})
if err != nil {
t.Fatal(err)
}
// Set value.
if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh {
t.Fatalf("unexpected error: %s", err)
}
})
}
func TestFrame_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", ".meta")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
}
// Ensure that frame name validation is consistent.
func TestFrame_NameValidation(t *testing.T) {
validFrameNames := []string{
"foo",
"hyphen-ated",
"under_score",
"abc123",
"trailing_",
}
invalidFrameNames := []string{
"",
"123abc",
"x.y",
"_foo",
"-bar",
"abc def",
"camelCase",
"UPPERCASE",
"a12345678901234567890123456789012345678901234567890123456789012345",
}
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
for _, name := range validFrameNames {
_, err := pilosa.NewFrame(path, "i", name)
if err != nil {
t.Fatalf("unexpected frame name: %s %s", name, err)
}
}
for _, name := range invalidFrameNames {
_, err := pilosa.NewFrame(path, "i", name)
if err == nil {
t.Fatalf("expected error on frame name: %s", name)
}
}
}
// Ensure frame can open and retrieve a view.
func TestFrame_DeleteView(t *testing.T) {
f := test.MustOpenFrame()
defer f.Close()
viewName := pilosa.ViewStandard + "_v"
// Create view.
view, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == nil {
t.Fatal("expected view")
}
err = f.DeleteView(viewName)
if err != nil {
t.Fatal(err)
}
if f.View(viewName) != nil {
t.Fatal("view still exists in frame")
}
// Recreate view with same name, verify that the old view was not reused.
view2, err := f.CreateViewIfNotExists(viewName)
if err != nil {
t.Fatal(err)
} else if view == view2 {
t.Fatal("failed to create new view")
}
}
// Ensure a field can adjust to its baseValue.
func TestField_BaseValue(t *testing.T) {
f0 := &pilosa.Field{
Name: "f0",
Type: pilosa.FieldTypeInt,
Min: -100,
Max: 900,
}
f1 := &pilosa.Field{
Name: "f1",
Type: pilosa.FieldTypeInt,
Min: 0,
Max: 1000,
}
f2 := &pilosa.Field{
Name: "f2",
Type: pilosa.FieldTypeInt,
Min: 100,
Max: 1100,
}
t.Run("Normal Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
op pql.Token
val int64
expBaseValue uint64
expOutOfRange bool
}{
// LT
{f0, pql.LT, 5, 105, false},
{f0, pql.LT, -8, 92, false},
{f0, pql.LT, -108, 0, true},
{f0, pql.LT, 1005, 1000, false},
{f0, pql.LT, 0, 100, false},
{f1, pql.LT, 5, 5, false},
{f1, pql.LT, -8, 0, true},
{f1, pql.LT, 1005, 1000, false},
{f1, pql.LT, 0, 0, false},
{f2, pql.LT, 5, 0, true},
{f2, pql.LT, -8, 0, true},
{f2, pql.LT, 105, 5, false},
{f2, pql.LT, 1105, 1000, false},
// GT
{f0, pql.GT, -105, 0, false},
{f0, pql.GT, 5, 105, false},
{f0, pql.GT, 905, 0, true},
{f0, pql.GT, 0, 100, false},
{f1, pql.GT, 5, 5, false},
{f1, pql.GT, -8, 0, false},
{f1, pql.GT, 1005, 0, true},
{f1, pql.GT, 0, 0, false},
{f2, pql.GT, 5, 0, false},
{f2, pql.GT, -8, 0, false},
{f2, pql.GT, 105, 5, false},
{f2, pql.GT, 1105, 0, true},
// EQ
{f0, pql.EQ, -105, 0, true},
{f0, pql.EQ, 5, 105, false},
{f0, pql.EQ, 905, 0, true},
{f0, pql.EQ, 0, 100, false},
{f1, pql.EQ, 5, 5, false},
{f1, pql.EQ, -8, 0, true},
{f1, pql.EQ, 1005, 0, true},
{f1, pql.EQ, 0, 0, false},
{f2, pql.EQ, 5, 0, true},
{f2, pql.EQ, -8, 0, true},
{f2, pql.EQ, 105, 5, false},
{f2, pql.EQ, 1105, 0, true},
} {
bv, oor := tt.f.BaseValue(tt.op, tt.val)
if oor != tt.expOutOfRange {
t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(bv, tt.expBaseValue) {
t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv)
}
}
})
t.Run("Betwween Condition", func(t *testing.T) {
for _, tt := range []struct {
f *pilosa.Field
predMin int64
predMax int64
expBaseValueMin uint64
expBaseValueMax uint64
expOutOfRange bool
}{
{f0, -205, -105, 0, 0, true},
{f0, -105, 80, 0, 180, false},
{f0, 5, 20, 105, 120, false},
{f0, 20, 1005, 120, 1000, false},
{f0, 1005, 2000, 0, 0, true},
{f1, -105, -5, 0, 0, true},
{f1, -5, 20, 0, 20, false},
{f1, 5, 20, 5, 20, false},
{f1, 20, 1005, 20, 1000, false},
{f1, 1005, 2000, 0, 0, true},
{f2, 5, 95, 0, 0, true},
{f2, 95, 120, 0, 20, false},
{f2, 105, 120, 5, 20, false},
{f2, 120, 1105, 20, 1000, false},
{f2, 1105, 2000, 0, 0, true},
} {
min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax)
if oor != tt.expOutOfRange {
t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor)
} else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) {
t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max)
}
}
})
}

209
handler.go
View file

@ -42,8 +42,6 @@ import (
type Handler struct {
Handler http.Handler
FileSystem FileSystem
Logger Logger
// Keeps the query argument validators for each handler
@ -62,7 +60,7 @@ var externalPrefixFlag = map[string]bool{
"import": true,
"export": true,
"index": true,
"frame": true,
"field": true,
"nodes": true,
"version": true,
}
@ -87,8 +85,7 @@ func OptHandlerAllowedOrigins(origins []string) HandlerOption {
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler(opts ...HandlerOption) (*Handler, error) {
handler := &Handler{
FileSystem: NopFileSystem,
Logger: NopLogger,
Logger: NopLogger,
}
handler.Handler = NewRouter(handler)
handler.populateValidators()
@ -108,10 +105,10 @@ func (h *Handler) populateValidators() {
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice")
h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice")
h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice")
}
func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
@ -137,8 +134,7 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
// NewRouter creates a new mux http router.
func NewRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
router.HandleFunc("/", handler.handleWebUI).Methods("GET")
router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET")
router.HandleFunc("/", handler.handleHome).Methods("GET")
router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
@ -165,13 +161,10 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
//router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
@ -229,19 +222,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) {
// If user is using curl, don't chuck HTML at them
if strings.HasPrefix(r.UserAgent(), "curl") {
http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound)
return
}
filesystem, err := h.FileSystem.New()
if err != nil {
_ = h.writeQueryResponse(w, r, &QueryResponse{Err: err})
h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.")
return
}
http.FileServer(filesystem).ServeHTTP(w, r)
func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound)
}
// handleGetSchema handles GET /schema requests.
@ -505,27 +487,27 @@ type postIndexAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// handlePostFrame handles POST /frame request.
func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
// handlePostField handles POST /field request.
func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
fieldName := mux.Vars(r)["field"]
// Decode request.
var req postFrameRequest
var req postFieldRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err == io.EOF {
// If no data was provided (EOF), we still create the frame
// If no data was provided (EOF), we still create the field
// with default values.
} else if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options)
_, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options)
if err != nil {
switch errors.Cause(err) {
case ErrIndexNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case ErrFrameExists:
case ErrFieldExists:
http.Error(w, err.Error(), http.StatusConflict)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -533,30 +515,30 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil {
if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type _postFrameRequest postFrameRequest
type _postFieldRequest postFieldRequest
// Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions,
// adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed
func (p *postFrameRequest) UnmarshalJSON(b []byte) error {
// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions,
// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed
func (p *postFieldRequest) UnmarshalJSON(b []byte) error {
// m is an overflow map used to capture additional, unexpected keys.
m := make(map[string]interface{})
if err := json.Unmarshal(b, &m); err != nil {
return errors.Wrap(err, "unmarshaling unexpected keys")
}
validFrameOptions := getValidOptions(FrameOptions{})
err := validateOptions(m, validFrameOptions)
validFieldOptions := getValidOptions(FieldOptions{})
err := validateOptions(m, validFieldOptions)
if err != nil {
return err
}
// Unmarshal expected values.
var _p _postFrameRequest
var _p _postFieldRequest
if err := json.Unmarshal(b, &_p); err != nil {
return errors.Wrap(err, "unmarshalling expected keys")
}
@ -577,18 +559,18 @@ func getValidOptions(option interface{}) []string {
return validOptions
}
type postFrameRequest struct {
Options FrameOptions `json:"options"`
type postFieldRequest struct {
Options FieldOptions `json:"options"`
}
type postFrameResponse struct{}
type postFieldResponse struct{}
// handleDeleteFrame handles DELETE /frame request.
func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
// handleDeleteField handles DELETE /field request.
func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
fieldName := mux.Vars(r)["field"]
err := h.API.DeleteFrame(r.Context(), indexName, frameName)
err := h.API.DeleteField(r.Context(), indexName, fieldName)
if err != nil {
if errors.Cause(err) == ErrIndexNotFound {
if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
@ -601,119 +583,26 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil {
if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type deleteFrameResponse struct{}
type deleteFieldResponse struct{}
// handlePostFrameField handles POST /frame/field request.
func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) {
// handlePostFieldAttrDiff handles POST /field/attr/diff requests.
func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
fieldName := mux.Vars(r)["field"]
// Decode request.
var req postFrameFieldRequest
var req postFieldAttrDiffRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
field := &Field{
Name: fieldName,
Type: req.Type,
Min: req.Min,
Max: req.Max,
}
if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type postFrameFieldRequest struct {
Type string `json:"type,omitempty"`
Min int64 `json:"min,omitempty"`
Max int64 `json:"max,omitempty"`
}
type postFrameFieldResponse struct{}
// handleDeleteFrameField handles DELETE /frame/field request.
func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
fieldName := mux.Vars(r)["field"]
if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
fields, err := h.API.Fields(r.Context(), indexName, frameName)
if err != nil {
switch errors.Cause(err) {
case ErrIndexNotFound:
fallthrough
case ErrFrameNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type getFrameFieldsResponse struct {
Fields []*Field `json:"fields,omitempty"`
}
type deleteFrameFieldResponse struct{}
// handlePostFrameAttrDiff handles POST /frame/attr/diff requests.
func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Decode request.
var req postFrameAttrDiffRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks)
attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks)
if err != nil {
switch errors.Cause(err) {
case ErrFragmentNotFound:
@ -725,18 +614,18 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{
if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.Logger.Printf("response encoding error: %s", err)
}
}
type postFrameAttrDiffRequest struct {
type postFieldAttrDiffRequest struct {
Blocks []AttrBlock `json:"blocks"`
}
type postFrameAttrDiffResponse struct {
type postFieldAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
@ -845,7 +734,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
switch errors.Cause(err) {
case ErrIndexNotFound:
fallthrough
case ErrFrameNotFound:
case ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case ErrClusterDoesNotOwnSlice:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -898,7 +787,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
switch errors.Cause(err) {
case ErrIndexNotFound:
fallthrough
case ErrFrameNotFound:
case ErrFieldNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
case ErrClusterDoesNotOwnSlice:
http.Error(w, err.Error(), http.StatusPreconditionFailed)
@ -935,7 +824,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
// Parse query parameters.
q := r.URL.Query()
index, frame := q.Get("index"), q.Get("frame")
index, field := q.Get("index"), q.Get("field")
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
@ -943,7 +832,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
return
}
if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil {
if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil {
switch errors.Cause(err) {
case ErrFragmentNotFound:
break
@ -1011,7 +900,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
return
}
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice)
blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice)
if err != nil {
if errors.Cause(err) == ErrFragmentNotFound {
http.Error(w, err.Error(), http.StatusNotFound)

12
handler_internal_test.go
View file

@ -55,23 +55,23 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
}
}
// Test custom UnmarshalJSON for postFrameRequest object
func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
// Test custom UnmarshalJSON for postFieldRequest object
func TestPostFieldRequestUnmarshalJSON(t *testing.T) {
tests := []struct {
json string
expected postFrameRequest
expected postFieldRequest
err string
}{
{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
{json: `{"options": {}}`, expected: postFieldRequest{Options: FieldOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
{json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: FieldOptions{CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {
actual := &postFrameRequest{}
actual := &postFieldRequest{}
err := json.Unmarshal([]byte(test.json), actual)
if err != nil {
if test.err == "" || test.err != err.Error() {

311
handler_test.go
View file

@ -17,7 +17,6 @@ package pilosa_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@ -32,7 +31,6 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/test"
)
@ -84,17 +82,17 @@ func TestHandler_Schema(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -105,8 +103,8 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -121,17 +119,17 @@ func TestHandler_Status(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -537,7 +535,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
@ -561,7 +559,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -612,7 +610,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
@ -695,12 +693,12 @@ func TestHandler_Index_Delete(t *testing.T) {
}
}
// Ensure handler can delete a frame.
func TestHandler_DeleteFrame(t *testing.T) {
// Ensure handler can delete a field.
func TestHandler_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -708,13 +706,13 @@ func TestHandler_DeleteFrame(t *testing.T) {
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Frame("f1"); f != nil {
t.Fatal("expected nil frame")
} else if f := hldr.Index("i0").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
@ -767,8 +765,8 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) {
}
}
// Ensure the handler can return data in differing blocks for a frame.
func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Ensure the handler can return data in differing blocks for a field.
func TestHandler_Field_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -778,7 +776,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Set attributes on the index.
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{})
f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
@ -802,7 +800,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Send block checksums to determine diff.
resp, err := http.Post(
s.URL+"/index/i/frame/meta/attr/diff",
s.URL+"/index/i/field/meta/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
@ -817,245 +815,6 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
}
}
// Ensure the handler can create a new field on an existing frame.
func TestHandler_Frame_AddField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":100,"max":200}`),
)
if err != nil {
t.Fatal(err)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
if field := f.Field("x"); !reflect.DeepEqual(field, &pilosa.Field{Name: "x", Type: "int", Min: 100, Max: 200}) {
t.Fatalf("unexpected field: %#v", field)
}
})
t.Run("ErrInvalidFieldType", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"bad_type","min":100,"max":200}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field type`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":200,"max":100}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field range`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
t.Run("ErrFieldAlreadyExists", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}},
}); err != nil {
t.Fatal(err)
}
resp, err := http.Post(
s.URL+"/index/i/frame/f/field/x",
"application/json",
strings.NewReader(`{"type":"int","min":0,"max":100}`),
)
if err != nil {
t.Fatal(err)
} else if body := MustReadAll(resp.Body); string(body) != `creating field: field already exists`+"\n" {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
}
// Ensure the handler can delete existing fields.
func TestHandler_Frame_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/x", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
if field := f.Field("x"); field != nil {
t.Fatalf("expected nil field, got: %#v", field)
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/y", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) != `deleting field: field not found` {
t.Fatalf("unexpected body: %q", body)
} else if err := resp.Body.Close(); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
})
}
func TestHandler_Frame_GetFields(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
t.Run("OK", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil {
t.Fatal(err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f/fields")
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
var fields FrameFields
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if err = json.Unmarshal([]byte(body), &fields); err != nil {
t.Fatal(err)
}
field := fields.Fields[0]
if field.Name != "x" {
t.Fatalf("expected field's name: x, actuall name: %v", field.Name)
} else if field.Min != 1 {
t.Fatalf("expected field's min: x, actuall min: %v", field.Min)
} else if field.Max != 100 {
t.Fatalf("expected field's max: x, actuall max: %v", field.Max)
}
})
t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) {
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
_, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{})
if err != nil {
t.Fatalf("creating frame: %v", err)
}
resp, err := http.Get(s.URL + "/index/i/frame/f1/fields")
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
} else if body, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if strings.TrimSpace(string(body)) == `frame fields not allowed` {
t.Fatalf("shouldn't get frame fields not allowed error: %q", body)
}
})
}
type FrameFields struct {
Fields []pilosa.Field
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()
@ -1155,34 +914,6 @@ func TestHandler_RecalculateCaches(t *testing.T) {
}
func TestHandler_WebUI(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.FileSystem = &statik.FileSystem{}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
if !strings.Contains(w.Body.String(), "<title>Pilosa WebUI</title>") {
t.Fatalf("WebUI is not being served correctly.")
}
// If curl is the client, the response should be different
w = httptest.NewRecorder()
req := test.MustNewHTTPRequest("GET", "/", nil)
req.Header.Add("User-Agent", "curl/7.54.0")
h.ServeHTTP(w, req)
if !strings.Contains(w.Body.String(), "try the WebUI") {
t.Fatalf("WebUI is not being served correctly.")
}
}
func TestHandler_CORS(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()

88
holder.go
View file

@ -209,20 +209,20 @@ func (h *Holder) MaxSlices() map[string]uint64 {
return a
}
// Schema returns schema information for all indexes, frames, and views.
// Schema returns schema information for all indexes, fields, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()}
for _, view := range frame.Views() {
for _, field := range index.Fields() {
fi := &FieldInfo{Name: field.Name(), Options: field.Options()}
for _, view := range field.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
di.Fields = append(di.Fields, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
sort.Sort(fieldInfoSlice(di.Fields))
a = append(a, di)
}
sort.Sort(indexInfoSlice(a))
@ -238,16 +238,16 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
if err != nil {
return errors.Wrap(err, "creating index")
}
// Create frames that don't exist.
for _, f := range index.Frames {
opt := decodeFrameOptions(f.Meta)
frame, err := idx.CreateFrameIfNotExists(f.Name, *opt)
// Create fields that don't exist.
for _, f := range index.Fields {
opt := decodeFieldOptions(f.Meta)
field, err := idx.CreateFieldIfNotExists(f.Name, *opt)
if err != nil {
return errors.Wrap(err, "creating frame")
return errors.Wrap(err, "creating field")
}
// Create views that don't exist.
for _, v := range f.Views {
_, err := frame.CreateViewIfNotExists(v)
_, err := field.CreateViewIfNotExists(v)
if err != nil {
return errors.Wrap(err, "creating view")
}
@ -390,27 +390,27 @@ func (h *Holder) DeleteIndex(name string) error {
return nil
}
// Frame returns the frame for an index and name.
func (h *Holder) Frame(index, name string) *Frame {
// Field returns the field for an index and name.
func (h *Holder) Field(index, name string) *Field {
idx := h.Index(index)
if idx == nil {
return nil
}
return idx.Frame(name)
return idx.Field(name)
}
// View returns the view for an index, frame, and name.
func (h *Holder) View(index, frame, name string) *View {
f := h.Frame(index, frame)
// View returns the view for an index, field, and name.
func (h *Holder) View(index, field, name string) *View {
f := h.Field(index, field)
if f == nil {
return nil
}
return f.View(name)
}
// Fragment returns the fragment for an index, frame & slice.
func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment {
v := h.View(index, frame, view)
// Fragment returns the fragment for an index, field & slice.
func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment {
v := h.View(index, field, view)
if v == nil {
return nil
}
@ -435,8 +435,8 @@ func (h *Holder) monitorCacheFlush() {
func (h *Holder) flushCaches() {
for _, index := range h.Indexes() {
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.Fragments() {
select {
case <-h.closing:
@ -600,15 +600,15 @@ func (s *HolderSyncer) SyncHolder() error {
}
tf := time.Now()
for _, fi := range di.Frames {
for _, fi := range di.Fields {
// Verify syncer has not closed.
if s.IsClosing() {
return nil
}
// Sync frame row attributes.
if err := s.syncFrame(di.Name, fi.Name); err != nil {
return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err)
// Sync field row attributes.
if err := s.syncField(di.Name, fi.Name); err != nil {
return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err)
}
for _, vi := range fi.Views {
@ -630,11 +630,11 @@ func (s *HolderSyncer) SyncHolder() error {
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
}
}
}
s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0)
s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0)
tf = time.Now() // reset tf
}
s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0)
@ -689,22 +689,22 @@ func (s *HolderSyncer) syncIndex(index string) error {
return nil
}
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve frame reference.
f := s.Holder.Frame(index, name)
// syncField synchronizes field attributes with the rest of the cluster.
func (s *HolderSyncer) syncField(index, name string) error {
// Retrieve field reference.
f := s.Holder.Field(index, name)
if f == nil {
return nil
}
indexTag := fmt.Sprintf("index:%s", index)
frameTag := fmt.Sprintf("frame:%s", name)
fieldTag := fmt.Sprintf("field:%s", name)
// Read block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return errors.Wrap(err, "getting blocks")
}
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag})
s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag})
// Sync with every other host.
for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) {
@ -713,14 +713,14 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.RowAttrDiff(context.Background(), index, name, blks)
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
if err == ErrFieldNotFound {
continue // field not created remotely yet, skip
} else if err != nil {
return errors.Wrap(err, "getting differing blocks")
} else if len(m) == 0 {
continue
}
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID})
s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID})
// Update local copy.
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
@ -738,11 +738,11 @@ func (s *HolderSyncer) syncFrame(index, name string) error {
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error {
// Retrieve local frame.
f := s.Holder.Frame(index, frame)
func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error {
// Retrieve local field.
f := s.Holder.Field(index, field)
if f == nil {
return ErrFrameNotFound
return ErrFieldNotFound
}
// Ensure view exists locally.
@ -806,8 +806,8 @@ func (c *HolderCleaner) CleanHolder() error {
containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node)
// Get the fragments registered in memory.
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, field := range index.Fields() {
for _, view := range field.Views() {
for _, fragment := range view.Fragments() {
fragSlice := fragment.Slice()
// Ignore fragments that should be present.

52
holder_test.go
View file

@ -91,7 +91,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFramePermission", func(t *testing.T) {
t.Run("ErrFieldPermission", func(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -100,7 +100,7 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -113,13 +113,13 @@ func TestHolder_Open(t *testing.T) {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFrameMetaCorrupt", func(t *testing.T) {
t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -127,17 +127,17 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") {
t.Fatalf("unexpected error: %s", err)
}
})
t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) {
t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -145,7 +145,7 @@ func TestHolder_Open(t *testing.T) {
t.Fatal(err)
}
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") {
if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") {
t.Fatalf("unexpected error: %s", err)
}
})
@ -159,9 +159,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -183,9 +183,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -208,9 +208,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
t.Fatal(err)
@ -231,9 +231,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
t.Fatal(err)
@ -257,9 +257,9 @@ func TestHolder_Open(t *testing.T) {
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil {
} else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
} else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil {
t.Fatal(err)
} else if _, err := view.SetBit(0, 0); err != nil {
t.Fatal(err)
@ -391,11 +391,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
cluster.Nodes[1].URI = *uri
// Create frames on nodes.
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0, hldr1} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
hldr.MustCreateFieldIfNotExists("i", "f")
hldr.MustCreateFieldIfNotExists("i", "f0")
hldr.MustCreateFieldIfNotExists("y", "z")
}
// Set data on the local holder.
@ -496,11 +496,11 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0)
// Create frames on nodes.
// Create fields on nodes.
for _, hldr := range []*test.Holder{hldr0} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
hldr.MustCreateFieldIfNotExists("i", "f")
hldr.MustCreateFieldIfNotExists("i", "f0")
hldr.MustCreateFieldIfNotExists("y", "z")
}
// Set data on the local holder.

153
index.go
View file

@ -28,14 +28,14 @@ import (
"github.com/pkg/errors"
)
// Index represents a container for frames.
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
path string
name string
// Frames by name.
frames map[string]*Frame
// Fields by name.
fields map[string]*Field
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
@ -61,7 +61,7 @@ func NewIndex(path, name string) (*Index, error) {
return &Index{
path: path,
name: name,
frames: make(map[string]*Frame),
fields: make(map[string]*Field),
remoteMaxSlice: 0,
@ -106,8 +106,8 @@ func (i *Index) Open() error {
return errors.Wrap(err, "loading meta file")
}
if err := i.openFrames(); err != nil {
return errors.Wrap(err, "opening frames")
if err := i.openFields(); err != nil {
return errors.Wrap(err, "opening fields")
}
if err := i.columnAttrStore.Open(); err != nil {
@ -117,8 +117,8 @@ func (i *Index) Open() error {
return nil
}
// openFrames opens and initializes the frames inside the index.
func (i *Index) openFrames() error {
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields() error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -135,14 +135,14 @@ func (i *Index) openFrames() error {
continue
}
fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := fr.Open(); err != nil {
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
i.frames[fr.Name()] = fr
i.fields[fld.Name()] = fld
}
return nil
}
@ -189,7 +189,7 @@ func (i *Index) saveMeta() error {
}
*/
// Close closes the index and its frames.
// Close closes the index and its fields.
func (i *Index) Close() error {
i.mu.Lock()
defer i.mu.Unlock()
@ -197,13 +197,13 @@ func (i *Index) Close() error {
// Close the attribute store.
i.columnAttrStore.Close()
// Close all frames.
for _, f := range i.frames {
// Close all fields.
for _, f := range i.fields {
if err := f.Close(); err != nil {
return errors.Wrap(err, "closing frame")
return errors.Wrap(err, "closing field")
}
}
i.frames = make(map[string]*Frame)
i.fields = make(map[string]*Field)
return nil
}
@ -217,7 +217,7 @@ func (i *Index) MaxSlice() uint64 {
defer i.mu.RUnlock()
max := i.remoteMaxSlice
for _, f := range i.frames {
for _, f := range i.fields {
if slice := f.MaxSlice(); slice > max {
max = slice
}
@ -234,154 +234,139 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.remoteMaxSlice = newmax
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
// FieldPath returns the path to a field in the index.
func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) }
// Frame returns a frame in the index by name.
func (i *Index) Frame(name string) *Frame {
// Field returns a field in the index by name.
func (i *Index) Field(name string) *Field {
i.mu.RLock()
defer i.mu.RUnlock()
return i.frame(name)
return i.field(name)
}
func (i *Index) frame(name string) *Frame { return i.frames[name] }
func (i *Index) field(name string) *Field { return i.fields[name] }
// Frames returns a list of all frames in the index.
func (i *Index) Frames() []*Frame {
// Fields returns a list of all fields in the index.
func (i *Index) Fields() []*Field {
i.mu.RLock()
defer i.mu.RUnlock()
a := make([]*Frame, 0, len(i.frames))
for _, f := range i.frames {
a := make([]*Field, 0, len(i.fields))
for _, f := range i.fields {
a = append(a, f)
}
sort.Sort(frameSlice(a))
sort.Sort(fieldSlice(a))
return a
}
// RecalculateCaches recalculates caches on every frame in the index.
// RecalculateCaches recalculates caches on every field in the index.
func (i *Index) RecalculateCaches() {
for _, frame := range i.Frames() {
frame.RecalculateCaches()
for _, field := range i.Fields() {
field.RecalculateCaches()
}
}
// CreateFrame creates a frame.
func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
// CreateField creates a field.
func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Ensure frame doesn't already exist.
if i.frames[name] != nil {
return nil, ErrFrameExists
// Ensure field doesn't already exist.
if i.fields[name] != nil {
return nil, ErrFieldExists
}
return i.createFrame(name, opt)
return i.createField(name, opt)
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find frame in cache first.
if f := i.frames[name]; f != nil {
// Find field in cache first.
if f := i.fields[name]; f != nil {
return f, nil
}
return i.createFrame(name, opt)
return i.createField(name, opt)
}
func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
func (i *Index) createField(name string, opt FieldOptions) (*Field, error) {
if name == "" {
return nil, errors.New("frame name required")
return nil, errors.New("field name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Validate fields.
for _, field := range opt.Fields {
if err := ValidateField(field); err != nil {
return nil, err
}
// Validate options.
if err := opt.Validate(); err != nil {
return nil, errors.Wrap(err, "validating options")
}
// Initialize frame.
f, err := i.newFrame(i.FramePath(name), name)
// Initialize field.
f, err := i.newField(i.FieldPath(name), name)
if err != nil {
return nil, errors.Wrap(err, "initializing")
}
// Open frame.
// Open field.
if err := f.Open(); err != nil {
return nil, errors.Wrap(err, "opening")
}
// Set the time quantum.
if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil {
// Apply field options.
if err := f.applyOptions(opt); err != nil {
f.Close()
return nil, errors.Wrap(err, "setting time quantum")
return nil, errors.Wrap(err, "applying options")
}
// Set cache type.
if opt.CacheType == "" {
opt.CacheType = DefaultCacheType
}
f.cacheType = opt.CacheType
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
// Set fields.
f.fields = opt.Fields
if err := f.saveMeta(); err != nil {
f.Close()
return nil, errors.Wrap(err, "saving meta")
}
// Add to index's frame lookup.
i.frames[name] = f
// Add to index's field lookup.
i.fields[name] = f
return f, nil
}
func (i *Index) newFrame(path, name string) (*Frame, error) {
f, err := NewFrame(path, i.name, name)
func (i *Index) newField(path, name string) (*Field, error) {
f, err := NewField(path, i.name, name)
if err != nil {
return nil, err
}
f.Logger = i.Logger
f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name))
f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name))
f.broadcaster = i.broadcaster
f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data"))
return f, nil
}
// DeleteFrame removes a frame from the index.
func (i *Index) DeleteFrame(name string) error {
// DeleteField removes a field from the index.
func (i *Index) DeleteField(name string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if frame doesn't exist.
f := i.frame(name)
// Ignore if field doesn't exist.
f := i.field(name)
if f == nil {
return nil
}
// Close frame.
// Close field.
if err := f.Close(); err != nil {
return errors.Wrap(err, "closing")
}
// Delete frame directory.
if err := os.RemoveAll(i.FramePath(name)); err != nil {
// Delete field directory.
if err := os.RemoveAll(i.FieldPath(name)); err != nil {
return errors.Wrap(err, "removing directory")
}
// Remove reference.
delete(i.frames, name)
delete(i.fields, name)
return nil
}
@ -395,7 +380,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
Frames []*FrameInfo `json:"frames"`
Fields []*FieldInfo `json:"fields"`
}
type indexInfoSlice []*IndexInfo
@ -417,7 +402,7 @@ func EncodeIndexes(a []*Index) []*internal.Index {
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Frames: encodeFrames(d.Frames()),
Fields: encodeFields(d.Fields()),
}
}

44
filesystem.go → index_internal_test.go
View file

@ -15,28 +15,32 @@
package pilosa
import (
"fmt"
"net/http"
"io/ioutil"
)
// Ensure nopFileSystem implements interface.
var _ FileSystem = &nopFileSystem{}
// FileSystem represents an interface for a WebUI file system.
type FileSystem interface {
New() (http.FileSystem, error)
// mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func mustOpenIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := NewIndex(path, "i")
if err != nil {
panic(err)
}
if err := index.Open(); err != nil {
panic(err)
}
return index
}
func init() {
NopFileSystem = &nopFileSystem{}
}
// NopFileSystem represents a FileSystem that returns an error if called.
var NopFileSystem FileSystem
type nopFileSystem struct{}
// New is a no-op implementation of FileSystem New method.
func (n *nopFileSystem) New() (http.FileSystem, error) {
return nil, fmt.Errorf("file system not implemented")
// reopen closes the index and reopens it.
func (i *Index) reopen() error {
if err := i.Close(); err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}

215
index_test.go
View file

@ -23,181 +23,184 @@ import (
"github.com/pilosa/pilosa/test"
)
// Ensure index can open and retrieve a frame.
func TestIndex_CreateFrameIfNotExists(t *testing.T) {
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// Ensure index can open and retrieve a field.
func TestIndex_CreateFieldIfNotExists(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
// Create field.
f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
} else if f == nil {
t.Fatal("expected frame")
t.Fatal("expected field")
}
// Retrieve existing frame.
other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
// Retrieve existing field.
other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
} else if f.Frame != other.Frame {
t.Fatal("frame mismatch")
} else if f.Field != other.Field {
t.Fatal("field mismatch")
}
if f.Frame != index.Frame("f") {
t.Fatal("frame mismatch")
if f.Field != index.Field("f") {
t.Fatal("field mismatch")
}
}
func TestIndex_CreateFrame(t *testing.T) {
// Ensure time quantum can be set appropriately on a new frame.
func TestIndex_CreateField(t *testing.T) {
// Ensure time quantum can be set appropriately on a new field.
t.Run("TimeQuantum", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with explicit quantum.
f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")})
// Create field with explicit quantum.
f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeTime,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected frame time quantum: %s", q)
t.Fatalf("unexpected field time quantum: %s", q)
}
})
})
// Ensure frame can include range columns.
// Ensure field can include range columns.
t.Run("BSIFields", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
},
// Create field with schema and verify it exists.
if f, err := index.CreateField("f", pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: 10,
Max: 20,
}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected fields: %#v", f.Fields())
} else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
t.Fatalf("unexpected type: %#v", f.Type())
}
// Reopen the index & verify the fields are loaded.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
}) {
t.Fatalf("unexpected fields after reopen: %#v", f.Fields())
} else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) {
t.Fatalf("unexpected type after reopen: %#v", f.Type())
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// TODO: These errors don't apply here. Instead, we need these tests
// on field creation FieldOptions validation.
/*
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
CacheType: pilosa.CacheTypeRanked,
}); err != nil {
t.Fatal(err)
}
})
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateField("f", pilosa.FieldOptions{
CacheType: pilosa.CacheTypeNone,
CacheSize: uint32(5),
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFrameFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrFieldFieldsAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != nil {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt},
},
}); err != nil {
t.Fatal(err)
}
})
t.Run("ErrFieldNameRequired", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrFieldNameRequired", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFieldNameRequired {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "", Type: pilosa.FieldTypeInt},
},
}); err != pilosa.ErrFieldNameRequired {
t.Fatal(err)
}
})
t.Run("ErrInvalidFieldType", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrInvalidFieldType", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
}); err != pilosa.ErrInvalidFieldType {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: "bad_type"},
},
}); err != pilosa.ErrInvalidFieldType {
t.Fatal(err)
}
})
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
}); err != pilosa.ErrInvalidFieldRange {
t.Fatal(err)
}
})
if _, err := index.CreateField("f", pilosa.FieldOptions{
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},
}); err != pilosa.ErrInvalidBSIGroupRange {
t.Fatal(err)
}
})
*/
})
}
// Ensure index can delete a frame.
func TestIndex_DeleteFrame(t *testing.T) {
// Ensure index can delete a field.
func TestIndex_DeleteField(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
// Create field.
if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Delete frame & verify it's gone.
if err := index.DeleteFrame("f"); err != nil {
// Delete field & verify it's gone.
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
} else if index.Frame("f") != nil {
t.Fatal("expected nil frame")
} else if index.Field("f") != nil {
t.Fatal("expected nil field")
}
// Delete again to make sure it doesn't error.
if err := index.DeleteFrame("f"); err != nil {
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
}
}
// Ensure index can delete a frame.
// Ensure index can delete a field.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {

1201
internal/private.pb.go
View file

File diff suppressed because it is too large Load diff

40
internal/private.proto
View file

@ -5,11 +5,13 @@ package internal;
message IndexMeta {
}
message FrameMeta {
message FieldOptions {
string Type = 8;
string CacheType = 3;
uint32 CacheSize = 4;
int64 Min = 9;
int64 Max = 10;
string TimeQuantum = 5;
repeated Field Fields = 7;
}
message ImportResponse {
@ -18,7 +20,7 @@ message ImportResponse {
message BlockDataRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 5;
uint64 Slice = 4;
uint64 Block = 3;
@ -51,32 +53,20 @@ message CreateIndexMessage {
IndexMeta Meta = 2;
}
message CreateFrameMessage {
string Index = 1;
string Frame = 2;
FrameMeta Meta = 3;
}
message DeleteFrameMessage {
string Index = 1;
string Frame = 2;
}
message CreateFieldMessage {
string Index = 1;
string Frame = 2;
Field Field = 3;
string Field = 2;
FieldOptions Meta = 3;
}
message DeleteFieldMessage {
string Index = 1;
string Frame = 2;
string Field = 3;
string Field = 2;
}
message Frame {
message Field {
string Name = 1;
FrameMeta Meta = 2;
FieldOptions Meta = 2;
repeated string Views = 3;
}
@ -86,7 +76,7 @@ message Schema {
message Index {
string Name = 1;
repeated Frame Frames = 4;
repeated Field Fields = 4;
}
message URI {
@ -123,7 +113,7 @@ message ClusterStatus {
repeated Node Nodes = 3;
}
message Field {
message BSIGroup {
string Name = 1;
string Type = 2;
int64 Min = 3;
@ -132,13 +122,13 @@ message Field {
message CreateViewMessage {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 3;
}
message DeleteViewMessage {
string Index = 1;
string Frame = 2;
string Field = 2;
string View = 3;
}
@ -154,7 +144,7 @@ message ResizeInstruction {
message ResizeSource {
Node Node = 1;
string Index = 2;
string Frame = 3;
string Field = 3;
string View = 4;
uint64 Slice = 5;
}

243
internal/public.pb.go
View file

@ -1,5 +1,6 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// Code generated by protoc-gen-gogo.
// source: public.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
@ -27,8 +28,6 @@ import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import binary "encoding/binary"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
@ -412,7 +411,7 @@ func (m *QueryResult) GetChanged() bool {
type ImportRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
@ -433,9 +432,9 @@ func (m *ImportRequest) GetIndex() string {
return ""
}
func (m *ImportRequest) GetFrame() string {
func (m *ImportRequest) GetField() string {
if m != nil {
return m.Frame
return m.Field
}
return ""
}
@ -484,9 +483,8 @@ func (m *ImportRequest) GetTimestamps() []int64 {
type ImportValueRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"`
Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
@ -504,9 +502,9 @@ func (m *ImportValueRequest) GetIndex() string {
return ""
}
func (m *ImportValueRequest) GetFrame() string {
func (m *ImportValueRequest) GetField() string {
if m != nil {
return m.Frame
return m.Field
}
return ""
}
@ -518,13 +516,6 @@ func (m *ImportValueRequest) GetSlice() uint64 {
return 0
}
func (m *ImportValueRequest) GetField() string {
if m != nil {
return m.Field
}
return ""
}
func (m *ImportValueRequest) GetColumnIDs() []uint64 {
if m != nil {
return m.ColumnIDs
@ -808,8 +799,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
i += 8
i = encodeFixed64Public(dAtA, i, uint64(math.Float64bits(float64(m.FloatValue))))
}
return i, nil
}
@ -1064,11 +1054,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
if len(m.Field) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
dAtA[i] = 0x18
@ -1181,23 +1171,17 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
if len(m.Field) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame)))
i += copy(dAtA[i:], m.Frame)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if m.Slice != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
}
if len(m.Field) > 0 {
dAtA[i] = 0x22
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.Field)))
i += copy(dAtA[i:], m.Field)
}
if len(m.ColumnIDs) > 0 {
dAtA14 := make([]byte, len(m.ColumnIDs)*10)
var j13 int
@ -1251,6 +1235,24 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func encodeFixed64Public(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Public(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
@ -1472,7 +1474,7 @@ func (m *ImportRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Frame)
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
@ -1522,17 +1524,13 @@ func (m *ImportValueRequest) Size() (n int) {
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
l = len(m.Frame)
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if m.Slice != 0 {
n += 1 + sovPublic(uint64(m.Slice))
}
l = len(m.Field)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
if len(m.ColumnIDs) > 0 {
l = 0
for _, e := range m.ColumnIDs {
@ -2335,8 +2333,15 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
if (iNdEx + 8) > l {
return io.ErrUnexpectedEOF
}
v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
v = uint64(dAtA[iNdEx-8])
v |= uint64(dAtA[iNdEx-7]) << 8
v |= uint64(dAtA[iNdEx-6]) << 16
v |= uint64(dAtA[iNdEx-5]) << 24
v |= uint64(dAtA[iNdEx-4]) << 32
v |= uint64(dAtA[iNdEx-3]) << 40
v |= uint64(dAtA[iNdEx-2]) << 48
v |= uint64(dAtA[iNdEx-1]) << 56
m.FloatValue = float64(math.Float64frombits(v))
default:
iNdEx = preIndex
@ -3067,7 +3072,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -3092,7 +3097,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Frame = string(dAtA[iNdEx:postIndex])
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
@ -3437,54 +3442,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Frame = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
}
@ -3513,6 +3470,25 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
m.Field = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType)
}
m.Slice = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Slice |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType == 0 {
var v uint64
@ -3795,50 +3771,49 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 709 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c,
0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4,
0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd,
0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64,
0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8,
0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82,
0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69,
0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f,
0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd,
0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e,
0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e,
0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62,
0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf,
0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8,
0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77,
0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e,
0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35,
0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88,
0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59,
0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7,
0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c,
0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd,
0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85,
0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3,
0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18,
0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3,
0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4,
0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96,
0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60,
0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec,
0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd,
0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e,
0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c,
0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41,
0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62,
0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9,
0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b,
0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82,
0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8,
0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd,
0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7,
0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7,
0xa4, 0x5a, 0x06, 0x00, 0x00,
// 699 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e,
0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54,
0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d,
0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73,
0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26,
0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0,
0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe,
0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46,
0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee,
0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3,
0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5,
0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12,
0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b,
0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98,
0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c,
0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf,
0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6,
0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda,
0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa,
0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3,
0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57,
0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2,
0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd,
0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10,
0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3,
0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95,
0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1,
0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59,
0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5,
0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab,
0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb,
0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94,
0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82,
0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9,
0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9,
0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7,
0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1,
0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f,
0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d,
0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66,
0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90,
0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00,
0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00,
}

5
internal/public.proto
View file

@ -70,7 +70,7 @@ message QueryResult {
message ImportRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
uint64 Slice = 3;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
@ -81,9 +81,8 @@ message ImportRequest {
message ImportValueRequest {
string Index = 1;
string Frame = 2;
string Field = 2;
uint64 Slice = 3;
string Field = 4;
repeated uint64 ColumnIDs = 5;
repeated string ColumnKeys = 7;
repeated int64 Values = 6;

32
pilosa.go
View file

@ -31,26 +31,26 @@ var (
ErrIndexExists = errors.New("index already exists")
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
// ErrFieldRequired is returned when no field is specified.
ErrFieldRequired = errors.New("field required")
ErrFieldExists = errors.New("field already exists")
ErrFieldNotFound = errors.New("field not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")
ErrFieldNameRequired = errors.New("field name required")
ErrInvalidFieldType = errors.New("invalid field type")
ErrInvalidFieldRange = errors.New("invalid field range")
ErrInvalidFieldValueType = errors.New("invalid field value type")
ErrFieldValueTooLow = errors.New("field value too low")
ErrFieldValueTooHigh = errors.New("field value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrBSIGroupNotFound = errors.New("bsigroup not found")
ErrBSIGroupExists = errors.New("bsigroup already exists")
ErrBSIGroupNameRequired = errors.New("bsigroup name required")
ErrInvalidBSIGroupType = errors.New("invalid bsigroup type")
ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range")
ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type")
ErrBSIGroupValueTooLow = errors.New("bsigroup value too low")
ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high")
ErrInvalidRangeOperation = errors.New("invalid range operation")
ErrInvalidBetweenValue = errors.New("invalid value for between operation")
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]")
ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]")
ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]")
// ErrFragmentNotFound is returned when a fragment does not exist.
@ -78,7 +78,7 @@ type BadRequestError struct {
error
}
// Regular expression to validate index and frame names.
// Regular expression to validate index and field names.
var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in an index.

4
pql/ast_test.go
View file

@ -33,11 +33,11 @@ func TestCall_String(t *testing.T) {
c := &pql.Call{
Name: "Range",
Args: map[string]interface{}{
"frame": "f",
"other": "f",
"field0": &pql.Condition{Op: pql.GTE, Value: 10},
},
}
if s := c.String(); s != `Range(field0 >= 10, frame="f")` {
if s := c.String(); s != `Range(field0 >= 10, other="f")` {
t.Fatalf("unexpected string: %s", s)
}
})

10
pql/parser_test.go
View file

@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) {
// Parse with both child calls and arguments.
t.Run("ChildrenAndArguments", func(t *testing.T) {
q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`)
q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
@ -143,9 +143,9 @@ func TestParser_Parse(t *testing.T) {
Name: "TopN",
Children: []*pql.Call{{
Name: "Bitmap",
Args: map[string]interface{}{"id": int64(100), "frame": "other"},
Args: map[string]interface{}{"id": int64(100), "field": "other"},
}},
Args: map[string]interface{}{"n": int64(3), "frame": "f"},
Args: map[string]interface{}{"n": int64(3), "field": "f"},
},
) {
t.Fatalf("unexpected call: %#v", q.Calls[0])
@ -154,14 +154,14 @@ func TestParser_Parse(t *testing.T) {
// Parse a list argument.
t.Run("ListArgument", func(t *testing.T) {
q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`)
q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q.Calls[0],
&pql.Call{
Name: "TopN",
Args: map[string]interface{}{
"frame": "f",
"field": "f",
"ids": []interface{}{int64(0), int64(10), int64(30)},
},
},

29
server.go
View file

@ -455,45 +455,34 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if err := s.Holder.DeleteIndex(obj.Index); err != nil {
return err
}
case *internal.CreateFrameMessage:
case *internal.CreateFieldMessage:
idx := s.Holder.Index(obj.Index)
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
opt := decodeFrameOptions(obj.Meta)
_, err := idx.CreateFrame(obj.Frame, *opt)
opt := decodeFieldOptions(obj.Meta)
_, err := idx.CreateField(obj.Field, *opt)
if err != nil {
return err
}
case *internal.DeleteFrameMessage:
idx := s.Holder.Index(obj.Index)
if err := idx.DeleteFrame(obj.Frame); err != nil {
return err
}
case *internal.CreateFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
field := decodeField(obj.Field)
if err := f.CreateField(field); err != nil {
return err
}
case *internal.DeleteFieldMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
if err := f.DeleteField(obj.Field); err != nil {
idx := s.Holder.Index(obj.Index)
if err := idx.DeleteField(obj.Field); err != nil {
return err
}
case *internal.CreateViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
f := s.Holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
return fmt.Errorf("Local Field not found: %s", obj.Field)
}
_, _, err := f.createViewIfNotExistsBase(obj.View)
if err != nil {
return err
}
case *internal.DeleteViewMessage:
f := s.Holder.Frame(obj.Index, obj.Frame)
f := s.Holder.Field(obj.Index, obj.Field)
if f == nil {
return fmt.Errorf("Local Frame not found: %s", obj.Frame)
return fmt.Errorf("Local Field not found: %s", obj.Field)
}
err := f.DeleteView(obj.View)
if err != nil {

48
server/cluster_test.go
View file

@ -42,7 +42,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Expected indexes and Frames
// Expected indexes and Fields
expected := map[string][]string{
"i": []string{"f"},
}
@ -51,14 +51,14 @@ func TestMain_SendReceiveMessage(t *testing.T) {
client0 := m0.Client()
client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Make sure node0 knows about the index and frame created.
// Make sure node0 knows about the index and field created.
schema0, err := client0.Schema(context.Background())
if err != nil {
t.Fatal(err)
@ -66,15 +66,15 @@ func TestMain_SendReceiveMessage(t *testing.T) {
received0 := map[string][]string{}
for _, idx := range schema0 {
received0[idx.Name] = []string{}
for _, frame := range idx.Frames {
received0[idx.Name] = append(received0[idx.Name], frame.Name)
for _, field := range idx.Fields {
received0[idx.Name] = append(received0[idx.Name], field.Name)
}
}
if !reflect.DeepEqual(received0, expected) {
t.Fatalf("unexpected schema on node0: %s", received0)
}
// Make sure node1 knows about the index and frame created.
// Make sure node1 knows about the index and field created.
schema1, err := client1.Schema(context.Background())
if err != nil {
t.Fatal(err)
@ -82,8 +82,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
received1 := map[string][]string{}
for _, idx := range schema1 {
received1[idx.Name] = []string{}
for _, frame := range idx.Frames {
received1[idx.Name] = append(received1[idx.Name], frame.Name)
for _, field := range idx.Fields {
received1[idx.Name] = append(received1[idx.Name], field.Name)
}
}
if !reflect.DeepEqual(received1, expected) {
@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -206,10 +206,10 @@ func TestClusterResize_AddNode(t *testing.T) {
// Create a client for each node.
client0 := m0.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -250,17 +250,17 @@ func TestClusterResize_AddNode(t *testing.T) {
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=1300000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=1300000)
`); err != nil {
t.Fatal(err)
}
@ -302,17 +302,17 @@ func TestClusterResize_AddNode(t *testing.T) {
client0 := m0.Client()
//client1 := m1.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first node. Note that no data is placed on slice 1.
if _, err := m0.Query("i", "", `
SetBit(row=1, frame="f", col=1)
SetBit(row=1, frame="f", col=2400000)
SetBit(row=1, field="f", col=1)
SetBit(row=1, field="f", col=2400000)
`); err != nil {
t.Fatal(err)
}
@ -455,10 +455,10 @@ func TestClusterResize_RemoveNode(t *testing.T) {
t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) {
client0 := m0.Client()
// Create indexes and frames on one node.
// Create indexes and fields on one node.
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// TODO: Deterministic node IDs would ensure consistent results
setColumns := ""
for i := 0; i < 20; i++ {
setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setColumns); err != nil {

2
server/config.go
View file

@ -61,7 +61,7 @@ type Config struct {
Handler struct {
// CORS Allowed Origins
AllowedOrigins []string `toml:"allowed-origins"`
}
} `toml:"handler"`
// TLS
TLS TLSConfig `toml:"tls"`

2
server/server.go
View file

@ -39,7 +39,6 @@ import (
"github.com/pilosa/pilosa/gcnotify"
"github.com/pilosa/pilosa/gopsutil"
"github.com/pilosa/pilosa/gossip"
"github.com/pilosa/pilosa/statik"
"github.com/pilosa/pilosa/statsd"
"github.com/pkg/errors"
)
@ -170,7 +169,6 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "wrapping handler")
}
handler.Logger = m.logger
handler.FileSystem = &statik.FileSystem{}
handler.API = pilosa.NewAPI()
handler.API.Logger = m.logger

104
server/server_test.go
View file

@ -27,7 +27,7 @@ import (
"testing"
"testing/quick"
"github.com/BurntSushi/toml"
"github.com/pelletier/go-toml"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pilosa/pilosa/test"
@ -51,20 +51,20 @@ func TestMain_Set_Quick(t *testing.T) {
// Execute SetBit() commands.
for _, cmd := range cmds {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
}
if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") {
if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists {
t.Fatal(err)
}
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil {
t.Fatal(err)
}
}
// Validate data.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, columnIDs := range frameSet {
for field, fieldSet := range SetCommands(cmds).Fields() {
for id, columnIDs := range fieldSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -86,8 +86,8 @@ func TestMain_Set_Quick(t *testing.T) {
}
// Validate data after reopening.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, columnIDs := range frameSet {
for field, fieldSet := range SetCommands(cmds).Fields() {
for id, columnIDs := range fieldSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -119,49 +119,49 @@ func TestMain_SetRowAttrs(t *testing.T) {
m := test.MustRunMain()
defer m.Close()
// Create frames.
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set columns on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
// Set columns on different rows in different fields.
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -196,18 +196,18 @@ func TestMain_SetColumnAttrs(t *testing.T) {
m := test.MustRunMain()
defer m.Close()
// Create frames.
// Create fields.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil {
t.Fatal(err)
}
@ -217,7 +217,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
@ -286,15 +286,15 @@ func TestMain_RecalculateHashes(t *testing.T) {
if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create frame:", err)
if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil {
t.Fatal("create field:", err)
}
// Set some columns
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID))
data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID))
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
@ -311,7 +311,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
// Run a TopN query on all nodes. The result should be the same as the target.
for _, m := range cluster {
res, err := m.Query("i", "", `TopN(frame="f")`)
res, err := m.Query("i", "", `TopN(field="f")`)
if err != nil {
t.Fatal(err)
}
@ -325,37 +325,37 @@ func TestMain_RecalculateHashes(t *testing.T) {
// SetCommand represents a command to set a column.
type SetCommand struct {
ID uint64
Frame string
Field string
ColumnID uint64
}
type SetCommands []SetCommand
// Frames returns the set of column ids for each frame/row.
func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
// Fields returns the set of column ids for each field/row.
func (a SetCommands) Fields() map[string]map[uint64][]uint64 {
// Create a set of unique commands.
m := make(map[SetCommand]struct{})
for _, cmd := range a {
m[cmd] = struct{}{}
}
// Build unique ids for each frame & row.
frames := make(map[string]map[uint64][]uint64)
// Build unique ids for each field & row.
fields := make(map[string]map[uint64][]uint64)
for cmd := range m {
if frames[cmd.Frame] == nil {
frames[cmd.Frame] = make(map[uint64][]uint64)
if fields[cmd.Field] == nil {
fields[cmd.Field] = make(map[uint64][]uint64)
}
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID)
fields[cmd.Field][cmd.ID] = append(fields[cmd.Field][cmd.ID], cmd.ColumnID)
}
// Sort each set of column ids.
for _, frame := range frames {
for id := range frame {
sort.Sort(uint64Slice(frame[id]))
for _, field := range fields {
for id := range field {
sort.Sort(uint64Slice(field[id]))
}
}
return frames
return fields
}
// GenerateSetCommands generates random SetCommand objects.
@ -364,7 +364,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
for i := range cmds {
cmds[i] = SetCommand{
ID: uint64(rand.Intn(1000)),
Frame: "x",
Field: "x",
ColumnID: uint64(rand.Intn(10)),
}
}
@ -374,7 +374,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
// ParseConfig parses s into a Config.
func ParseConfig(s string) (server.Config, error) {
var c server.Config
_, err := toml.Decode(s, &c)
err := toml.Unmarshal([]byte(s), &c)
return c, err
}

4
server_test.go
View file

@ -19,9 +19,9 @@ func TestMonitorAntiEntropy(t *testing.T) {
if err != nil {
t.Fatalf("creating index: %v", err)
}
err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{})
err = client.CreateField(context.Background(), "balh", "fralh", pilosa.FieldOptions{})
if err != nil {
t.Fatalf("creating frame: %v", err)
t.Fatalf("creating field: %v", err)
}
time.Sleep(time.Millisecond * 2)

1
statik/.gitignore vendored
View file

@ -1 +0,0 @@
/statik.go

37
statik/filesystem.go
View file

@ -1,37 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//go:generate statik -src=../webui -dest=..
//
// Package statik contains static assets for the Web UI. `go generate` or
// `make generate-statik` will produce statik.go, which is ignored by git.
package statik
import (
"net/http"
"github.com/pilosa/pilosa"
"github.com/rakyll/statik/fs"
)
// Ensure nopFileSystem implements interface.
var _ pilosa.FileSystem = &FileSystem{}
// FileSystem represents a static FileSystem.
type FileSystem struct{}
// New is a statik implementation of FileSystem New method.
func (s *FileSystem) New() (http.FileSystem, error) {
return fs.New()
}

46
stats_test.go
View file

@ -42,39 +42,39 @@ func TestMultiStatClient_Expvar(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1)
if pilosa.Expvar.String() != `{"index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"})
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Gauge creates a unique key, subsequent Gauge calls will overwrite
hldr.Stats.Gauge("g", 5, 1.0)
hldr.Stats.Gauge("g", 8, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Set creates a unique key, subsequent sets will overwrite
hldr.Stats.Set("s", "4", 1.0)
hldr.Stats.Set("s", "7", 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Record timing duration and a uniquely Set key/value
dur, _ := time.ParseDuration("123us")
hldr.Stats.Timing("tt", dur, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Expvar histogram is implemented as a gauge
hldr.Stats.Histogram("hh", 3, 1.0)
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
@ -109,7 +109,7 @@ func TestStatsCount_TopN(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -138,7 +138,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -155,12 +155,12 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
called := false
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
frame := e.Holder.Frame("d", "f")
if frame == nil {
t.Fatal("frame not found")
field := e.Holder.Field("d", "f")
if field == nil {
t.Fatal("field not found")
}
frame.Stats = &MockStats{
field.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "SetRowAttrs" {
t.Errorf("Expected SetRowAttrs, Results %s", name)
@ -168,7 +168,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -199,7 +199,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
called = true
},
}
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -257,7 +257,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) {
}
}
func TestStatsCount_CreateFrame(t *testing.T) {
func TestStatsCount_CreateField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -272,8 +272,8 @@ func TestStatsCount_CreateFrame(t *testing.T) {
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createFrame" {
t.Errorf("Expected createFrame, Results %s", name)
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
@ -282,13 +282,13 @@ func TestStatsCount_CreateFrame(t *testing.T) {
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteFrame(t *testing.T) {
func TestStatsCount_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -298,13 +298,13 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
called := false
// Create index.
indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil {
if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteFrame" {
t.Errorf("Expected deleteFrame, Results %s", name)
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
@ -313,7 +313,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}

42
test/cluster.go
View file

@ -114,19 +114,19 @@ func (t *TestCluster) CreateIndex(name string) error {
return nil
}
func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error {
func (t *TestCluster) CreateField(index, field string, opt pilosa.FieldOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
if _, err := idx.CreateField(field, opt); err != nil {
return err
}
}
return nil
}
func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
func (t *TestCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / pilosa.SliceWidth
@ -137,9 +137,9 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
f := c.Holder.Field(index, field)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
return fmt.Errorf("index/field does not exist: %s/%s", index, field)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
@ -150,30 +150,6 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *
return nil
}
func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / pilosa.SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *TestCluster) clusterByID(id string) *pilosa.Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
@ -391,7 +367,7 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction)
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI)
instrNode := pilosa.DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
@ -404,11 +380,11 @@ func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction)
srcNode := pilosa.DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
f := destCluster.Holder.Field(src.Index, src.Field)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)

103
test/fragment.go
View file

@ -15,9 +15,6 @@
package test
import (
"io/ioutil"
"os"
"github.com/pilosa/pilosa"
)
@ -30,61 +27,6 @@ type Fragment struct {
RowAttrStore pilosa.AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
}
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore
return f
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
if cacheType == "" {
cacheType = pilosa.DefaultCacheType
}
f := NewFragment(index, frame, view, slice, cacheType)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the fragment and removes all underlying data.
func (f *Fragment) Close() error {
defer os.Remove(f.Path())
defer os.Remove(f.CachePath())
defer f.RowAttrStore.Close()
return f.Fragment.Close()
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *Fragment) Reopen() error {
cacheType := f.Fragment.CacheType
path := f.Path()
if err := f.Fragment.Close(); err != nil {
return err
}
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
@ -94,48 +36,3 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
}
// MustClearColumns clears columns on a row. Panic on error.
func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// RowAttrStore provides simple storage for attributes.
type RowAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewRowAttrStore returns a new instance of RowAttrStore.
func NewRowAttrStore() *RowAttrStore {
return &RowAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// RowAttrs returns the attributes set to a row id.
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetRowAttrs assigns a set of attributes to a row id.
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {
if i%100 >= ipct {
continue
}
rowIDs = append(rowIDs, uint64(i%SliceWidth))
columnIDs = append(columnIDs, uint64(i/SliceWidth))
}
return
}

52
test/frame.go
View file

@ -23,48 +23,48 @@ import (
"github.com/pilosa/pilosa"
)
// Frame represents a test wrapper for pilosa.Frame.
type Frame struct {
*pilosa.Frame
// Field represents a test wrapper for pilosa.Field.
type Field struct {
*pilosa.Field
}
// NewFrame returns a new instance of Frame d/0.
func NewFrame() *Frame {
path, err := ioutil.TempDir("", "pilosa-frame-")
// NewField returns a new instance of Field d/0.
func NewField(opt ...pilosa.FieldOption) *Field {
path, err := ioutil.TempDir("", "pilosa-field-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", "f")
field, err := pilosa.NewField(path, "i", "f", opt...)
if err != nil {
panic(err)
}
return &Frame{Frame: frame}
return &Field{Field: field}
}
// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error.
func MustOpenFrame() *Frame {
f := NewFrame()
// MustOpenField returns a new, opened field at a temporary path. Panic on error.
func MustOpenField(opt ...pilosa.FieldOption) *Field {
f := NewField(opt...)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the frame and removes the underlying data.
func (f *Frame) Close() error {
// Close closes the field and removes the underlying data.
func (f *Field) Close() error {
defer os.RemoveAll(f.Path())
return f.Frame.Close()
return f.Field.Close()
}
// Reopen closes the index and reopens it.
func (f *Frame) Reopen() error {
func (f *Field) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
if err := f.Field.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Frame, err = pilosa.NewFrame(path, index, name)
f.Field, err = pilosa.NewField(path, index, name)
if err != nil {
return err
}
@ -75,8 +75,8 @@ func (f *Frame) Reopen() error {
return nil
}
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
// MustSetBit sets a bit on the field. Panic on error.
func (f *Field) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {
panic(err)
@ -84,23 +84,23 @@ func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (c
return changed
}
// Ensure frame can set its cache
func TestFrame_SetCacheSize(t *testing.T) {
f := MustOpenFrame()
// Ensure field can set its cache
func TestField_SetCacheSize(t *testing.T) {
f := MustOpenField()
defer f.Close()
cacheSize := uint32(100)
// Set & retrieve frame cache size.
// Set & retrieve field cache size.
if err := f.SetCacheSize(cacheSize); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size: %d", q)
t.Fatalf("unexpected field cache size: %d", q)
}
// Reload frame and verify that it is persisted.
// Reload field and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size (reopen): %d", q)
t.Fatalf("unexpected field cache size (reopen): %d", q)
}
}

14
test/holder.go
View file

@ -80,9 +80,9 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption
return &Index{Index: idx}
}
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
// MustCreateFieldIfNotExists returns a given field. Panic on error.
func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
@ -90,9 +90,9 @@ func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
func (h *Holder) MustCreateFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{})
if err != nil {
panic(err)
}
@ -108,9 +108,9 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
panic(err)
}

16
test/index.go
View file

@ -73,20 +73,20 @@ func (i *Index) Reopen() error {
return nil
}
// CreateFrame creates a frame with the given options.
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrame(name, opt)
// CreateField creates a field with the given options.
func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateField(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
return &Field{Field: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrameIfNotExists(name, opt)
// CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) {
f, err := i.Index.CreateFieldIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
return &Field{Field: f}, nil
}

24
time_test.go
View file

@ -65,7 +65,7 @@ func TestViewByTimeUnit(t *testing.T) {
})
}
// Ensure all applicable frame names can be generated when mutating a time bit.
// Ensure all applicable field names can be generated when mutating a time bit.
func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)
@ -84,66 +84,66 @@ func TestViewsByTime(t *testing.T) {
})
}
// Ensure sets of frames can be returned for a given time range.
// Ensure sets of fields can be returned for a given time range.
func TestViewsByTimeRange(t *testing.T) {
t.Run("Y", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y"))
if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YM", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM"))
if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD"))
if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("YMDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH"))
if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("M", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M"))
if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MD", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD"))
if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("MDH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH"))
if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("D", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D"))
if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("DH", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH"))
if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
t.Run("H", func(t *testing.T) {
a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H"))
if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) {
t.Fatalf("unexpected frames: %#v", a)
t.Fatalf("unexpected fields: %#v", a)
}
})
}

42
utils_test.go
View file

@ -104,20 +104,20 @@ func (t *ClusterCluster) CreateIndex(name string) error {
return nil
}
func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error {
func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error {
for _, c := range t.Clusters {
idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
return err
}
if _, err := idx.CreateFrame(frame, opt); err != nil {
if _, err := idx.CreateField(field, opt); err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error {
func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error {
// Determine which node should receive the SetBit.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := colID / SliceWidth
@ -128,9 +128,9 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64,
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
f := c.Holder.Field(index, field)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
return fmt.Errorf("index/field does not exist: %s/%s", index, field)
}
_, err := f.SetBit(view, rowID, colID, x)
if err != nil {
@ -141,30 +141,6 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64,
return nil
}
func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error {
// Determine which node should receive the SetFieldValue.
c0 := t.Clusters[0] // use the first node's cluster to determine slice location.
slice := columnID / SliceWidth
nodes := c0.SliceNodes(index, slice)
for _, node := range nodes {
c := t.clusterByID(node.ID)
if c == nil {
continue
}
f := c.Holder.Frame(index, frame)
if f == nil {
return fmt.Errorf("index/frame does not exist: %s/%s", index, frame)
}
_, err := f.SetFieldValue(columnID, name, value)
if err != nil {
return err
}
}
return nil
}
func (t *ClusterCluster) clusterByID(id string) *Cluster {
for _, c := range t.Clusters {
if c.Node.ID == id {
@ -380,7 +356,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
if err := func() error {
// figure out which node it was meant for, then call the operation on that cluster
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI)
// basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI)
instrNode := DecodeNode(instr.Node)
destCluster := t.clusterByID(instrNode.ID)
@ -393,11 +369,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi
srcNode := DecodeNode(src.Node)
srcCluster := t.clusterByID(srcNode.ID)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice)
srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice)
if destFragment == nil {
// Create fragment on destination if it doesn't exist.
f := destCluster.Holder.Frame(src.Index, src.Frame)
f := destCluster.Holder.Field(src.Index, src.Field)
v := f.View(src.View)
var err error
destFragment, err = v.CreateFragmentIfNotExists(src.Slice)

64
view.go
View file

@ -31,7 +31,7 @@ import (
const (
ViewStandard = "standard"
ViewFieldPrefix = "field_"
viewBSIGroupPrefix = "bsig_"
)
// IsValidView returns true if name is valid.
@ -39,18 +39,18 @@ func IsValidView(name string) bool {
return name == ViewStandard
}
// View represents a container for frame data.
// View represents a container for field data.
type View struct {
mu sync.RWMutex
path string
index string
frame string
field string
name string
cacheSize uint32
// Fragments by slice.
cacheType string // passed in by frame
cacheType string // passed in by field
fragments map[uint64]*Fragment
// maxSlice maintains this view's max slice in order to
@ -65,11 +65,11 @@ type View struct {
}
// NewView returns a new instance of View.
func NewView(path, index, frame, name string, cacheSize uint32) *View {
func NewView(path, index, field, name string, cacheSize uint32) *View {
return &View{
path: path,
index: index,
frame: frame,
field: field,
name: name,
cacheSize: cacheSize,
@ -88,8 +88,8 @@ func (v *View) Name() string { return v.name }
// Index returns the index name the view was initialized with.
func (v *View) Index() string { return v.index }
// Frame returns the frame name the view was initialized with.
func (v *View) Frame() string { return v.frame }
// Field returns the field name the view was initialized with.
func (v *View) Field() string { return v.field }
// Path returns the path the view was initialized with.
func (v *View) Path() string { return v.path }
@ -98,7 +98,7 @@ func (v *View) Path() string { return v.path }
func (v *View) Open() error {
// Never keep a cache for field views.
if strings.HasPrefix(v.name, ViewFieldPrefix) {
if strings.HasPrefix(v.name, viewBSIGroupPrefix) {
v.cacheType = CacheTypeNone
}
@ -265,7 +265,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
}
func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.index, v.frame, v.name, slice)
frag := NewFragment(path, v.index, v.field, v.name, slice)
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
frag.Logger = v.Logger
@ -281,7 +281,7 @@ func (v *View) DeleteFragment(slice uint64) error {
return ErrFragmentNotFound
}
v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)
v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
@ -323,30 +323,30 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
return frag.ClearBit(rowID, columnID)
}
// FieldValue uses a column of bits to read a multi-bit value.
func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// value uses a column of bits to read a multi-bit value.
func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return value, exists, err
}
return frag.FieldValue(columnID, bitDepth)
return frag.Value(columnID, bitDepth)
}
// SetFieldValue uses a column of bits to set a multi-bit value.
func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// setValue uses a column of bits to set a multi-bit value.
func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.SetFieldValue(columnID, bitDepth, value)
return frag.SetValue(columnID, bitDepth, value)
}
// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// sum returns the sum & count of a field.
func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
for _, f := range v.Fragments() {
fsum, fcount, err := f.FieldSum(filter, bitDepth)
fsum, fcount, err := f.Sum(filter, bitDepth)
if err != nil {
return sum, count, err
}
@ -356,11 +356,11 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro
return sum, count, nil
}
// FieldMin returns the min and count of a field.
func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) {
// min returns the min and count of a field.
func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
var minHasValue bool
for _, f := range v.Fragments() {
fmin, fcount, err := f.FieldMin(filter, bitDepth)
fmin, fcount, err := f.Min(filter, bitDepth)
if err != nil {
return min, count, err
}
@ -384,10 +384,10 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro
return min, count, nil
}
// FieldMax returns the max and count of a field.
func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) {
// max returns the max and count of a field.
func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
for _, f := range v.Fragments() {
fmax, fcount, err := f.FieldMax(filter, bitDepth)
fmax, fcount, err := f.Max(filter, bitDepth)
if err != nil {
return max, count, err
}
@ -399,11 +399,11 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro
return max, count, nil
}
// FieldRange returns rows with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
// rangeOp returns rows with a field value encoding matching the predicate.
func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.FieldRange(op, bitDepth, predicate)
other, err := frag.RangeOp(op, bitDepth, predicate)
if err != nil {
return nil, err
}
@ -412,12 +412,12 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row,
return r, nil
}
// FieldRangeBetween returns bitmaps with a field value encoding matching any
// rangeBetween returns bitmaps with a field value encoding matching any
// value between predicateMin and predicateMax.
func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax)
other, err := frag.RangeBetween(bitDepth, predicateMin, predicateMax)
if err != nil {
return nil, err
}

10
view_test.go
View file

@ -30,14 +30,14 @@ type View struct {
}
// NewView returns a new instance of View with a temporary path.
func NewView(index, frame, name string) *View {
func NewView(index, field, name string) *View {
path, err := ioutil.TempDir("", "pilosa-view-")
if err != nil {
panic(err)
}
v := &View{
View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize),
View: pilosa.NewView(path, index, field, name, pilosa.DefaultCacheSize),
RowAttrStore: test.MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore
@ -45,8 +45,8 @@ func NewView(index, frame, name string) *View {
}
// MustOpenView creates and opens an view at a temporary path. Panic on error.
func MustOpenView(index, frame, name string) *View {
v := NewView(index, frame, name)
func MustOpenView(index, field, name string) *View {
v := NewView(index, field, name)
if err := v.Open(); err != nil {
panic(err)
}
@ -67,7 +67,7 @@ func (v *View) Reopen() error {
return err
}
v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View = pilosa.NewView(path, v.Index(), v.Field(), v.Name(), pilosa.DefaultCacheSize)
v.View.RowAttrStore = v.RowAttrStore
return v.Open()
}

BIN
webui/assets/chevron-down.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 3.6 KiB

594
webui/assets/main.js
View file

@ -1,594 +0,0 @@
class REPL {
constructor(input, output, button, completer) {
this.input = input
this.output = output
this.button = button
this.completer = completer
this.history = []
this.history_index = 0
this.history_buffer = ''
this.result_number = 0
}
bind_events() {
var repl = this
var keys = {
TAB: 9,
ENTER: 13,
UP_ARROW: 38,
DOWN_ARROW: 40
}
this.input.addEventListener("keydown", function(e) {
if (e.keyCode == keys.UP_ARROW) {
e.preventDefault()
if (repl.input.value.substring(0, repl.input.selectionStart).indexOf('\n') == '-1') {
if (repl.history_index == 0) {
return
} else {
if (repl.history_index == repl.history.length) {
repl.history_buffer = repl.input.value
}
repl.history_index--
repl.input.value = repl.history[repl.history_index]
repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length)
}
}
}
if (e.keyCode == keys.DOWN_ARROW) {
e.preventDefault()
if (repl.input.value.substring(repl.input.selectionEnd, repl.input.length).indexOf('\n') == '-1') {
if (repl.history_index == repl.history.length) {
return
} else {
repl.history_index++
if (repl.history_index == repl.history.length) {
repl.input.value = repl.history_buffer
} else {
repl.input.value = repl.history[repl.history_index]
}
repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length)
}
}
}
if (e.keyCode == keys.ENTER && !e.shiftKey) {
e.preventDefault()
repl.submit();
}
if (e.keyCode == keys.TAB) {
e.preventDefault()
repl.completer.complete()
}
})
repl.button.onclick = function() {
repl.submit();
};
}
submit() {
this.history_buffer = ''
this.history_index = this.history.length
this.history[this.history_index] = this.input.value
this.history_index++
this.process_query(this.input.value)
this.input.value = ""
}
process_query(query) {
var xhr = new XMLHttpRequest();
var url, data, request, command_name;
var e = document.getElementById("index-dropdown");
var indexname = e.options[e.selectedIndex].text;
var repl = this;
if (query.startsWith(":")) {
var parsed_query = parse_query(query, indexname);
if (Object.keys(parsed_query).length === 0) {
repl.create_single_output({
"input": query,
"output": "invalid query",
"status": 400,
"indexname": indexname,
});
return;
} else {
// set selectedIndex from dropdown list
if (parsed_query.command === "use") {
for (var i = 0; i < e.options.length; i++) {
if (e.options[i].text === parsed_query.command_name) {
e.selectedIndex = i;
break;
}
}
return;
}
url = parsed_query.url;
data = parsed_query.data;
request = parsed_query.request;
command_name = parsed_query.command_name;
}
}
else {
url = '/index/' + indexname + '/query'
request = "POST"
data = query
}
xhr.open(request, url);
xhr.setRequestHeader('Content-Type', 'application/text');
var start_time = new Date().getTime();
xhr.onload = function () {
var end_time = new Date().getTime();
repl.result_number++
repl.create_single_output({
"input": query,
"output": xhr.responseText,
"status": xhr.status,
"indexname": indexname,
"querytime_ms": end_time - start_time,
});
};
xhr.send(data);
// Remove index from dropdown with delete index command
if (request === 'DELETE' && url === '/index/' + command_name){
for (var i = 0; i < e.options.length; i++) {
if (e.options[i].text === command_name) {
e.remove(i);
break;
}
}
}
}
create_single_output(res) {
var node = document.createElement("div");
node.classList.add('output');
var output_string = res['output']
var result_class = "result-output"
var getting_started_errors = [
'index not found',
'frame not found',
]
var output_json;
if (isJSON(output_string)) {
output_json = JSON.parse(output_string)
}
// handle output formatting
if (res["status"] != 200) {
result_class = "result-error";
if (output_json) {
if ("error" in output_json) {
if (getting_started_errors.indexOf(output_json['error']) >= 0) {
output_string += `<br />
<br />
Just getting started? Try this:<br />
:create index test<br />
:use test<br />
:create frame foo<br />
SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit
`
}
}
}
}
var markup =`
<div class="panes">
<div class="pane active">
<div class="result-io">
<div class="result-io-header">
<h5>Input</h5>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<em>Source: ${res.indexname}</em>
</div>
<div class="result-input">
${res.input}
</div>
</div>
<div class="result-io">
<div class="result-io-header">
<h5>output</h5>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<em>${res.querytime_ms} ms</em>
</div>
<div class="${result_class}">
${output_string}
</div>
<a href="#" class="expand"><h5>Expand</h5></a>
</div>
</div>
<div class="pane">
<div class="raw">
<a href="${res.url}">Export raw .csv</a>
</div>
</div>
</div>
`
node.innerHTML = markup;
this.output.insertBefore(node, this.output.firstChild);
// Expand when overflow
var element = this.output.firstChild.getElementsByClassName(result_class)[0];
var expand = this.output.firstChild.getElementsByClassName("expand")[0];
if (element.clientHeight < element.scrollHeight) {
expand.style.display = 'block';
} else {
expand.style.display = 'none';
}
expand.onclick = function () {
element.style.height = element.scrollHeight + "px";
expand.style.display = 'none';
return false;
};
}
populate_index_dropdown() {
var xhr = new XMLHttpRequest();
xhr.open('GET', '/schema')
var select = document.getElementById('index-dropdown')
xhr.onload = function() {
var schema = JSON.parse(xhr.responseText)
for(var i=0; i<schema['indexes'].length; i++) {
var opt = document.createElement('option')
opt.value = i+1
opt.innerHTML = schema['indexes'][i]['name']
select.appendChild(opt)
}
// set the active option to one of the populated options, instead of invalid "Index"
if(i > 0) {
select.value = 1;
}
}
xhr.send(null)
}
}
function populate_version() {
var xhr = new XMLHttpRequest();
xhr.open('GET', '/version')
var node = document.getElementById('server-version')
xhr.onload = function() {
var version = JSON.parse(xhr.responseText)['version']
var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1]
var doc_link = document.getElementById('nav-documentation')
doc_link.onclick = function() {
window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/')
}
node.innerHTML = "Pilosa v" + version
}
xhr.send(null)
}
function handle_nav_click(e) {
// e.id = "nav-xxx"
name = e.id.substring(4)
set_active_pane_by_name(name)
window.location.hash = name
}
function set_active_pane_by_name(name) {
// toggle the nav buttons
document.getElementsByClassName("nav-active")[0].classList.remove("nav-active")
document.getElementById("nav-" + name).classList.add("nav-active")
// toggle the main interface content divs
document.getElementsByClassName("interface-active")[0].classList.remove("interface-active")
document.getElementById('interface-' + name).classList.add("interface-active")
// hack hack
switch(name) {
case "cluster":
update_cluster_status()
break
case "documentation":
open_external_docs()
break
}
}
function update_cluster_status() {
var xhr = new XMLHttpRequest();
xhr.open('GET', '/status')
xhr.onload = function() {
var status = JSON.parse(xhr.responseText)
render_status(status)
}
xhr.send(null)
var xhrSchema = new XMLHttpRequest();
xhrSchema.open('GET', '/schema')
xhrSchema.onload = function() {
var schema = JSON.parse(xhrSchema.responseText)
render_schema(schema)
}
xhrSchema.send(null)
}
function render_status(status) {
// render node table
var nodes_div = document.getElementById("status-nodes")
while (nodes_div.firstChild) {
nodes_div.removeChild(nodes_div.firstChild);
}
var nodes = status["nodes"]
table = document.createElement("table")
tbody = document.createElement("tbody")
table.appendChild(tbody)
var caption = document.createElement("caption")
caption.innerHTML = "(" + nodes.length + ")"
table.appendChild(caption)
var header = document.createElement('tr')
markup = `<th>Host</th>
<th>ID</th>
<th>Coordinator</th>`
header.innerHTML = markup
tbody.appendChild(header)
for(var n=0; n<nodes.length; n++) {
var row = document.createElement("tr")
markup = `<td>${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]}</td>
<td>${nodes[n]["id"]}</td>
<td>${nodes[n]["isCoordinator"]}</td>`
row.innerHTML = markup
tbody.appendChild(row)
}
nodes_div.appendChild(table)
}
function render_schema(schema) {
// render index tables
var indexes_div = document.getElementById("status-indexes")
while (indexes_div.firstChild) {
indexes_div.removeChild(indexes_div.firstChild);
}
var indexes = schema["indexes"] // TODO currently comes from only node 0
for(var n=0; n<indexes.length; n++) {
table = document.createElement("table")
tbody = document.createElement("tbody")
table.appendChild(tbody)
var caption = document.createElement("caption")
caption.innerHTML = indexes[n]["name"]
table.appendChild(caption)
var header = document.createElement('tr')
markup = `<th>Name</th>
<th>Cache Type</th>
<th>Cache Size</th>`
header.innerHTML = markup
tbody.appendChild(header)
var frames = indexes[n]["frames"]
if(frames) {
for(var m=0; m<frames.length; m++) {
var row = document.createElement("tr")
row.innerHTML = `<td>${frames[m]["name"]}</td>
<td>${frames[m]["options"]["cacheType"]}</td>
<td>${frames[m]["options"]["cacheSize"]}</td>`
tbody.appendChild(row)
}
}
indexes_div.appendChild(table)
}
// render slice tables
// TODO enable when Slices element is present in status response
/*
var slices_div = document.getElementById("status-slices")
data = ""
for(var n=0; n<nodes.length; n++) {
var indexes = nodes[n]["Indexes"]
for(var m=0; m<indexes.length; m++) {
data += nodes[n]["Host"] + "/" + indexes[m]["Name"] + ": " + indexes[m]["Slices"] + "<br />"
}
}
slices_div.innerHTML = data
*/
}
function open_external_docs() {
window.open("https://www.pilosa.com/docs");
}
function check_anchor_uri() {
var pane_names = {"console": 0, "cluster": 0, "documentation": 0}
var anchor = window.location.hash.substr(1);
if(anchor in pane_names) {
set_active_pane_by_name(anchor)
}
}
Date.prototype.today = function () {
return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate();
}
Date.prototype.timeNow = function () {
return ((this.getHours() < 10)?"0":"") + this.getHours() +":"+ ((this.getMinutes() < 10)?"0":"") + this.getMinutes() +":"+ ((this.getSeconds() < 10)?"0":"") + this.getSeconds();
}
populate_version()
class Autocompleter {
constructor(input, output) {
this.input = input
this.output = output
this.keyword_map = this.static_keywords
this.init_dynamic_keywords()
}
get static_keywords() {
return {
// keyword: length of substring that comes after cursor
"SetBit()": 1,
"ClearBit()": 1,
"SetRowAttrs()": 1,
"SetColumnAttrs()": 1,
"Bitmap()": 1,
"Union()": 1,
"Intersect()": 1,
"Difference()": 1,
"Count()": 1,
"Range()": 1,
"TopN()": 1,
"frame=": 0,
}
}
complete() {
var completer = this
// extract word fragment ending at cursor. a word fragment:
// - starts with last nonalpha character before cursor (or beginning of string)
// - ends at cursor
var word_start = completer.input.selectionEnd-1
while(word_start>0) {
var c = completer.input.value.charCodeAt(word_start)
if(!((c>64 && c<91) || (c>96 && c<123))) {
word_start++
break
}
word_start--
}
var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)
// check for keyword match and insert if exactly one match
var matches = []
for(var keyword in this.keyword_map) {
if(keyword.startsWith(input_word)){
matches.push(keyword)
}
}
if(matches.length > 1) {
// completer.output.innerHTML = whatever
}
if(matches.length == 1) {
// completer.output.innerHTML = ""
var cursor_pos = completer.input.selectionEnd
var completion = matches[0].substring(input_word.length)
var before = completer.input.value.substring(0, cursor_pos)
var after = completer.input.value.substring(cursor_pos)
completer.input.value = before + completion + after
var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]
completer.input.setSelectionRange(new_pos, new_pos)
}
}
init_dynamic_keywords() {
// hit /schema, parse indexes, frames, add to list
}
add_keyword() {
// call when index or frame created in webui
}
remove_keyword() {
// call when index or frame deleted in webui
// issue: if e.g. multiple indexes have same frame, removing one removes all.
// solution: maintain count. requires more elaborate representation of keywords.
}
}
var input = document.getElementById('query')
var output = document.getElementById('outputs')
var button = document.getElementById('query-btn')
var autocomplete_output = document.getElementById('autocomplete-container')
autocompleter = new Autocompleter(input, autocomplete_output)
repl = new REPL(input, output, button, autocompleter)
repl.populate_index_dropdown()
repl.bind_events()
input.focus()
check_anchor_uri()
function isJSON(str) {
try {
JSON.parse(str)
} catch (e) {
return false
}
return true
}
function parse_query(query, indexname) {
var keys = query.replace(/\s+/g, " ").split(" ");
var command = keys[0];
var command_type = keys[1];
var command_name = keys[2];
var option_str = keys.slice(3, keys.length)
var options = parse_options(option_str);
if (command !== ":use") {
if (!command_name){
return {}
}
}
var parsed_query = {};
parsed_query["command"] = command.substr(1, command.length);
parsed_query["command_name"] = command_name;
switch (command) {
case ":create":
parsed_query["request"] = "POST";
if(Object.keys(options).length === 0) {
parsed_query["data"] = "";
} else {
var opts = {"options":{}};
for (var o in options) {
opts.options[o] = options[o]
}
parsed_query["data"] = JSON.stringify(opts);
}
switch (command_type){
case "index":
parsed_query["url"] = '/index/' + command_name;
break;
case "frame":
parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
break
}
break;
case ":delete":
parsed_query["request"] = "DELETE";
switch (command_type){
case "index":
parsed_query["url"] = '/index/' + command_name;
parsed_query["data"] = "";
break;
case "frame":
parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name;
parsed_query["data"] = "";
break;
}
break;
case ":use":
parsed_query["command_name"] = keys[1];
break;
default:
return {}
}
return parsed_query;
}
function parse_options(option_str) {
var int_keys = ["cacheSize"];
var options = {};
for (var i = 0; i < option_str.length; i++) {
var parts = option_str[i].split('=');
if (int_keys.indexOf(parts[0]) !== -1 ){
options[parts[0]] = Number(parts[1])
} else {
options[parts[0]] = parts[1]
}
}
return options;
}

1
webui/assets/nav-cluster-active.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100.75 36.67"><defs><style>.cls-1{fill:#1db598;}.cls-2{fill:none;stroke:#3c5f8d;stroke-miterlimit:10;stroke-width:5px;}</style></defs><title>nav_cluster_1</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><circle class="cls-1" cx="55.14" cy="18.24" r="9.61"/><circle class="cls-1" cx="18.33" cy="18.33" r="9.61"/><circle class="cls-1" cx="91.14" cy="18.24" r="9.61"/><circle class="cls-2" cx="18.33" cy="18.33" r="15.83"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 521 B

1
webui/assets/nav-cluster.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100.75 36.67"><defs><style>.cls-1{fill:#1db598;}.cls-2{fill:none;stroke:#fff;stroke-miterlimit:10;stroke-width:5px;}</style></defs><title>nav_cluster_1</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><circle class="cls-1" cx="55.14" cy="18.24" r="9.61"/><circle class="cls-1" cx="18.33" cy="18.33" r="9.61"/><circle class="cls-1" cx="91.14" cy="18.24" r="9.61"/><circle class="cls-2" cx="18.33" cy="18.33" r="15.83"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 517 B

1
webui/assets/nav-console-active.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 78.47 48.95"><defs><style>.cls-1{fill:none;stroke:#3c5f8d;stroke-miterlimit:10;stroke-width:6px;}.cls-2{fill:#1db598;}</style></defs><title>nav_console</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect class="cls-1" x="3" y="3" width="72.47" height="42.95"/><circle class="cls-2" cx="39.25" cy="25.92" r="9.61"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 416 B

1
webui/assets/nav-console.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 78.47 48.95"><defs><style>.cls-1{fill:none;stroke:#fff;stroke-miterlimit:10;stroke-width:6px;}.cls-2{fill:#1db598;}</style></defs><title>nav_console</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect class="cls-1" x="3" y="3" width="72.47" height="42.95"/><circle class="cls-2" cx="39.25" cy="25.92" r="9.61"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 414 B

1
webui/assets/nav-documentation-active.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 70 44.5"><defs><style>.cls-1{fill:none;stroke:#3c5f8d;stroke-miterlimit:10;stroke-width:5px;}.cls-2{fill:#1db598;}</style></defs><title>documentation</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><line class="cls-1" y1="2.5" x2="70" y2="2.5"/><line class="cls-1" y1="20" x2="70" y2="20"/><line class="cls-1" y1="38.5" x2="52.82" y2="38.5"/><circle class="cls-2" cx="63.5" cy="38.5" r="6"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 492 B

1
webui/assets/nav-documentation.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 70 44.5"><defs><style>.cls-1{fill:none;stroke:#fff;stroke-miterlimit:10;stroke-width:5px;}.cls-2{fill:#1db598;}</style></defs><title>documentation</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><line class="cls-1" y1="2.5" x2="70" y2="2.5"/><line class="cls-1" y1="20" x2="70" y2="20"/><line class="cls-1" y1="38.5" x2="52.82" y2="38.5"/><circle class="cls-2" cx="63.5" cy="38.5" r="6"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 488 B

1
webui/assets/nav_item1.svg
View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 75.47 45.95"><defs><style>.cls-1{fill:none;stroke:#3c5f8d;stroke-miterlimit:10;stroke-width:3px;}.cls-2{fill:#1db598;}</style></defs><title>nav_item1</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><rect class="cls-1" x="1.5" y="1.5" width="72.47" height="42.95"/><circle class="cls-2" cx="37.75" cy="24.42" r="9.61"/></g></g></svg>
Side by side

Before

Width:  |  Height:  |  Size: 418 B

336
webui/assets/style.css
View file

@ -1,336 +0,0 @@
*{
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
body{
font-family: sans-serif;
background-color: #fbfcfd;
margin: 0;
color: #102445;
}
h2{
margin-bottom: 30px;
}
h5{
text-transform: uppercase;
letter-spacing: 2px;
line-height: 1.21;
margin: 0;
}
a{
line-height: 1.38;
letter-spacing: 0.2px;
text-decoration: none;
color: #102445;
}
a:hover{
color: #1db598;
}
textarea{
width: 100%;
margin-bottom: 10px;
border-radius: 2px;
background-color: #fbfcfd;
border: solid 1.5px #e4eff4;
font-family: monospace;
font-size: 16px;
line-height: 1.5;
letter-spacing: 1.1px;
outline: none;
padding: 30px;
}
select{
/*-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
background: url("img/chevron-down.png") no-repeat calc(100% - 10px) !important;*/
border-radius: 3px;
background-color: #fbfcfd;
width: 187px;
height: 50px;
border: solid 1.5px #e4eff4;
font-size: 18px;
font-weight: bold;
line-height: 1.39;
letter-spacing: 0.2px;
color: #102445;
padding: 10.5px;
}
button{
width: 165px;
height: 50px;
border-radius: 3px;
background-color: #1db598;
outline: none;
border: none;
font-size: 16px;
color: white;
}
em{
font-style: normal;
opacity: 0.5;
font-size: 14px;
font-weight: 500;
letter-spacing: 0.2px;
color: #102445;
}
.header{
height: 92px;
display: flex;
align-items: center;
justify-content: space-between;
width: 90%;
margin: auto;
}
.container{
display: flex;
height:100%;
min-height: 100vh;
}
.nav{
color: white;
display: flex;
flex-direction: column;
width: 150px;
background: #3c5f8d;
}
.nav-item{
height:150px;
display: flex;
align-items: center;
justify-content: center;
flex-direction: column;
border-bottom: 3px solid #2a4871;
cursor: pointer;
}
.nav-active{
background: #f2f7f9;
font-weight: bold;
color: #1db598;
}
.nav-item > .nav-image {
display: flex;
}
.nav-item > .nav-image-active {
display: none;
}
.nav-active > .nav-image {
display: none;
}
.nav-active > .nav-image-active {
display: flex;
}
.interface{
display: none;
flex: 1;
flex-direction: column;
align-items: center;
background: #f2f7f9;
}
.interface-active{
display: flex;
}
.query{
margin-bottom: 30px;
}
.query,
.output-container,
.status-container{
width: 75%;
}
.output{
margin-bottom: 30px;
}
.input-controls{
display: flex;
justify-content: flex-end;
}
.tabs{
display: flex;
background: #eaf2f6;
}
.active-tab{
background: white;
font-weight: bold;
color: #1db598;
}
.tab{
height:60px;
width: 100px;
border-top-right-radius: 5px;
display: flex;
align-items: center;
justify-content: center;
visibility: visible;
cursor: pointer;
}
.pane{
background: white;
padding: 30px;
display: none;
}
.active{
display: block;
}
.result-io-header{
display: flex;
align-items: center;
margin-bottom: 15px;
}
.result-input,
.result-output,
.result-error{
height: 60px;
border-radius: 2px;
background-color: #fafafa;
border: solid 1.5px #e4eff4;
font-family: monospace;
font-size: 16px;
line-height: 1.5;
letter-spacing: 1.1px;
color: #102445;
padding: 15px;
margin-bottom: 15px;
word-break: break-all;
overflow-wrap: break-word;
overflow:hidden;
}
.result-output{
background-color: #edf9f7;
border-left: solid 4px #1db598;
}
.result-error{
background-color: #fbf1f0;
border-left: solid 4px #fa3035;
color: #fa3035;
}
.raw{
height: 253px;
display: flex;
align-items: center;
justify-content: center;
}
.result-table > table {
border-left: solid 4px #1db598;
}
table{
border: solid 0.5px #e0e0e0;
width: 100%;
margin-bottom: 30px;
/*color:#3c5f8d;*/
}
caption{
text-align:left;
font-size: 16px;
font-weight: bold;
line-height: 1.21;
letter-spacing: 2px;
text-align: left;
}
th{
font-size: 14px;
font-weight: bold;
line-height: 1.21;
letter-spacing: 2px;
color: #102445;
text-transform: uppercase;
text-align: left;
padding: 21px 30px;
background-color: white;
}
tr{
border: solid 0.5px #e0e0e0;
background-color: white;
}
tr:nth-child(even) {
background-color: #f2f7f9;
}
td{
padding: 21px 30px;
}
.expand {
text-align: center;
}
.query h2 {
display: inline-block;
}
.query-tooltip {
position: relative;
display: inline;
color: #000;
margin-left: 5px;
}
.query-tooltip:hover {
color: #000;
}
.query-tooltip-content {
background-color: rgb(250, 250, 250);
border: solid 1.5px #e4eff4;
color: #102445;
border-radius: 2px;
padding: 15px;
margin-bottom: 15px;
position: absolute;
left: 80px;
top: -30px;
z-index: 1;
}
.query-tooltip-container {
position: relative;
visibility: hidden;
}
.query-tooltip:hover+.query-tooltip-container{
visibility: visible;
}
.code{
font-family: monospace;
}

123
webui/index.html
View file

@ -1,123 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="/assets/style.css">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Pilosa WebUI</title>
<link rel="icon" type="image/png" href="https://www.pilosa.com/img/favicon.png">
</head>
<body>
<div class="header">
<img src="https://www.pilosa.com/img/logo.svg" width="110" alt="">
<div id="server-version" class=""></div>
</div>
<div class="container">
<div class="nav">
<div class="nav-item nav-active" id="nav-console" onclick="handle_nav_click(this)">
<img style='margin-bottom:10px;' height='40px' src="/assets/nav-console.svg" alt="" class="nav-image">
<img style='margin-bottom:10px;' height='40px' src="/assets/nav-console-active.svg" alt="" class="nav-image-active">
Console
</div>
<div class="nav-item" id="nav-cluster" onclick="handle_nav_click(this)">
<img style='margin-bottom:10px;' height='20px' src="/assets/nav-cluster.svg" alt="" class="nav-image">
<img style='margin-bottom:10px;' height='20px' src="/assets/nav-cluster-active.svg" alt="" class="nav-image-active">
Cluster Admin
</div>
<div class="nav-item" id="nav-documentation" onclick="window.open('https://www.pilosa.com/docs/')">
<img style='margin-bottom:10px;' height='30px' src="/assets/nav-documentation.svg" alt="" class="nav-image">
<img style='margin-bottom:10px;' height='30px' src="/assets/nav-documentation-active.svg" alt="" class="nav-image-active">
Documentation
</div>
</div>
<div class="interface interface-active" id="interface-console">
<div class="query">
<h2>Query</h2>
<a class="query-tooltip" href="https://www.pilosa.com/docs/webui/" target="_blank"><sup>?</sup></a>
<div class="query-tooltip-container">
<div class='query-tooltip-content'>
<h5>PQL</h5>
<div class="code">
SetBit(frame=foo, row=0, col=0)<br />
ClearBit(frame=foo, row=0, col=0)<br />
SetRowAttrs(frame=foo, row=0, color="blue")<br />
SetColumnAttrs(frame=foo, col=0, shape="circle")<br />
SetFieldValue(frame=foo, col=0, age=30)<br />
Bitmap(frame=foo, row=0)<br />
Range(frame=foo, row=0, start="2010-01", end="2017-03")<br />
Count(&lt;BITMAP_CALL&gt;)<br />
TopN([BITMAP_CALL], frame=foo, n=20)<br />
Union([BITMAP_CALL, ...])<br />
Intersect(&lt;BITMAP_CALL&gt;, [BITMAP_CALL, ...])<br />
Difference(&lt;BITMAP_CALL&gt;, &lt;BITMAP_CALL&gt;)<br />
Xor([BITMAP_CALL, ...])<br />
Min([BITMAP_CALL], frame=foo, field=age)<br />
Max([BITMAP_CALL], frame=foo, field=age)<br />
Sum([BITMAP_CALL], frame=foo, field=age)
</div>
<br />
<h5>Special commands</h5>
<div class="code">
:create index test<br />
:use test<br />
:create frame foo<br />
:delete index test<br />
:delete frame foo
</div>
<br />
&lt;tab&gt;: autocomplete<br />
&lt;up&gt;/&lt;down&gt;: history<br />
</div>
</div>
<textarea name="name" id='query' rows="5" required></textarea>
<div class='input-controls'>
<div class="input-hints">
<select class="" name="" id="index-dropdown">
<option value="0">Select index</option>
</select>
&nbsp;&nbsp;&nbsp;
<button id='query-btn'>Run Query</button>
</div>
</div>
</div>
<div class="output-container">
<h2>Output</h2>
<div id="outputs">
<!-- results -->
</div>
</div>
</div>
<div class="interface" id="interface-cluster">
<div class="status-container">
<h2>Nodes</h2>
<div class="status-table-container" id="status-nodes">
</div>
</div>
<div class="status-container">
<h2>Indexes</h2>
<div class="status-table-container" id="status-indexes">
</div>
</div>
<div class="status-container">
<!--
<h2>Slice Distribution</h2>
<div class="status-table-container" id="status-slices">
</div>
-->
</div>
</div>
<div class="interface" id="interface-documentation">
docs!
</div>
</div>
<script type="text/javascript" src="/assets/main.js"></script>
</body>
</html>