diff --git a/.travis.yml b/.travis.yml index fbe1e0c9b..0e4f44d1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - GOARCH=amd64 - GOARCH=amd64 ENTERPRISE=1 install: - - make install-dep install-statik vendor generate-statik + - make install-dep vendor script: - make test before_deploy: @@ -27,9 +27,16 @@ deploy: go: "1.10" condition: $GOARCH = amd64 matrix: + # Excluding or allowing failures on non-primary matrix configurations due to long running times. + fast_finish: true allow_failures: - go: master - fast_finish: true + - go: 1.9 + exclude: + - go: 1.9 + env: GOARCH=386 + - go: 1.9 + env: GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c991caeab..2ad9ab13d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,7 +78,7 @@ Pilosa includes a Makefile that automates several tasks: make install ``` -- Install build dependencies (dep, statik, and protoc): +- Install build dependencies (dep and protoc): ```sh make install-build-deps @@ -114,12 +114,6 @@ Pilosa includes a Makefile that automates several tasks: make release ``` -- Generate static assets for the WebUI: - - ```sh - make generate-statik - ``` - - Regenerate protocol buffer files in `internal/`: ```sh diff --git a/Dockerfile b/Dockerfile index 7cdb93cc0..2b83c71c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM golang:1.10.2 as builder COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + && CGO_ENABLED=0 make install-dep install FLAGS="-a" FROM scratch diff --git a/Gopkg.lock b/Gopkg.lock index 8bb1744bd..b3fc9be2b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - name = "github.com/BurntSushi/toml" - packages = ["."] - revision = "b26d9c308763d68093482582cea63d69be07a0f0" - version = "v0.3.0" - [[projects]] branch = "master" name = "github.com/CAFxX/gcnotifier" @@ -76,6 +70,18 @@ packages = ["proto"] revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/cmpopts", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] name = "github.com/gorilla/context" packages = ["."] @@ -212,14 +218,22 @@ [[projects]] name = "github.com/shirou/gopsutil" packages = [ + "cpu", "host", "internal/common", "mem", + "net", "process" ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" +[[projects]] + branch = "master" + name = "github.com/shirou/w32" + packages = ["."] + revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" + [[projects]] branch = "master" name = "github.com/spf13/afero" @@ -302,6 +316,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba" + inputs-digest = "40bd9c0a1a403580ad77f9ae84e81a97da1d1622b3f620bd000271c52b50b8b5" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 3f17f0b2e..821ba33b6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc install install-build-deps install-dep install-protoc install-protoc-gen-gofast prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast test CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -88,16 +88,12 @@ install: vendor generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -# `go generate` statik assets (WebUI) -generate-statik: require-statik - go generate github.com/pilosa/pilosa/statik - # `go generate` stringers generate-stringer: go generate github.com/pilosa/pilosa # `go generate` all needed packages -generate: generate-protoc generate-statik generate-stringer +generate: generate-protoc generate-stringer # Create Docker image from Dockerfile docker: @@ -126,23 +122,17 @@ endef require-dep: $(call require,dep) -require-statik: - $(call require,statik) - require-protoc-gen-gofast: $(call require,protoc-gen-gofast) require-protoc: $(call require,protoc) -install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc install-stringer +install-build-deps: install-dep install-protoc-gen-gofast install-protoc install-stringer install-dep: go get -u github.com/golang/dep/cmd/dep -install-statik: - go get -u github.com/rakyll/statik - install-stringer: go get -u golang.org/x/tools/cmd/stringer diff --git a/api.go b/api.go index 252480d52..7adff1d1b 100644 --- a/api.go +++ b/api.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "strconv" "strings" "time" @@ -45,7 +44,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - RemoteClient *http.Client + TranslateStore TranslateStore Logger Logger } @@ -126,6 +125,18 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if err != nil { return resp, errors.Wrap(err, "reading column attrs") } + + // Translate column attributes, if necessary. + if api.TranslateStore != nil { + for _, col := range resp.ColumnAttrSets { + v, err := api.TranslateStore.TranslateColumnToString(req.Index, col.ID) + if err != nil { + return resp, err + } + col.Key, col.ID = v, 0 + } + } + resp.ColumnAttrSets = columnAttrSets } return resp, nil @@ -217,9 +228,9 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return nil } -// CreateFrame makes the named frame in the named index with the given options. -func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { - if err := api.validate(apiCreateFrame); err != nil { +// CreateField makes the named field in the named index with the given options. +func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, options FieldOptions) (*Field, error) { + if err := api.validate(apiCreateField); err != nil { return nil, errors.Wrap(err, "validating api method") } @@ -229,32 +240,32 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str return nil, ErrIndexNotFound } - // Create frame. - frame, err := index.CreateFrame(frameName, options) + // Create field. + field, err := index.CreateField(fieldName, options) if err != nil { - return nil, errors.Wrap(err, "creating frame") + return nil, errors.Wrap(err, "creating field") } - // Send the create frame message to all nodes. + // Send the create field message to all nodes. err = api.Broadcaster.SendSync( - &internal.CreateFrameMessage{ + &internal.CreateFieldMessage{ Index: indexName, - Frame: frameName, + Field: fieldName, Meta: options.Encode(), }) if err != nil { - api.Logger.Printf("problem sending CreateFrame message: %s", err) - return nil, errors.Wrap(err, "sending CreateFrame message") + api.Logger.Printf("problem sending CreateField message: %s", err) + return nil, errors.Wrap(err, "sending CreateField message") } - api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) - return frame, nil + api.Holder.Stats.CountWithCustomTags("createField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return field, nil } -// DeleteFrame removes the named frame from the named index. If the index is not -// found, an error is returned. If the frame is not found, it is ignored and no +// DeleteField removes the named field from the named index. If the index is not +// found, an error is returned. If the field is not found, it is ignored and no // action is taken. -func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { - if err := api.validate(apiDeleteFrame); err != nil { +func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { return errors.Wrap(err, "validating api method") } @@ -264,40 +275,40 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return ErrIndexNotFound } - // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { - return errors.Wrap(err, "deleting frame") + // Delete field from the index. + if err := index.DeleteField(fieldName); err != nil { + return errors.Wrap(err, "deleting field") } - // Send the delete frame message to all nodes. + // Send the delete field message to all nodes. err := api.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ + &internal.DeleteFieldMessage{ Index: indexName, - Frame: frameName, + Field: fieldName, }) if err != nil { - api.Logger.Printf("problem sending DeleteFrame message: %s", err) - return errors.Wrap(err, "sending DeleteFrame message") + api.Logger.Printf("problem sending DeleteField message: %s", err) + return errors.Wrap(err, "sending DeleteField message") } - api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + api.Holder.Stats.CountWithCustomTags("deleteField", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } -// ExportCSV encodes the fragment designated by the index,frame,slice as +// ExportCSV encodes the fragment designated by the index,field,slice as // CSV of the form , -func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, slice uint64, w io.Writer) error { +func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { return errors.Wrap(err, "validating api method") } // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return ErrClusterDoesNotOwnSlice } // Find the fragment. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return ErrFragmentNotFound } @@ -306,7 +317,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin cw := csv.NewWriter(w) // Iterate over each column. - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), strconv.FormatUint(columnID, 10), @@ -327,19 +338,19 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) return nil, errors.Wrap(err, "validating api method") } - return api.Cluster.SliceNodes(indexName, slice), nil + return api.Cluster.sliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64) (io.WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } @@ -349,15 +360,15 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // UnmarshalFragment creates a new fragment (if necessary) and reads data from a // Reader which was previously written by MarshalFragment to populate the // fragment's data. -func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, slice uint64, reader io.ReadCloser) error { +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, fieldName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { return errors.Wrap(err, "validating api method") } - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) + // Retrieve field. + f := api.Holder.Field(indexName, fieldName) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Retrieve view. @@ -366,7 +377,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa return errors.Wrap(err, "creating view") } - // Retrieve fragment from frame. + // Retrieve fragment from field. frag, err := view.CreateFragmentIfNotExists(slice) if err != nil { return errors.Wrap(err, "creating fragment") @@ -397,13 +408,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // Retrieve fragment from holder. - f := api.Holder.Fragment(req.Index, req.Frame, ViewStandard, req.Slice) + f := api.Holder.Fragment(req.Index, req.Field, ViewStandard, req.Slice) if f == nil { return nil, ErrFragmentNotFound } var resp = internal.BlockDataResponse{} - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs = f.blockData(int(req.Block)) // Encode response. buf, err := proto.Marshal(&resp) @@ -415,13 +426,13 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, } // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. -func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, slice uint64) ([]FragmentBlock, error) { +func (api *API) FragmentBlocks(ctx context.Context, indexName string, fieldName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve fragment from holder. - f := api.Holder.Fragment(indexName, frameName, ViewStandard, slice) + f := api.Holder.Fragment(indexName, fieldName, ViewStandard, slice) if f == nil { return nil, ErrFragmentNotFound } @@ -482,101 +493,22 @@ func (api *API) LocalID() string { return api.Cluster.Node.ID } -// Schema returns information about each index in Pilosa including which frames +// Schema returns information about each index in Pilosa including which fields // and views they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } -// CreateField creates a new BSI field in the given index and frame. -func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { - if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Create new field. - if err := f.CreateField(field); err != nil { - return errors.Wrap(err, "creating field") - } - - // Send the create field message to all nodes. - err := api.Broadcaster.SendSync( - &internal.CreateFieldMessage{ - Index: indexName, - Frame: frameName, - Field: encodeField(field), - }) - if err != nil { - api.Logger.Printf("problem sending CreateField message: %s", err) - } - return errors.Wrap(err, "sending CreateField message") -} - -// DeleteField deletes the given field. -func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { - if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Retrieve frame by name. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Delete field. - if err := f.DeleteField(fieldName); err != nil { - return errors.Wrap(err, "deleting field") - } - - // Send the delete field message to all nodes. - err := api.Broadcaster.SendSync( - &internal.DeleteFieldMessage{ - Index: indexName, - Frame: frameName, - Field: fieldName, - }) - if err != nil { - api.Logger.Printf("problem sending DeleteField message: %s", err) - } - return errors.Wrap(err, "sending DeleteField message") -} - -// Fields returns the fields in the given frame. -func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { - if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validating api method") - } - - index := api.Holder.index(indexName) - if index == nil { - return nil, ErrIndexNotFound - } - - frame := index.frame(frameName) - if frame == nil { - return nil, ErrFrameNotFound - } - - return frame.GetFields() -} - -// Views returns the views in the given frame. -func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { +// Views returns the views in the given field. +func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve views. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Fetch views. @@ -585,15 +517,15 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( } // DeleteView removes the given view. -func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { +func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { return errors.Wrap(err, "validating api method") } - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) + // Retrieve field. + f := api.Holder.Field(indexName, fieldName) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Delete the view. @@ -608,7 +540,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri err := api.Broadcaster.SendSync( &internal.DeleteViewMessage{ Index: indexName, - Frame: frameName, + Field: fieldName, View: viewName, }) if err != nil { @@ -653,15 +585,15 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At return attrs, nil } -func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { - if err := api.validate(apiFrameAttrDiff); err != nil { +func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFieldAttrDiff); err != nil { return nil, errors.Wrap(err, "validating api method") } // Retrieve index from holder. - f := api.Holder.Frame(indexName, frameName) + f := api.Holder.Field(indexName, fieldName) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Retrieve local blocks. @@ -687,15 +619,15 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s return attrs, nil } -// Import bulk imports data into a particular index,frame,slice. +// Import bulk imports data into a particular index,field,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Slice) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } // Convert timestamps to time.Time. @@ -709,9 +641,9 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { } // Import into fragment. - err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + err = field.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -722,15 +654,15 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return errors.Wrap(err, "validating api method") } - _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + _, field, err := api.indexField(req.Index, req.Field, req.Slice) if err != nil { - return errors.Wrap(err, "getting frame") + return errors.Wrap(err, "getting field") } // Import into fragment. - err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + err = field.ImportValue(req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, field=%s, slice=%d, columns=%d, err=%s", req.Index, req.Field, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -758,28 +690,28 @@ func (api *API) LongQueryTime() time.Duration { return api.Cluster.LongQueryTime } -func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { +func (api *API) indexField(indexName string, fieldName string, slice uint64) (*Index, *Field, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { + if !api.Cluster.ownsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("node %s does not own slice %d of index %s", api.LocalID(), slice, indexName) return nil, nil, ErrClusterDoesNotOwnSlice } // Find the Index. - api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) + api.Logger.Printf("importing: %v %v %v", indexName, fieldName, slice) index := api.Holder.Index(indexName) if index == nil { - api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + api.Logger.Printf("fragment error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } - // Retrieve frame. - frame := index.Frame(frameName) - if frame == nil { - api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) - return nil, nil, ErrFrameNotFound + // Retrieve field. + field := index.Field(fieldName) + if field == nil { + api.Logger.Printf("field error: index=%s, field=%s, slice=%d, err=%s", indexName, fieldName, slice, ErrFieldNotFound.Error()) + return nil, nil, ErrFieldNotFound } - return index, frame, nil + return index, field, nil } // SetCoordinator makes a new Node the cluster coordinator. @@ -788,15 +720,15 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validating api method") } - oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.NodeByID(id) + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.nodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } // If the new coordinator is this node, do the SetCoordinator directly. if newNode.ID == api.LocalID() { - return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + return oldNode, newNode, api.Cluster.setCoordinator(newNode) } // Send the set-coordinator message to new node. @@ -818,13 +750,13 @@ func (api *API) RemoveNode(id string) (*Node, error) { return nil, errors.Wrap(err, "validating api method") } - removeNode := api.Cluster.nodeByID(id) + removeNode := api.Cluster.unprotectedNodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") } // Start the resize process (similar to NodeJoin) - err := api.Cluster.NodeLeave(removeNode) + err := api.Cluster.nodeLeave(removeNode) if err != nil { return removeNode, errors.Wrap(err, "calling node leave") } @@ -837,7 +769,7 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validating api method") } - err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + err := api.Cluster.completeCurrentJob(resizeJobStateAborted) return errors.Wrap(err, "complete current job") } @@ -870,17 +802,14 @@ type apiMethod int const ( apiClusterMessage apiMethod = iota apiCreateField - apiCreateFrame apiCreateIndex apiDeleteField - apiDeleteFrame apiDeleteIndex apiDeleteView apiExportCSV - apiFields apiFragmentBlockData apiFragmentBlocks - apiFrameAttrDiff + apiFieldAttrDiff //apiHosts // not implemented apiImport apiImportValue @@ -916,17 +845,14 @@ var methodsResizing = map[apiMethod]struct{}{ var methodsNormal = map[apiMethod]struct{}{ apiCreateField: struct{}{}, - apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, apiDeleteField: struct{}{}, - apiDeleteFrame: struct{}{}, apiDeleteIndex: struct{}{}, apiDeleteView: struct{}{}, apiExportCSV: struct{}{}, - apiFields: struct{}{}, apiFragmentBlockData: struct{}{}, apiFragmentBlocks: struct{}{}, - apiFrameAttrDiff: struct{}{}, + apiFieldAttrDiff: struct{}{}, apiImport: struct{}{}, apiImportValue: struct{}{}, apiIndex: struct{}{}, diff --git a/apimethod_string.go b/apimethod_string.go index 2eb2913ae..a2b934e0f 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -2,15 +2,15 @@ package pilosa -import "strconv" +import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViews" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 87, 101, 114, 126, 135, 155, 172, 188, 197, 211, 219, 235, 253, 261, 281, 294, 308, 323, 340, 353, 373, 381} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 151, 160, 174, 182, 198, 216, 224, 244, 257, 271, 288, 301, 321, 329} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { - return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")" + return fmt.Sprintf("apiMethod(%d)", i) } return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] } diff --git a/attr.go b/attr.go index 03ea4f43f..e73f303af 100644 --- a/attr.go +++ b/attr.go @@ -24,10 +24,10 @@ import ( // Attribute data type enum. const ( - AttrTypeString = 1 - AttrTypeInt = 2 - AttrTypeBool = 3 - AttrTypeFloat = 4 + attrTypeString = 1 + attrTypeInt = 2 + attrTypeBool = 3 + attrTypeFloat = 4 ) // AttrStore represents an interface for handling row/column attributes. @@ -165,19 +165,19 @@ func encodeAttr(key string, value interface{}) *internal.Attr { pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.Type = AttrTypeString + pb.Type = attrTypeString pb.StringValue = value case float64: - pb.Type = AttrTypeFloat + pb.Type = attrTypeFloat pb.FloatValue = value case uint64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = int64(value) case int64: - pb.Type = AttrTypeInt + pb.Type = attrTypeInt pb.IntValue = value case bool: - pb.Type = AttrTypeBool + pb.Type = attrTypeBool pb.BoolValue = value } return pb @@ -186,13 +186,13 @@ func encodeAttr(key string, value interface{}) *internal.Attr { // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { switch attr.Type { - case AttrTypeString: + case attrTypeString: return attr.Key, attr.StringValue - case AttrTypeInt: + case attrTypeInt: return attr.Key, attr.IntValue - case AttrTypeBool: + case attrTypeBool: return attr.Key, attr.BoolValue - case AttrTypeFloat: + case attrTypeFloat: return attr.Key, attr.FloatValue default: return attr.Key, nil @@ -221,3 +221,56 @@ func DecodeAttrs(v []byte) (map[string]interface{}, error) { } return decodeAttrs(pb.GetAttrs()), nil } + +func newMemAttrStore() AttrStore { + return &memAttrStore{ + store: make(map[uint64]map[string]interface{}), + } +} + +// memAttrStore represents an in-memory implementation of the AttrStore interface. +type memAttrStore struct { + store map[uint64]map[string]interface{} +} + +// Path is an in-memory implementation of AttrStore Path method. +func (s *memAttrStore) Path() string { return "" } + +// Open is an in-memory implementation of AttrStore Open method. +func (s *memAttrStore) Open() error { + return nil +} + +// Close is an in-memory implementation of AttrStore Close method. +func (s *memAttrStore) Close() error { + return nil +} + +// Attrs returns a set of attributes by ID. +func (s *memAttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { + return s.store[id], nil +} + +// SetAttrs sets attribute values for a given ID. +func (s *memAttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + s.store[id] = m + return nil +} + +// SetBulkAttrs sets attribute values for a set of ids. +func (s *memAttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error { + for id, v := range m { + s.store[id] = v + } + return nil +} + +// Blocks is an in-memory implementation of AttrStore Blocks method. +func (s *memAttrStore) Blocks() ([]AttrBlock, error) { + return nil, nil +} + +// BlockData is an in-memory implementation of AttrStore BlockData method. +func (s *memAttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) { + return nil, nil +} diff --git a/attr_test.go b/attr_test.go index 8253c8e1d..0c848e2ff 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,15 +15,20 @@ package pilosa_test import ( + "io/ioutil" + "os" "reflect" + "runtime" + "sync" "testing" - "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/boltdb" ) // Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -52,7 +57,7 @@ func TestAttrStore_Attrs(t *testing.T) { // Ensure database returns a non-nil empty map if unset. func TestAttrStore_Attrs_Empty(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() if m, err := s.Attrs(100); err != nil { @@ -64,7 +69,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) { // Ensure database can unset attributes if explicitly set to nil. func TestAttrStore_Attrs_Unset(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -84,7 +89,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) { // Ensure attribute block checksums can be returned. func TestAttrStore_Blocks(t *testing.T) { - s := test.MustOpenAttrStore() + s := MustOpenAttrStore() defer s.Close() // Set attributes. @@ -123,3 +128,67 @@ func TestAttrStore_Blocks(t *testing.T) { t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2]) } } + +// AttrStore represents a test wrapper for pilosa.AttrStore. +type AttrStore struct { + pilosa.AttrStore +} + +// NewAttrStore returns a new instance of AttrStore. +func NewAttrStore(string) pilosa.AttrStore { + f, err := ioutil.TempFile("", "pilosa-attr-") + if err != nil { + panic(err) + } + f.Close() + os.Remove(f.Name()) + + return &AttrStore{boltdb.NewAttrStore(f.Name())} +} + +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + +// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. +func MustOpenAttrStore() pilosa.AttrStore { + s := NewAttrStore("") + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +// Close closes the database and removes the underlying data. +func (s *AttrStore) Close() error { + defer os.RemoveAll(s.Path()) + return s.AttrStore.Close() +} diff --git a/broadcast.go b/broadcast.go index 77b76126d..9b894fea2 100644 --- a/broadcast.go +++ b/broadcast.go @@ -120,23 +120,21 @@ func (n *nopGossiper) SendAsync(pb proto.Message) error { // Broadcast message types. const ( - MessageTypeCreateSlice = iota - MessageTypeCreateIndex - MessageTypeDeleteIndex - MessageTypeCreateFrame - MessageTypeDeleteFrame - MessageTypeCreateView - MessageTypeDeleteView - MessageTypeCreateField - MessageTypeDeleteField - MessageTypeClusterStatus - MessageTypeResizeInstruction - MessageTypeResizeInstructionComplete - MessageTypeSetCoordinator - MessageTypeUpdateCoordinator - MessageTypeNodeState - MessageTypeRecalculateCaches - MessageTypeNodeEvent + messageTypeCreateSlice = iota + messageTypeCreateIndex + messageTypeDeleteIndex + messageTypeCreateField + messageTypeDeleteField + messageTypeCreateView + messageTypeDeleteView + messageTypeClusterStatus + messageTypeResizeInstruction + messageTypeResizeInstructionComplete + messageTypeSetCoordinator + messageTypeUpdateCoordinator + messageTypeNodeState + messageTypeRecalculateCaches + messageTypeNodeEvent ) // MarshalMessage encodes the protobuf message into a byte slice. @@ -144,39 +142,35 @@ func MarshalMessage(m proto.Message) ([]byte, error) { var typ uint8 switch obj := m.(type) { case *internal.CreateSliceMessage: - typ = MessageTypeCreateSlice + typ = messageTypeCreateSlice case *internal.CreateIndexMessage: - typ = MessageTypeCreateIndex + typ = messageTypeCreateIndex case *internal.DeleteIndexMessage: - typ = MessageTypeDeleteIndex - case *internal.CreateFrameMessage: - typ = MessageTypeCreateFrame - case *internal.DeleteFrameMessage: - typ = MessageTypeDeleteFrame - case *internal.CreateViewMessage: - typ = MessageTypeCreateView - case *internal.DeleteViewMessage: - typ = MessageTypeDeleteView + typ = messageTypeDeleteIndex case *internal.CreateFieldMessage: - typ = MessageTypeCreateField + typ = messageTypeCreateField case *internal.DeleteFieldMessage: - typ = MessageTypeDeleteField + typ = messageTypeDeleteField + case *internal.CreateViewMessage: + typ = messageTypeCreateView + case *internal.DeleteViewMessage: + typ = messageTypeDeleteView case *internal.ClusterStatus: - typ = MessageTypeClusterStatus + typ = messageTypeClusterStatus case *internal.ResizeInstruction: - typ = MessageTypeResizeInstruction + typ = messageTypeResizeInstruction case *internal.ResizeInstructionComplete: - typ = MessageTypeResizeInstructionComplete + typ = messageTypeResizeInstructionComplete case *internal.SetCoordinatorMessage: - typ = MessageTypeSetCoordinator + typ = messageTypeSetCoordinator case *internal.UpdateCoordinatorMessage: - typ = MessageTypeUpdateCoordinator + typ = messageTypeUpdateCoordinator case *internal.NodeStateMessage: - typ = MessageTypeNodeState + typ = messageTypeNodeState case *internal.RecalculateCaches: - typ = MessageTypeRecalculateCaches + typ = messageTypeRecalculateCaches case *internal.NodeEventMessage: - typ = MessageTypeNodeEvent + typ = messageTypeNodeEvent default: return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj)) } @@ -193,39 +187,35 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) { var m proto.Message switch typ { - case MessageTypeCreateSlice: + case messageTypeCreateSlice: m = &internal.CreateSliceMessage{} - case MessageTypeCreateIndex: + case messageTypeCreateIndex: m = &internal.CreateIndexMessage{} - case MessageTypeDeleteIndex: + case messageTypeDeleteIndex: m = &internal.DeleteIndexMessage{} - case MessageTypeCreateFrame: - m = &internal.CreateFrameMessage{} - case MessageTypeDeleteFrame: - m = &internal.DeleteFrameMessage{} - case MessageTypeCreateView: - m = &internal.CreateViewMessage{} - case MessageTypeDeleteView: - m = &internal.DeleteViewMessage{} - case MessageTypeCreateField: + case messageTypeCreateField: m = &internal.CreateFieldMessage{} - case MessageTypeDeleteField: + case messageTypeDeleteField: m = &internal.DeleteFieldMessage{} - case MessageTypeClusterStatus: + case messageTypeCreateView: + m = &internal.CreateViewMessage{} + case messageTypeDeleteView: + m = &internal.DeleteViewMessage{} + case messageTypeClusterStatus: m = &internal.ClusterStatus{} - case MessageTypeResizeInstruction: + case messageTypeResizeInstruction: m = &internal.ResizeInstruction{} - case MessageTypeResizeInstructionComplete: + case messageTypeResizeInstructionComplete: m = &internal.ResizeInstructionComplete{} - case MessageTypeSetCoordinator: + case messageTypeSetCoordinator: m = &internal.SetCoordinatorMessage{} - case MessageTypeUpdateCoordinator: + case messageTypeUpdateCoordinator: m = &internal.UpdateCoordinatorMessage{} - case MessageTypeNodeState: + case messageTypeNodeState: m = &internal.NodeStateMessage{} - case MessageTypeRecalculateCaches: + case messageTypeRecalculateCaches: m = &internal.RecalculateCaches{} - case MessageTypeNodeEvent: + case messageTypeNodeEvent: m = &internal.NodeEventMessage{} default: return nil, fmt.Errorf("invalid message type: %d", typ) diff --git a/cache.go b/cache.go index 06046220a..e6a7dc283 100644 --- a/cache.go +++ b/cache.go @@ -27,8 +27,8 @@ import ( ) const ( - // ThresholdFactor is used to calculate the threshold for new items entering the cache - ThresholdFactor = 1.1 + // thresholdFactor is used to calculate the threshold for new items entering the cache + thresholdFactor = 1.1 ) // Cache represents a cache of counts. @@ -158,7 +158,7 @@ type RankCache struct { func NewRankCache(maxEntries uint32) *RankCache { return &RankCache{ maxEntries: maxEntries, - thresholdBuffer: int(ThresholdFactor * float64(maxEntries)), + thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), stats: NopStatsClient, } @@ -409,7 +409,7 @@ func (p Pairs) String() string { return buf.String() } -func encodePairs(a Pairs) []*internal.Pair { +func EncodePairs(a Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { other[i] = encodePair(a[i]) diff --git a/client.go b/client.go index f3e0e5dce..b5dbe809a 100644 --- a/client.go +++ b/client.go @@ -1,879 +1,13 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( - "bytes" "context" - "encoding/json" - "fmt" "io" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "sort" - "strconv" - - "crypto/tls" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" - "github.com/pkg/errors" ) -// ClientOptions represents the configuration for a InternalHTTPClient -type ClientOptions struct { - TLS *tls.Config -} - -// InternalHTTPClient represents a client to the Pilosa cluster. -type InternalHTTPClient struct { - defaultURI *URI - - // The client to use for HTTP communication. - HTTPClient *http.Client -} - -// NewInternalHTTPClient returns a new instance of InternalHTTPClient to connect to host. -func NewInternalHTTPClient(host string, remoteClient *http.Client) (*InternalHTTPClient, error) { - if host == "" { - return nil, ErrHostRequired - } - - uri, err := NewURIFromAddress(host) - if err != nil { - return nil, errors.Wrap(err, "getting URI") - } - - client := NewInternalHTTPClientFromURI(uri, remoteClient) - return client, nil -} - -func NewInternalHTTPClientFromURI(defaultURI *URI, remoteClient *http.Client) *InternalHTTPClient { - return &InternalHTTPClient{ - defaultURI: defaultURI, - HTTPClient: remoteClient, - } -} - -// Host returns the host the client was initialized with. -func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } - -// MaxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx) -} - -// maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/slices/max") - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSlicesMaxResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return rsp.Standard, nil -} - -// Schema returns all index and frame schema information. -func (c *InternalHTTPClient) Schema(ctx context.Context) ([]*IndexInfo, error) { - // Execute request against the host. - u := c.defaultURI.Path("/schema") - - // Build request. - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var rsp getSchemaResponse - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - return rsp.Indexes, nil -} - -// CreateIndex creates a new index on the server. -func (c *InternalHTTPClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { - // Encode query request. - buf, err := json.Marshal(&postIndexRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "encoding request") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrIndexExists - default: - return errors.New(string(body)) - } -} - -// FragmentNodes returns a list of nodes that own a slice. -func (c *InternalHTTPClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { - // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/fragment/nodes") - u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - var a []*Node - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("http: status=%d", resp.StatusCode) - } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { - return nil, fmt.Errorf("json decode: %s", err) - } - - return a, nil -} - -// Query executes query against the index. -func (c *InternalHTTPClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - return c.QueryNode(ctx, c.defaultURI, index, queryRequest) -} - -// QueryNode executes query against the index, sending the request to the node specified. -func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { - if index == "" { - return nil, ErrIndexRequired - } else if queryRequest.Query == "" { - return nil, ErrQueryRequired - } - - // Encode request object. - buf, err := proto.Marshal(queryRequest) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Create HTTP request. - u := uri.Path(fmt.Sprintf("/index/%s/query", index)) - req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return nil, errors.New(string(body)) - } - - qresp := &internal.QueryResponse{} - if err := proto.Unmarshal(body, qresp); err != nil { - return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.Err; s != "" { - return nil, errors.New(s) - } - - return qresp, nil -} - -// Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportPayload(index, frame, slice, bits) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// ImportK bulk imports bits specified by string keys to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportPayloadK(index, frame, columns) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - node := &Node{ - URI: *c.defaultURI, - } - - // Import to node. - if err := c.importNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - - return nil -} - -func (c *InternalHTTPClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { - err := c.CreateIndex(ctx, name, options) - if err == nil || err == ErrIndexExists { - return nil - } - return err -} - -func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error { - err := c.CreateFrame(ctx, indexName, frameName, options) - if err == nil || err == ErrFrameExists { - return nil - } - return err -} - -// marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowIDs := Bits(bits).RowIDs() - columnIDs := Bits(bits).ColumnIDs() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Frame: frame, - Slice: slice, - RowIDs: rowIDs, - ColumnIDs: columnIDs, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - rowKeys := Bits(bits).RowKeys() - columnKeys := Bits(bits).ColumnKeys() - timestamps := Bits(bits).Timestamps() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportRequest{ - Index: index, - Frame: frame, - RowKeys: rowKeys, - ColumnKeys: columnKeys, - Timestamps: timestamps, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ImportValue bulk imports field values for a single slice to a host. -func (c *InternalHTTPClient) ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - buf, err := marshalImportValuePayload(index, frame, field, slice, vals) - if err != nil { - return fmt.Errorf("Error Creating Payload: %s", err) - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Import to each node. - for _, node := range nodes { - if err := c.importValueNode(ctx, node, buf); err != nil { - return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) - } - } - - return nil -} - -// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. -func marshalImportValuePayload(index, frame, field string, slice uint64, vals []FieldValue) ([]byte, error) { - // Separate row and column IDs to reduce allocations. - columnIDs := FieldValues(vals).ColumnIDs() - values := FieldValues(vals).Values() - - // Marshal data to protobuf. - buf, err := proto.Marshal(&internal.ImportValueRequest{ - Index: index, - Frame: frame, - Slice: slice, - Field: field, - ColumnIDs: columnIDs, - Values: values, - }) - if err != nil { - return nil, fmt.Errorf("marshal import request: %s", err) - } - return buf, nil -} - -// importValueNode sends a pre-marshaled import request to a node. -func (c *InternalHTTPClient) importValueNode(ctx context.Context, node *Node, buf []byte) error { - // Create URL & HTTP request. - u := nodePathToURL(node, "/import-value") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("Accept", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } else if resp.StatusCode != http.StatusOK { - return errors.New(string(body)) - } - - var isresp internal.ImportResponse - if err := proto.Unmarshal(body, &isresp); err != nil { - return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.Err; s != "" { - return errors.New(s) - } - - return nil -} - -// ExportCSV bulk exports data for a single slice from a host to CSV format. -func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Attempt nodes in random order. - var e error - for _, i := range rand.Perm(len(nodes)) { - node := nodes[i] - - if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil { - e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) - continue - } else { - return nil - } - } - - return e -} - -// exportNode copies a CSV export from a node to w. -func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error { - // Create URL. - u := nodePathToURL(node, "/export") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Generate HTTP request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Accept", "text/csv") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Validate status code. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("invalid status: %d", resp.StatusCode) - } - - // Copy body to writer. - if _, err := io.Copy(w, resp.Body); err != nil { - return errors.Wrap(err, "copying") - } - - return nil -} - -func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame string, slice uint64, uri URI) (io.ReadCloser, error) { - node := &Node{ - URI: uri, - } - return c.backupSliceNode(ctx, index, frame, slice, node) -} - -func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame string, slice uint64, node *Node) (io.ReadCloser, error) { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - - // Return error if status is not OK. - if resp.StatusCode == http.StatusNotFound { - resp.Body.Close() - return nil, ErrFragmentNotFound - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - - return resp.Body, nil -} - -// CreateFrame creates a new frame on the server. -func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { - if index == "" { - return ErrIndexRequired - } - - // Encode query request. - buf, err := json.Marshal(&postFrameRequest{ - Options: opt, - }) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Create URL & HTTP request. - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s", index, frame)) - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request against the host. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "reading") - } - - // Handle response based on status code. - switch resp.StatusCode { - case http.StatusOK: - return nil // ok - case http.StatusConflict: - return ErrFrameExists - default: - return errors.New(string(body)) - } -} - -// FragmentBlocks returns a list of block checksums for a fragment on a host. -// Only returns blocks which contain data. -func (c *InternalHTTPClient) FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) { - u := uriPathToURL(c.defaultURI, "/fragment/blocks") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFragmentNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp getFragmentBlocksResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Blocks, nil -} - -// BlockData returns row/column id pairs for a block. -func (c *InternalHTTPClient) BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) { - buf, err := proto.Marshal(&internal.BlockDataRequest{ - Index: index, - Frame: frame, - Slice: slice, - Block: uint64(block), - }) - if err != nil { - return nil, nil, errors.Wrap(err, "marshaling") - } - - u := uriPathToURL(c.defaultURI, "/fragment/block/data") - req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/protobuf") - req.Header.Set("Content-Length", strconv.Itoa(len(buf))) - req.Header.Set("Accept", "application/protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // fallthrough - case http.StatusNotFound: - return nil, nil, nil - default: - return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp internal.BlockDataResponse - if body, err := ioutil.ReadAll(resp.Body); err != nil { - return nil, nil, errors.Wrap(err, "reading") - } else if err := proto.Unmarshal(body, &rsp); err != nil { - return nil, nil, errors.Wrap(err, "unmarshalling") - } - return rsp.RowIDs, rsp.ColumnIDs, nil -} - -// ColumnAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/attr/diff", index)) - - // Encode request. - buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postIndexAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// RowAttrDiff returns data from differing blocks on a remote host. -func (c *InternalHTTPClient) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { - u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame)) - - // Encode request. - buf, err := json.Marshal(postFrameAttrDiffRequest{Blocks: blks}) - if err != nil { - return nil, errors.Wrap(err, "marshaling") - } - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, errors.Wrap(err, "executing request") - } - defer resp.Body.Close() - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - case http.StatusNotFound: - return nil, ErrFrameNotFound - default: - return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) - } - - // Decode response object. - var rsp postFrameAttrDiffResponse - if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return nil, errors.Wrap(err, "decoding") - } - return rsp.Attrs, nil -} - -// SendMessage posts a message synchronously. -func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { - msg, err := MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshaling message: %v", err) - } - - u := uriPathToURL(uri, "/cluster/message") - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) - if err != nil { - return errors.Wrap(err, "making new request") - } - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("User-Agent", "pilosa/"+Version) - - // Execute request. - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return fmt.Errorf("executing http request: %v", err) - } - defer resp.Body.Close() - - // Read body. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("reading response body: %v", err) - } - - // Return error if status is not OK. - switch resp.StatusCode { - case http.StatusOK: // ok - default: - return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) - } - - return nil -} - // Bit represents the intersection of a row and a column. It can be specifed by // integer ids or string keys. type Bit struct { @@ -884,163 +18,13 @@ type Bit struct { Timestamp int64 } -// Bits is a slice of Bit. -type Bits []Bit - -func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Bits) Len() int { return len(p) } - -func (p Bits) Less(i, j int) bool { - if p[i].RowID == p[j].RowID { - if p[i].ColumnID < p[j].ColumnID { - return p[i].Timestamp < p[j].Timestamp - } - return p[i].ColumnID < p[j].ColumnID - } - return p[i].RowID < p[j].RowID -} - -// RowIDs returns a slice of all the row IDs. -func (p Bits) RowIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].RowID - } - return other -} - -// ColumnIDs returns a slice of all the column IDs. -func (p Bits) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// RowKeys returns a slice of all the row keys. -func (p Bits) RowKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].RowKey - } - return other -} - -// ColumnKeys returns a slice of all the column keys. -func (p Bits) ColumnKeys() []string { - other := make([]string, len(p)) - for i := range p { - other[i] = p[i].ColumnKey - } - return other -} - -// Timestamps returns a slice of all the timestamps. -func (p Bits) Timestamps() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Timestamp - } - return other -} - -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]Bit { - m := make(map[uint64][]Bit) - for _, bit := range p { - slice := bit.ColumnID / SliceWidth - m[slice] = append(m[slice], bit) - } - - for slice, bits := range m { - sort.Sort(Bits(bits)) - m[slice] = bits - } - - return m -} - -// FieldValues represents the value for a column within a -// range-encoded frame. +// FieldValue represents the value for a column within a +// range-encoded field. type FieldValue struct { ColumnID uint64 Value int64 } -// FieldValues represents a slice of field values. -type FieldValues []FieldValue - -func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p FieldValues) Len() int { return len(p) } - -func (p FieldValues) Less(i, j int) bool { - return p[i].ColumnID < p[j].ColumnID -} - -// ColumnIDs returns a slice of all the column IDs. -func (p FieldValues) ColumnIDs() []uint64 { - other := make([]uint64, len(p)) - for i := range p { - other[i] = p[i].ColumnID - } - return other -} - -// Values returns a slice of all the values. -func (p FieldValues) Values() []int64 { - other := make([]int64, len(p)) - for i := range p { - other[i] = p[i].Value - } - return other -} - -// GroupBySlice returns a map of field values by slice. -func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { - m := make(map[uint64][]FieldValue) - for _, val := range p { - slice := val.ColumnID / SliceWidth - m[slice] = append(m[slice], val) - } - - for slice, vals := range m { - sort.Sort(FieldValues(vals)) - m[slice] = vals - } - - return m -} - -// BitsByPos is a slice of bits sorted row then column. -type BitsByPos []Bit - -func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitsByPos) Len() int { return len(p) } -func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) - if p0 == p1 { - return p[i].Timestamp < p[j].Timestamp - } - return p0 < p1 -} - -func uriPathToURL(uri *URI, path string) url.URL { - return url.URL{ - Scheme: uri.Scheme(), - Host: uri.HostPort(), - Path: path, - } -} - -func nodePathToURL(node *Node, path string) url.URL { - return url.URL{ - Scheme: node.URI.Scheme(), - Host: node.URI.HostPort(), - Path: path, - } -} - // InternalClient should be implemented by any struct that enables any transport between nodes // TODO: Refactor // Note from Travis: Typically an interface containing more than two or three methods is an indication that @@ -1054,16 +38,103 @@ type InternalClient interface { FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error - ImportK(ctx context.Context, index, frame string, bits []Bit) error + Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error + ImportK(ctx context.Context, index, field string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error - EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error - ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error - ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error - CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error - FragmentBlocks(ctx context.Context, index, frame string, slice uint64) ([]FragmentBlock, error) - BlockData(ctx context.Context, index, frame string, slice uint64, block int) ([]uint64, []uint64, error) - ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) - RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error + ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error + ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error + CreateField(ctx context.Context, index, field string, opt FieldOptions) error + FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) + BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) + ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) + RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error + RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) +} + +//=============== + +type InternalQueryClient interface { + QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) +} + +type NopInternalQueryClient struct{} + +func (n *NopInternalQueryClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} + +func NewNopInternalQueryClient() *NopInternalQueryClient { + return &NopInternalQueryClient{} +} + +var _ InternalQueryClient = NewNopInternalQueryClient() + +//=============== + +type NopInternalClient struct{} + +func NewNopInternalClient() *NopInternalClient { + return &NopInternalClient{} +} + +var _ InternalClient = NewNopInternalClient() + +func (n *NopInternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return nil, nil +} +func (n *NopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { + return nil, nil +} +func (n *NopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error { + return nil +} +func (n *NopInternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) { + return nil, nil +} +func (n *NopInternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return nil, nil +} +func (n *NopInternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []Bit) error { + return nil +} +func (n *NopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { + return nil +} +func (n *NopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { + return nil +} +func (n *NopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string, options FieldOptions) error { + return nil +} +func (n *NopInternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []FieldValue) error { + return nil +} +func (n *NopInternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + return nil +} +func (n *NopInternalClient) CreateField(ctx context.Context, index, field string, opt FieldOptions) error { + return nil +} +func (n *NopInternalClient) FragmentBlocks(ctx context.Context, uri *URI, index, field string, slice uint64) ([]FragmentBlock, error) { + return nil, nil +} +func (n *NopInternalClient) BlockData(ctx context.Context, uri *URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + return nil, nil, nil +} +func (n *NopInternalClient) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { + return nil, nil +} +func (n *NopInternalClient) SendMessage(ctx context.Context, uri *URI, pb proto.Message) error { + return nil +} +func (n *NopInternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri URI) (io.ReadCloser, error) { + return nil, nil } diff --git a/cluster.go b/cluster.go index 63f4f62b8..8dc3a60d1 100644 --- a/cluster.go +++ b/cluster.go @@ -21,7 +21,6 @@ import ( "hash/fnv" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "sort" @@ -49,14 +48,14 @@ const ( NodeStateLoading = "LOADING" NodeStateReady = "READY" - // ResizeJob states. - ResizeJobStateRunning = "RUNNING" + // resizeJob states. + resizeJobStateRunning = "RUNNING" // Final states. - ResizeJobStateDone = "DONE" - ResizeJobStateAborted = "ABORTED" + resizeJobStateDone = "DONE" + resizeJobStateAborted = "ABORTED" - ResizeJobActionAdd = "ADD" - ResizeJobActionRemove = "REMOVE" + resizeJobActionAdd = "ADD" + resizeJobActionRemove = "REMOVE" ) // Node represents a node in the cluster. @@ -255,8 +254,8 @@ type Cluster struct { joined bool mu sync.RWMutex - jobs map[int64]*ResizeJob - currentJob *ResizeJob + jobs map[int64]*resizeJob + currentJob *resizeJob // Close management wg sync.WaitGroup @@ -264,8 +263,7 @@ type Cluster struct { Logger Logger - // - RemoteClient *http.Client + InternalClient InternalClient } // NewCluster returns a new instance of Cluster with defaults. @@ -277,35 +275,37 @@ func NewCluster() *Cluster { EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel - jobs: make(map[int64]*ResizeJob), + jobs: make(map[int64]*resizeJob), closing: make(chan struct{}), joining: make(chan struct{}), + InternalClient: NewNopInternalClient(), + Logger: NopLogger, } } -// Coordinator returns the coordinator node. -func (c *Cluster) CoordinatorNode() *Node { - return c.nodeByID(c.Coordinator) +// coordinatorNode returns the coordinator node. +func (c *Cluster) coordinatorNode() *Node { + return c.unprotectedNodeByID(c.Coordinator) } -// IsCoordinator is true if this node is the coordinator. -func (c *Cluster) IsCoordinator() bool { +// isCoordinator is true if this node is the coordinator. +func (c *Cluster) isCoordinator() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.isCoordinator() + return c.unprotectedIsCoordinator() } -func (c *Cluster) isCoordinator() bool { +func (c *Cluster) unprotectedIsCoordinator() bool { return c.Coordinator == c.Node.ID } -// SetCoordinator tells the current node to become the +// setCoordinator tells the current node to become the // Coordinator. In response to this, the current node // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. -func (c *Cluster) SetCoordinator(n *Node) error { +func (c *Cluster) setCoordinator(n *Node) error { c.mu.Lock() // Verify that the new Coordinator value matches // this node. @@ -315,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.updateCoordinator(n) + _ = c.unprotectedUpdateCoordinator(n) c.mu.Unlock() // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( @@ -330,17 +330,17 @@ func (c *Cluster) SetCoordinator(n *Node) error { return c.Broadcaster.SendSync(c.Status()) } -// UpdateCoordinator updates this nodes Coordinator value as well as +// updateCoordinator updates this nodes Coordinator value as well as // changing the corresponding node's IsCoordinator value // to true, and sets all other nodes to false. Returns true if the value // changed. -func (c *Cluster) UpdateCoordinator(n *Node) bool { +func (c *Cluster) updateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - return c.updateCoordinator(n) + return c.unprotectedUpdateCoordinator(n) } -func (c *Cluster) updateCoordinator(n *Node) bool { +func (c *Cluster) unprotectedUpdateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID @@ -356,9 +356,9 @@ func (c *Cluster) updateCoordinator(n *Node) bool { return changed } -// AddNode adds a node to the Cluster and updates and saves the +// addNode adds a node to the Cluster and updates and saves the // new topology. -func (c *Cluster) AddNode(node *Node) error { +func (c *Cluster) addNode(node *Node) error { c.Logger.Printf("add node %s to cluster on %s", node, c.Node) // If the node being added is the coordinator, set it for this node. @@ -383,9 +383,9 @@ func (c *Cluster) AddNode(node *Node) error { return c.saveTopology() } -// RemoveNode removes a node from the Cluster and updates and saves the +// removeNode removes a node from the Cluster and updates and saves the // new topology. -func (c *Cluster) RemoveNode(node *Node) error { +func (c *Cluster) removeNode(node *Node) error { // remove from cluster if !c.removeNodeBasicSorted(node) { return nil @@ -403,8 +403,8 @@ func (c *Cluster) RemoveNode(node *Node) error { return c.saveTopology() } -// NodeIDs returns the list of IDs in the cluster. -func (c *Cluster) NodeIDs() []string { +// nodeIDs returns the list of IDs in the cluster. +func (c *Cluster) nodeIDs() []string { return Nodes(c.Nodes).IDs() } @@ -468,9 +468,9 @@ func (c *Cluster) setState(state string) { } } -func (c *Cluster) SetNodeState(state string) error { - if c.IsCoordinator() { - return c.ReceiveNodeState(c.Node.ID, state) +func (c *Cluster) setNodeState(state string) error { + if c.isCoordinator() { + return c.receiveNodeState(c.Node.ID, state) } // Send node state to coordinator. @@ -480,18 +480,18 @@ func (c *Cluster) SetNodeState(state string) error { } c.Logger.Printf("Sending State %s (%s)", state, c.Coordinator) - if err := c.sendTo(c.CoordinatorNode(), ns); err != nil { + if err := c.sendTo(c.coordinatorNode(), ns); err != nil { return fmt.Errorf("sending node state error: err=%s", err) } return nil } -// ReceiveNodeState sets node state in Topology in order for the +// receiveNodeState sets node state in Topology in order for the // Coordinator to keep track of, during startup, which nodes have // finished opening their Holder. -func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { - if !c.IsCoordinator() { +func (c *Cluster) receiveNodeState(nodeID string, state string) error { + if !c.isCoordinator() { return nil } @@ -511,11 +511,6 @@ func (c *Cluster) ReceiveNodeState(nodeID string, state string) error { return nil } -// localNode is not being used. -//func (c *Cluster) localNode() *Node { -// return c.NodeByURI(c.URI) -//} - // Status returns the internal ClusterStatus representation. func (c *Cluster) Status() *internal.ClusterStatus { return &internal.ClusterStatus{ @@ -525,14 +520,14 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } -func (c *Cluster) NodeByID(id string) *Node { +func (c *Cluster) nodeByID(id string) *Node { c.mu.RLock() defer c.mu.RUnlock() - return c.nodeByID(id) + return c.unprotectedNodeByID(id) } -// nodeByID returns a node reference by ID. -func (c *Cluster) nodeByID(id string) *Node { +// unprotectedNodeByID returns a node reference by ID. +func (c *Cluster) unprotectedNodeByID(id string) *Node { for _, n := range c.Nodes { if n.ID == id { return n @@ -554,7 +549,7 @@ func (c *Cluster) nodePositionByID(nodeID string) int { // addNodeBasicSorted adds a node to the cluster, sorted by id. // Returns a pointer to the node and true if the node was added. func (c *Cluster) addNodeBasicSorted(node *Node) bool { - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) if n != nil { return false } @@ -584,7 +579,7 @@ func (c *Cluster) removeNodeBasicSorted(node *Node) bool { // frag is a struct of basic fragment information. type frag struct { - frame string + field string view string slice uint64 } @@ -617,36 +612,36 @@ func (a fragsByHost) add(b fragsByHost) fragsByHost { return a } -type viewsByFrame map[string][]string +type viewsByField map[string][]string -func (a viewsByFrame) addView(frame, view string) { - a[frame] = append(a[frame], view) +func (a viewsByField) addView(field, view string) { + a[field] = append(a[field], view) } func (c *Cluster) fragsByHost(idx *Index) fragsByHost { - // frameViews is a map of frame to slice of views. - frameViews := make(viewsByFrame) + // fieldViews is a map of field to slice of views. + fieldViews := make(viewsByField) - for _, frame := range idx.Frames() { - for _, view := range frame.Views() { - frameViews.addView(frame.Name(), view.Name()) + for _, field := range idx.Fields() { + for _, view := range field.Views() { + fieldViews.addView(field.Name(), view.name) } } - return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) + return c.fragCombos(idx.Name(), idx.MaxSlice(), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index -// by creating every combination of frame/view specified in `frameViews` up to maxSlice. -func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { +// by creating every combination of field/view specified in `fieldViews` up to maxSlice. +func (c *Cluster) fragCombos(idx string, maxSlice uint64, fieldViews viewsByField) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - nodes := c.SliceNodes(idx, i) + nodes := c.sliceNodes(idx, i) for _, n := range nodes { - // for each frame/view combination: - for frame, views := range frameViews { + // for each field/view combination: + for field, views := range fieldViews { for _, view := range views { - t[n.ID] = append(t[n.ID], frag{frame, view, i}) + t[n.ID] = append(t[n.ID], frag{field, view, i}) } } } @@ -669,10 +664,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenTo-lenFrom > 1 { return "", "", errors.New("adding more than one node at a time is not supported") } - action = ResizeJobActionAdd + action = resizeJobActionAdd // Determine the node ID that is being added. for _, n := range other.Nodes { - if c.nodeByID(n.ID) == nil { + if c.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -682,10 +677,10 @@ func (c *Cluster) diff(other *Cluster) (action string, nodeID string, err error) if lenFrom-lenTo > 1 { return "", "", errors.New("removing more than one node at a time is not supported") } - action = ResizeJobActionRemove + action = resizeJobActionRemove // Determine the node ID that is being removed. for _, n := range c.Nodes { - if other.nodeByID(n.ID) == nil { + if other.unprotectedNodeByID(n.ID) == nil { nodeID = n.ID break } @@ -717,7 +712,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R // If a node is being removed, however, then it will most likely // require that a replica fragment be the source data. srcCluster := c - if action == ResizeJobActionAdd && c.ReplicaN > 1 { + if action == resizeJobActionAdd && c.ReplicaN > 1 { srcCluster = NewCluster() srcCluster.Nodes = Nodes(c.Nodes).Clone() srcCluster.Hasher = c.Hasher @@ -736,7 +731,7 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R srcNodesByFrag := make(map[frag]string) for nodeID, frags := range srcFrags { // If a node is being removed, don't consider it as a source. - if action == ResizeJobActionRemove && nodeID == diffNodeID { + if action == resizeJobActionRemove && nodeID == diffNodeID { continue } for _, frag := range frags { @@ -768,9 +763,9 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R } src := &internal.ResizeSource{ - Node: EncodeNode(c.nodeByID(srcNodeID)), + Node: EncodeNode(c.unprotectedNodeByID(srcNodeID)), Index: idx.Name(), - Frame: frag.frame, + Field: frag.field, View: frag.view, Slice: frag.slice, } @@ -782,8 +777,8 @@ func (c *Cluster) fragSources(to *Cluster, idx *Index) (map[string][]*internal.R return m, nil } -// Partition returns the partition that a slice belongs to. -func (c *Cluster) Partition(index string, slice uint64) int { +// partition returns the partition that a slice belongs to. +func (c *Cluster) partition(index string, slice uint64) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], slice) @@ -794,18 +789,18 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// SliceNodes returns a list of nodes that own a fragment. -func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { - return c.PartitionNodes(c.Partition(index, slice)) +// sliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) sliceNodes(index string, slice uint64) []*Node { + return c.partitionNodes(c.partition(index, slice)) } -// OwnsSlice returns true if a host owns a fragment. -func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { - return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) +// ownsSlice returns true if a host owns a fragment. +func (c *Cluster) ownsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.sliceNodes(index, slice)).ContainsID(nodeID) } -// PartitionNodes returns a list of nodes that own a partition. -func (c *Cluster) PartitionNodes(partitionID int) []*Node { +// partitionNodes returns a list of nodes that own a partition. +func (c *Cluster) partitionNodes(partitionID int) []*Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. replicaN := c.ReplicaN @@ -827,27 +822,13 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node { return nodes } -// OwnsSlices finds the set of slices owned by the node per Index -func (c *Cluster) OwnsSlices(index string, maxSlice uint64, uri URI) []uint64 { +// containsSlices is like OwnsSlices, but it includes replicas. +func (c *Cluster) containsSlices(index string, maxSlice uint64, node *Node) []uint64 { var slices []uint64 for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) - // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(p), len(c.Nodes)) - if c.Nodes[nodeIndex].URI == uri { - slices = append(slices, i) - } - } - return slices -} - -// ContainsSlices is like OwnsSlices, but it includes replicas. -func (c *Cluster) ContainsSlices(index string, maxSlice uint64, node *Node) []uint64 { - var slices []uint64 - for i := uint64(0); i <= maxSlice; i++ { - p := c.Partition(index, i) + p := c.partition(index, i) // Determine the nodes for partition. - nodes := c.PartitionNodes(p) + nodes := c.partitionNodes(p) for _, n := range nodes { if n.ID == node.ID { slices = append(slices, i) @@ -880,7 +861,7 @@ func (h *jmphasher) Hash(key uint64, n int) int { return int(b) } -func (c *Cluster) Open() error { +func (c *Cluster) open() error { // Cluster always comes up in state STARTING until cluster membership is determined. c.state = ClusterStateStarting @@ -892,7 +873,7 @@ func (c *Cluster) Open() error { c.ID = c.Topology.ClusterID // Only the coordinator needs to consider the .topology file. - if c.IsCoordinator() { + if c.isCoordinator() { err := c.considerTopology() if err != nil { return fmt.Errorf("considerTopology: %v", err) @@ -900,7 +881,7 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - err := c.AddNode(c.Node) + err := c.addNode(c.Node) if err != nil { return errors.Wrap(err, "adding local node") } @@ -916,7 +897,7 @@ func (c *Cluster) Open() error { } // If not coordinator then wait for ClusterStatus from coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { // In the case where a node has been restarted and memberlist has // not had enough time to determine the node went down/up, then // the coorninator needs to be alerted that this node is back up @@ -929,7 +910,7 @@ func (c *Cluster) Open() error { Event: uint32(NodeJoin), Node: EncodeNode(c.Node), } - if err := c.Broadcaster.SendAsync(msg); err != nil { + if err := c.Broadcaster.SendSync(msg); err != nil { return fmt.Errorf("sending restart NodeJoin: %v", err) } @@ -941,7 +922,7 @@ func (c *Cluster) Open() error { return nil } -func (c *Cluster) Close() error { +func (c *Cluster) close() error { // Notify goroutines of closing and wait for completion. close(c.closing) c.wg.Wait() @@ -958,14 +939,14 @@ func (c *Cluster) markAsJoined() { } func (c *Cluster) needTopologyAgreement() bool { - return c.State() == ClusterStateStarting && !StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return c.State() == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) haveTopologyAgreement() bool { if c.Static { return true } - return StringSlicesAreEqual(c.Topology.NodeIDs, c.NodeIDs()) + return stringSlicesAreEqual(c.Topology.NodeIDs, c.nodeIDs()) } func (c *Cluster) allNodesReady() bool { @@ -995,10 +976,10 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { // channel, which is not consumed until the code below. var eg errgroup.Group eg.Go(func() error { - return j.Run() + return j.run() }) - // Wait for the ResizeJob to finish or be aborted. + // Wait for the resizeJob to finish or be aborted. c.Logger.Printf("wait for jobResult") jobResult := <-j.result @@ -1009,18 +990,18 @@ func (c *Cluster) handleNodeAction(nodeAction nodeAction) error { c.Logger.Printf("received jobResult: %s", jobResult) switch jobResult { - case ResizeJobStateDone: - if err := c.CompleteCurrentJob(ResizeJobStateDone); err != nil { + case resizeJobStateDone: + if err := c.completeCurrentJob(resizeJobStateDone); err != nil { return errors.Wrap(err, "completing finished job") } // Add/remove uri to/from the cluster. - if j.action == ResizeJobActionRemove { - return c.RemoveNode(nodeAction.node) - } else if j.action == ResizeJobActionAdd { - return c.AddNode(nodeAction.node) + if j.action == resizeJobActionRemove { + return c.removeNode(nodeAction.node) + } else if j.action == resizeJobActionAdd { + return c.addNode(nodeAction.node) } - case ResizeJobStateAborted: - if err := c.CompleteCurrentJob(ResizeJobStateAborted); err != nil { + case resizeJobStateAborted: + if err := c.completeCurrentJob(resizeJobStateAborted); err != nil { return errors.Wrap(err, "completing aborted job") } } @@ -1041,64 +1022,63 @@ func (c *Cluster) sendTo(node *Node, msg proto.Message) error { return nil } -// ListenForJoins handles cluster-resize events. -func (c *Cluster) ListenForJoins() { - c.wg.Add(1) - go func() { defer c.wg.Done(); c.listenForJoins() }() -} - +// listenForJoins handles cluster-resize events. func (c *Cluster) listenForJoins() { - // When a cluster starts, the state is STARTING. - // We first want to wait for at least one node to join. - // Then we want to clear out the joiningLeavingNodes queue (buffered channel). - // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. - // We use a bool `setNormal` to indicate when at least one node has joined. + c.wg.Add(1) + go func() { + defer c.wg.Done() - var setNormal bool + // When a cluster starts, the state is STARTING. + // We first want to wait for at least one node to join. + // Then we want to clear out the joiningLeavingNodes queue (buffered channel). + // Then we want to set the cluster state to NORMAL and resume processing of joiningLeavingNodes events. + // We use a bool `setNormal` to indicate when at least one node has joined. + var setNormal bool - for { + for { - // Handle all pending joins before changing state back to NORMAL. - select { - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) + // Handle all pending joins before changing state back to NORMAL. + select { + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true + continue + default: + } + + // Only change state to NORMAL if we have successfully added at least one host. + if setNormal { + // Put the cluster back to state NORMAL and broadcast. + if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { + c.Logger.Printf("setStateAndBroadcast error: err=%s", err) + } + } + + // Wait for a joining host or a close. + select { + case <-c.closing: + return + case nodeAction := <-c.joiningLeavingNodes: + err := c.handleNodeAction(nodeAction) + if err != nil { + c.Logger.Printf("handleNodeAction error: err=%s", err) + continue + } + setNormal = true continue } - setNormal = true - continue - default: } - - // Only change state to NORMAL if we have successfully added at least one host. - if setNormal { - // Put the cluster back to state NORMAL and broadcast. - if err := c.setStateAndBroadcast(ClusterStateNormal); err != nil { - c.Logger.Printf("setStateAndBroadcast error: err=%s", err) - } - } - - // Wait for a joining host or a close. - select { - case <-c.closing: - return - case nodeAction := <-c.joiningLeavingNodes: - err := c.handleNodeAction(nodeAction) - if err != nil { - c.Logger.Printf("handleNodeAction error: err=%s", err) - continue - } - setNormal = true - continue - } - } + }() } -// generateResizeJob creates a new ResizeJob based on the new node being -// added/removed. It also saves a reference to the ResizeJob in the `jobs` map +// generateResizeJob creates a new resizeJob based on the new node being +// added/removed. It also saves a reference to the resizeJob in the `jobs` map // for future lookup by JobID. -func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { +func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*resizeJob, error) { c.Logger.Printf("generateResizeJob: %v", nodeAction) c.mu.Lock() defer c.mu.Unlock() @@ -1107,7 +1087,7 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { if err != nil { return nil, errors.Wrap(err, "generating job") } - c.Logger.Printf("generated ResizeJob: %d", j.ID) + c.Logger.Printf("generated resizeJob: %d", j.ID) // Save job in jobs map for future reference. c.jobs[j.ID] = j @@ -1121,12 +1101,12 @@ func (c *Cluster) generateResizeJob(nodeAction nodeAction) (*ResizeJob, error) { return j, nil } -// generateResizeJobByAction returns a ResizeJob with instructions based on +// generateResizeJobByAction returns a resizeJob with instructions based on // the difference between Cluster and a new Cluster with/without uri. -// Broadcaster is associated to the ResizeJob here for use in broadcasting +// Broadcaster is associated to the resizeJob here for use in broadcasting // the resize instructions to other nodes in the cluster. -func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, error) { - j := NewResizeJob(c.Nodes, nodeAction.node, nodeAction.action) +func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*resizeJob, error) { + j := newResizeJob(c.Nodes, nodeAction.node, nodeAction.action) j.Broadcaster = c.Broadcaster // toCluster is a clone of Cluster with the new node added/removed for comparison. @@ -1135,9 +1115,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, toCluster.Hasher = c.Hasher toCluster.PartitionN = c.PartitionN toCluster.ReplicaN = c.ReplicaN - if nodeAction.action == ResizeJobActionRemove { + if nodeAction.action == resizeJobActionRemove { toCluster.removeNodeBasicSorted(nodeAction.node) - } else if nodeAction.action == ResizeJobActionAdd { + } else if nodeAction.action == resizeJobActionAdd { toCluster.addNodeBasicSorted(nodeAction.node) } @@ -1168,8 +1148,8 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, } instr := &internal.ResizeInstruction{ JobID: j.ID, - Node: EncodeNode(toCluster.nodeByID(id)), - Coordinator: EncodeNode(c.CoordinatorNode()), + Node: EncodeNode(toCluster.unprotectedNodeByID(id)), + Coordinator: EncodeNode(c.coordinatorNode()), Sources: sources, Schema: c.Holder.EncodeSchema(), // Include the schema to ensure it's in sync on the receiving node. ClusterStatus: c.Status(), @@ -1180,28 +1160,28 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, return j, nil } -// CompleteCurrentJob sets the state of the current ResizeJob +// completeCurrentJob sets the state of the current resizeJob // then removes the pointer to currentJob. -func (c *Cluster) CompleteCurrentJob(state string) error { +func (c *Cluster) completeCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() - if !c.isCoordinator() { + if !c.unprotectedIsCoordinator() { return ErrNodeNotCoordinator } if c.currentJob == nil { return ErrResizeNotRunning } - c.currentJob.SetState(state) + c.currentJob.setState(state) c.currentJob = nil return nil } -// FollowResizeInstruction is run by any node that receives a ResizeInstruction. -func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { +// followResizeInstruction is run by any node that receives a ResizeInstruction. +func (c *Cluster) followResizeInstruction(instr *internal.ResizeInstruction) error { c.Logger.Printf("follow resize instruction on %s", c.Node.ID) // Make sure the cluster status on this node agrees with the Coordinator // before attempting a resize. - if err := c.MergeClusterStatus(instr.ClusterStatus); err != nil { + if err := c.mergeClusterStatus(instr.ClusterStatus); err != nil { return errors.Wrap(err, "merging cluster status") } @@ -1230,19 +1210,16 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return errors.Wrap(err, "applying schema") } - // Create a client for calling remote nodes. - client := NewInternalHTTPClientFromURI(&c.Node.URI, c.RemoteClient) // TODO: ClientOptions - // Request each source file in ResizeSources. for _, src := range instr.Sources { c.Logger.Printf("get slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) srcURI := decodeURI(src.Node.URI) - // Retrieve frame. - f := c.Holder.Frame(src.Index, src.Frame) + // Retrieve field. + f := c.Holder.Field(src.Index, src.Field) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Create view. @@ -1259,7 +1236,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err // Stream slice from remote node. c.Logger.Printf("retrieve slice %d for index %s from host %s", src.Slice, src.Index, src.Node.URI) - rd, err := client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.Slice, srcURI) + rd, err := c.InternalClient.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.Slice, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a slice has been skipped and @@ -1275,7 +1252,7 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return fmt.Errorf("slice %v doesn't exist on host: %s", src.Slice, src.Node.URI) } - // Write to local frame and always close reader. + // Write to local field and always close reader. if err := func() error { defer rd.Close() _, err := frag.ReadFrom(rd) @@ -1296,13 +1273,13 @@ func (c *Cluster) FollowResizeInstruction(instr *internal.ResizeInstruction) err return nil } -func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { +func (c *Cluster) markResizeInstructionComplete(complete *internal.ResizeInstructionComplete) error { - j := c.Job(complete.JobID) + j := c.job(complete.JobID) // Abort the job if an error exists in the complete object. if complete.Error != "" { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.New(complete.Error) } @@ -1310,29 +1287,27 @@ func (c *Cluster) MarkResizeInstructionComplete(complete *internal.ResizeInstruc defer j.mu.Unlock() if j.isComplete() { - return fmt.Errorf("ResizeJob %d is no longer running", j.ID) + return fmt.Errorf("resize job %d is no longer running", j.ID) } // Mark host complete. j.IDs[complete.Node.ID] = true if !j.nodesArePending() { - j.result <- ResizeJobStateDone + j.result <- resizeJobStateDone } return nil } -// Job returns a ResizeJob by id. -func (c *Cluster) Job(id int64) *ResizeJob { +// job returns a resizeJob by id. +func (c *Cluster) job(id int64) *resizeJob { c.mu.RLock() defer c.mu.RUnlock() - return c.job(id) + return c.jobs[id] } -func (c *Cluster) job(id int64) *ResizeJob { return c.jobs[id] } - -type ResizeJob struct { +type resizeJob struct { ID int64 IDs map[string]bool Instructions []*internal.ResizeInstruction @@ -1347,15 +1322,15 @@ type ResizeJob struct { Logger Logger } -// NewResizeJob returns a new instance of ResizeJob. -func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { +// newResizeJob returns a new instance of resizeJob. +func newResizeJob(existingNodes []*Node, node *Node, action string) *resizeJob { // Build a map of uris to track their resize status. // The value for a node will be set to true after that node // has indicated that it has completed all resize instructions. ids := make(map[string]bool) - if action == ResizeJobActionRemove { + if action == resizeJobActionRemove { for _, n := range existingNodes { // Exclude the removed node from the map. if n.ID == node.ID { @@ -1363,7 +1338,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } ids[n.ID] = false } - } else if action == ResizeJobActionAdd { + } else if action == resizeJobActionAdd { for _, n := range existingNodes { ids[n.ID] = false } @@ -1371,7 +1346,7 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { ids[node.ID] = false } - return &ResizeJob{ + return &resizeJob{ ID: rand.Int63(), IDs: ids, action: action, @@ -1380,50 +1355,40 @@ func NewResizeJob(existingNodes []*Node, node *Node, action string) *ResizeJob { } } -func (j *ResizeJob) State() string { - j.mu.RLock() - defer j.mu.RUnlock() - return j.state -} - -func (j *ResizeJob) SetState(state string) { +func (j *resizeJob) setState(state string) { j.mu.Lock() - j.setState(state) + if j.state == "" || j.state == resizeJobStateRunning { + j.state = state + } j.mu.Unlock() } -func (j *ResizeJob) setState(state string) { - if j.state == "" || j.state == ResizeJobStateRunning { - j.state = state - } -} - -// Run distributes ResizeInstructions. -func (j *ResizeJob) Run() error { - j.Logger.Printf("run ResizeJob") +// run distributes ResizeInstructions. +func (j *resizeJob) run() error { + j.Logger.Printf("run resizeJob") // Set job state to RUNNING. - j.SetState(ResizeJobStateRunning) + j.setState(resizeJobStateRunning) // Job can be considered done in the case where it doesn't require any action. if !j.nodesArePending() { - j.Logger.Printf("ResizeJob contains no pending tasks; mark as done") - j.result <- ResizeJobStateDone + j.Logger.Printf("resizeJob contains no pending tasks; mark as done") + j.result <- resizeJobStateDone return nil } - j.Logger.Printf("distribute tasks for ResizeJob") + j.Logger.Printf("distribute tasks for resizeJob") err := j.distributeResizeInstructions() if err != nil { - j.result <- ResizeJobStateAborted + j.result <- resizeJobStateAborted return errors.Wrap(err, "distributing instructions") } return nil } // isComplete return true if the job is any one of several completion states. -func (j *ResizeJob) isComplete() bool { +func (j *resizeJob) isComplete() bool { switch j.state { - case ResizeJobStateDone, ResizeJobStateAborted: + case resizeJobStateDone, resizeJobStateAborted: return true default: return false @@ -1431,7 +1396,7 @@ func (j *ResizeJob) isComplete() bool { } // nodesArePending returns true if any node is still working on the resize. -func (j *ResizeJob) nodesArePending() bool { +func (j *resizeJob) nodesArePending() bool { for _, complete := range j.IDs { if !complete { return true @@ -1440,9 +1405,9 @@ func (j *ResizeJob) nodesArePending() bool { return false } -func (j *ResizeJob) distributeResizeInstructions() error { +func (j *resizeJob) distributeResizeInstructions() error { j.Logger.Printf("distributeResizeInstructions for job %d", j.ID) - // Loop through the ResizeInstructions in ResizeJob and send to each host. + // Loop through the ResizeInstructions in resizeJob and send to each host. for _, instr := range j.Instructions { // Because the node may not be in the cluster yet, create // a dummy node object to use in the SendTo() method. @@ -1658,7 +1623,7 @@ func (c *Cluster) ReceiveEvent(e *NodeEvent) error { case NodeJoin: c.Logger.Printf("received NodeJoin event: %v", e) // Ignore the event if this is not the coordinator. - if !c.IsCoordinator() { + if !c.isCoordinator() { return nil } return c.nodeJoin(e.Node) @@ -1680,7 +1645,7 @@ func (c *Cluster) nodeJoin(node *Node) error { return errors.New(err) } - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node for agreement") } @@ -1710,13 +1675,13 @@ func (c *Cluster) nodeJoin(node *Node) error { // If the cluster already contains the node, just send it the cluster status. // This is useful in the case where a node is restarted or temporarily leaves // the cluster. - if node := c.nodeByID(node.ID); node != nil { + if node := c.unprotectedNodeByID(node.ID); node != nil { return c.sendTo(node, c.Status()) } // If the holder does not yet contain data, go ahead and add the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1729,16 +1694,16 @@ func (c *Cluster) nodeJoin(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{node, ResizeJobActionAdd} + c.joiningLeavingNodes <- nodeAction{node, resizeJobActionAdd} return nil } -// NodeLeave initiates the removal of a node from the cluster. -func (c *Cluster) NodeLeave(node *Node) error { +// nodeLeave initiates the removal of a node from the cluster. +func (c *Cluster) nodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. - if !c.IsCoordinator() { - return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) + if !c.isCoordinator() { + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.coordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1746,7 +1711,7 @@ func (c *Cluster) NodeLeave(node *Node) error { } // Ensure that node is in the cluster. - if c.nodeByID(node.ID) == nil { + if c.unprotectedNodeByID(node.ID) == nil { return fmt.Errorf("Node is not a member of the cluster: %s", node.ID) } @@ -1756,18 +1721,12 @@ func (c *Cluster) NodeLeave(node *Node) error { } // See if resize job can be generated - _, err := c.generateResizeJobByAction(nodeAction{c.nodeByID(node.ID), ResizeJobActionRemove}) - - if err != nil { + if _, err := c.generateResizeJobByAction(nodeAction{c.unprotectedNodeByID(node.ID), resizeJobActionRemove}); err != nil { return errors.Wrap(err, "generating job") } - return c.nodeLeave(node) -} - -func (c *Cluster) nodeLeave(node *Node) error { // Get the actual node in the local cluster. - n := c.nodeByID(node.ID) + n := c.unprotectedNodeByID(node.ID) // Don't do anything else if the cluster doesn't contain the node. if n == nil { @@ -1776,7 +1735,7 @@ func (c *Cluster) nodeLeave(node *Node) error { // If the holder does not yet contain data, go ahead and remove the node. if ok, err := c.Holder.HasData(); !ok && err == nil { - if err := c.RemoveNode(n); err != nil { + if err := c.removeNode(n); err != nil { return errors.Wrap(err, "removing node") } return c.setStateAndBroadcast(ClusterStateNormal) @@ -1789,17 +1748,17 @@ func (c *Cluster) nodeLeave(node *Node) error { if err := c.setStateAndBroadcast(ClusterStateResizing); err != nil { return errors.Wrap(err, "broadcasting state") } - c.joiningLeavingNodes <- nodeAction{n, ResizeJobActionRemove} + c.joiningLeavingNodes <- nodeAction{n, resizeJobActionRemove} return nil } -func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { +func (c *Cluster) mergeClusterStatus(cs *internal.ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.isCoordinator() { + if c.unprotectedIsCoordinator() { return nil } @@ -1810,7 +1769,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { // Add all nodes from the coordinator. for _, node := range officialNodes { - if err := c.AddNode(node); err != nil { + if err := c.addNode(node); err != nil { return errors.Wrap(err, "adding node") } } @@ -1831,7 +1790,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } for _, nodeID := range nodeIDsToRemove { - if err := c.RemoveNode(c.nodeByID(nodeID)); err != nil { + if err := c.removeNode(c.unprotectedNodeByID(nodeID)); err != nil { return errors.Wrap(err, "removing node") } } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 6b1f6e72b..1fd238911 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -15,11 +15,15 @@ package pilosa import ( + "bytes" "io/ioutil" + "math/rand" "reflect" "strings" "testing" + "testing/quick" + "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/internal" ) @@ -45,13 +49,13 @@ func TestFragCombos(t *testing.T) { tests := []struct { idx string maxSlice uint64 - frameViews viewsByFrame + fieldViews viewsByField expected fragsByHost }{ { idx: "i", maxSlice: uint64(2), - frameViews: viewsByFrame{"f": []string{"v1", "v2"}}, + fieldViews: viewsByField{"f": []string{"v1", "v2"}}, expected: fragsByHost{ "node0": []frag{{"f", "v1", uint64(0)}, {"f", "v2", uint64(0)}}, "node1": []frag{{"f", "v1", uint64(1)}, {"f", "v2", uint64(1)}, {"f", "v1", uint64(2)}, {"f", "v2", uint64(2)}}, @@ -60,7 +64,7 @@ func TestFragCombos(t *testing.T) { { idx: "foo", maxSlice: uint64(3), - frameViews: viewsByFrame{"f": []string{"v0"}}, + fieldViews: viewsByField{"f": []string{"v0"}}, expected: fragsByHost{ "node0": []frag{{"f", "v0", uint64(1)}, {"f", "v0", uint64(2)}}, "node1": []frag{{"f", "v0", uint64(0)}, {"f", "v0", uint64(3)}}, @@ -69,7 +73,7 @@ func TestFragCombos(t *testing.T) { } for _, test := range tests { - actual := c.fragCombos(test.idx, test.maxSlice, test.frameViews) + actual := c.fragCombos(test.idx, test.maxSlice, test.fieldViews) if !reflect.DeepEqual(actual, test.expected) { t.Errorf("expected: %v, but got: %v", test.expected, actual) } @@ -145,23 +149,23 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node3) idx := newIndexWithTempPath("i") - frame, err := idx.CreateFrameIfNotExists("f", FrameOptions{}) + field, err := idx.CreateFieldIfNotExists("f", FieldOptions{}) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 101, nil) + _, err = field.SetBit("standard", 1, 101, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 1300000, nil) + _, err = field.SetBit("standard", 1, 1300000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 2600000, nil) + _, err = field.SetBit("standard", 1, 2600000, nil) if err != nil { t.Fatal(err) } - _, err = frame.SetBit("standard", 1, 3900000, nil) + _, err = field.SetBit("standard", 1, 3900000, nil) if err != nil { t.Fatal(err) } @@ -287,19 +291,19 @@ func TestResizeJob(t *testing.T) { { existingNodes: []*Node{node0, node1}, node: node2, - action: ResizeJobActionAdd, + action: resizeJobActionAdd, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false, node2.ID: false}, }, { existingNodes: []*Node{node0, node1, node2}, node: node2, - action: ResizeJobActionRemove, + action: resizeJobActionRemove, expectedIDs: map[string]bool{node0.ID: false, node1.ID: false}, }, } for _, test := range tests { - actual := NewResizeJob(test.existingNodes, test.node, test.action) + actual := newResizeJob(test.existingNodes, test.node, test.action) if err != nil { t.Fatal(err) } @@ -308,3 +312,463 @@ func TestResizeJob(t *testing.T) { } } } + +// Ensure the cluster can fairly distribute partitions across the nodes. +func TestCluster_Owners(t *testing.T) { + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, + }, + Hasher: NewTestModHasher(), + ReplicaN: 2, + } + + // Verify nodes are distributed. + if a := c.partitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } + + // Verify nodes go around the ring. + if a := c.partitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { + t.Fatalf("unexpected owners: %s", spew.Sdump(a)) + } +} + +// Ensure the partitioner can assign a fragment to a partition. +func TestCluster_Partition(t *testing.T) { + if err := quick.Check(func(index string, slice uint64, partitionN int) bool { + c := NewCluster() + c.PartitionN = partitionN + + partitionID := c.partition(index, slice) + if partitionID < 0 || partitionID >= partitionN { + t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) + } + + return true + }, &quick.Config{ + Values: func(values []reflect.Value, rand *rand.Rand) { + values[0], _ = quick.Value(reflect.TypeOf(""), rand) + values[1] = reflect.ValueOf(uint64(rand.Uint32())) + values[2] = reflect.ValueOf(rand.Intn(1000) + 1) + }, + }); err != nil { + t.Fatal(err) + } +} + +// Ensure the hasher can hash correctly. +func TestHasher(t *testing.T) { + for _, tt := range []struct { + key uint64 + bucket []int + }{ + // Generated from the reference C++ code + {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, + {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, + {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, + } { + for i, v := range tt.bucket { + if got := NewHasher().Hash(tt.key, i+1); got != v { + t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) + } + } + } +} + +// Ensure ContainsSlices can find the actual slice list for node and index. +func TestCluster_ContainsSlices(t *testing.T) { + c := NewTestCluster(5) + c.ReplicaN = 3 + slices := c.containsSlices("test", 10, c.Nodes[2]) + + if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { + t.Fatalf("unexpected slices for node's index: %v", slices) + } +} + +func TestCluster_Nodes(t *testing.T) { + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} + + nodes := []*Node{node0, node1, node2} + + t.Run("NodeIDs", func(t *testing.T) { + actual := Nodes(nodes).IDs() + expected := []string{node0.ID, node1.ID, node2.ID} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Filter", func(t *testing.T) { + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("FilterURI", func(t *testing.T) { + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("Contains", func(t *testing.T) { + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) + if !reflect.DeepEqual(actualTrue, true) { + t.Errorf("expected: %v, but got: %v", true, actualTrue) + } + if !reflect.DeepEqual(actualFalse, false) { + t.Errorf("expected: %v, but got: %v", false, actualTrue) + } + }) + + t.Run("Clone", func(t *testing.T) { + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) +} + +// NEXT: move this test to internal and unexport IsCoordinator +func TestCluster_Coordinator(t *testing.T) { + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + + c1 := *NewCluster() + c1.Node = node1 + c1.Coordinator = node1.ID + c2 := *NewCluster() + c2.Node = node2 + c2.Coordinator = node1.ID + + t.Run("IsCoordinator", func(t *testing.T) { + if !c1.isCoordinator() { + t.Errorf("!IsCoordinator error: %v", c1.Node) + } else if c2.isCoordinator() { + t.Errorf("IsCoordinator error: %v", c2.Node) + } + }) +} + +func TestCluster_Topology(t *testing.T) { + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) + + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} + + t.Run("AddNode", func(t *testing.T) { + err := c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + // add the same host. + err = c1.addNode(node1) + if err != nil { + t.Fatal(err) + } + err = c1.addNode(node2) + if err != nil { + t.Fatal(err) + } + + actual := c1.nodeIDs() + expected := []string{node0.ID, node1.ID, node2.ID} + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected: %v, but got: %v", expected, actual) + } + }) + + t.Run("ContainsID", func(t *testing.T) { + if !c1.Topology.ContainsID(node1.ID) { + t.Errorf("!ContainsHost error: %v", node1.ID) + } else if c1.Topology.ContainsID(nodeinvalid.ID) { + t.Errorf("ContainsHost error: %v", nodeinvalid.ID) + } + }) +} + +// Ensure that general cluster functionality works as expected. +func TestCluster_ResizeStates(t *testing.T) { + + t.Run("Single node, no data", func(t *testing.T) { + tc := NewClusterCluster(1) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + node := tc.Clusters[0] + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{node.Node.ID}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node comes up in state NORMAL. + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Single node, not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + node := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"some-other-host"}, + } + tc.WriteTopology(node.Path, top) + + // Open TestCluster. + expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" + err := tc.Open() + if err == nil || err.Error() != expected { + t.Errorf("did not receive expected error: %s", expected) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, no data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + tc.AddNode(false) + + node0 := tc.Clusters[0] + node1 := tc.Clusters[1] + + // Ensure that nodes comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // write topology to data file + top := &Topology{ + NodeIDs: []string{"node0", "node2"}, + } + tc.WriteTopology(node0.Path, top) + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Ensure that node is in state STARTING before the other node joins. + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) + } + + // Expect an error by adding a node not in the topology. + expectedError := "host is not in topology: node1" + err := tc.AddNode(false) + if err == nil || err.Error() != expectedError { + t.Errorf("did not receive expected error: %s", expectedError) + } + + tc.AddNode(false) + node2 := tc.Clusters[2] + + // Ensure that node comes up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) + + t.Run("Multiple nodes, with data", func(t *testing.T) { + tc := NewClusterCluster(0) + tc.AddNode(false) + node0 := tc.Clusters[0] + + // Open TestCluster. + if err := tc.Open(); err != nil { + t.Fatal(err) + } + + // Add Bit Data to node0. + if err := tc.CreateField("i", "f", FieldOptions{}); err != nil { + t.Fatal(err) + } + tc.SetBit("i", "f", "standard", 1, 101, nil) + tc.SetBit("i", "f", "standard", 1, 1300000, nil) + + // Before starting the resize, get the CheckSum to use for + // comparison later. + node0Field := node0.Holder.Field("i", "f") + node0View := node0Field.View("standard") + node0Fragment := node0View.Fragment(1) + node0Checksum := node0Fragment.Checksum() + + // AddNode needs to block until the resize process has completed. + tc.AddNode(false) + node1 := tc.Clusters[1] + + // Ensure that nodes come up in state NORMAL. + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) + } + + expectedTop := &Topology{ + NodeIDs: []string{node0.Node.ID, node1.Node.ID}, + } + + // Verify topology file. + if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) + } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { + t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) + } + + // Bits + // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. + node1Field := node1.Holder.Field("i", "f") + node1View := node1Field.View("standard") + node1Fragment := node1View.Fragment(1) + + // Ensure checksums are the same. + if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { + t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) + } + + // Close TestCluster. + if err := tc.Close(); err != nil { + t.Fatal(err) + } + }) +} + +// Ensures that coordinator can be changed. +func TestCluster_UpdateCoordinator(t *testing.T) { + t.Run("UpdateCoordinator", func(t *testing.T) { + c := NewTestCluster(2) + + oldNode := c.Nodes[0] + newNode := c.Nodes[1] + + // Update coordinator to the same value. + if c.updateCoordinator(oldNode) { + t.Errorf("did not expect coordinator to change") + } else if c.Coordinator != oldNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) + } + + // Update coordinator to a new value. + if !c.updateCoordinator(newNode) { + t.Errorf("expected coordinator to change") + } else if c.Coordinator != newNode.ID { + t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) + } + }) +} diff --git a/cluster_test.go b/cluster_test.go deleted file mode 100644 index 913cd5db3..000000000 --- a/cluster_test.go +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "math/rand" - "reflect" - "testing" - "testing/quick" - - "github.com/davecgh/go-spew/spew" -) - -// Ensure the cluster can fairly distribute partitions across the nodes. -func TestCluster_Owners(t *testing.T) { - c := Cluster{ - Nodes: []*Node{ - {URI: NewTestURIFromHostPort("serverA", 1000)}, - {URI: NewTestURIFromHostPort("serverB", 1000)}, - {URI: NewTestURIFromHostPort("serverC", 1000)}, - }, - Hasher: NewTestModHasher(), - ReplicaN: 2, - } - - // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } - - // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { - t.Fatalf("unexpected owners: %s", spew.Sdump(a)) - } -} - -// Ensure the partitioner can assign a fragment to a partition. -func TestCluster_Partition(t *testing.T) { - if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := NewCluster() - c.PartitionN = partitionN - - partitionID := c.Partition(index, slice) - if partitionID < 0 || partitionID >= partitionN { - t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) - } - - return true - }, &quick.Config{ - Values: func(values []reflect.Value, rand *rand.Rand) { - values[0], _ = quick.Value(reflect.TypeOf(""), rand) - values[1] = reflect.ValueOf(uint64(rand.Uint32())) - values[2] = reflect.ValueOf(rand.Intn(1000) + 1) - }, - }); err != nil { - t.Fatal(err) - } -} - -// Ensure the hasher can hash correctly. -func TestHasher(t *testing.T) { - for _, tt := range []struct { - key uint64 - bucket []int - }{ - // Generated from the reference C++ code - {0, []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {1, []int{0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 17, 17}}, - {0xdeadbeef, []int{0, 1, 2, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16}}, - {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, - } { - for i, v := range tt.bucket { - if got := NewHasher().Hash(tt.key, i+1); got != v { - t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) - } - } - } -} - -// Ensure OwnsSlices can find the actual slice list for node and index. -func TestCluster_OwnsSlices(t *testing.T) { - c := NewTestCluster(5) - slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) - - if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -// Ensure ContainsSlices can find the actual slice list for node and index. -func TestCluster_ContainsSlices(t *testing.T) { - c := NewTestCluster(5) - c.ReplicaN = 3 - slices := c.ContainsSlices("test", 10, c.Nodes[2]) - - if !reflect.DeepEqual(slices, []uint64{0, 2, 3, 5, 6, 9, 10}) { - t.Fatalf("unexpected slices for node's index: %v", slices) - } -} - -func TestCluster_Nodes(t *testing.T) { - uri0 := NewTestURIFromHostPort("node0", 0) - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - uri3 := NewTestURIFromHostPort("node3", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - node3 := &Node{ID: "node3", URI: uri3} - - nodes := []*Node{node0, node1, node2} - - t.Run("NodeIDs", func(t *testing.T) { - actual := Nodes(nodes).IDs() - expected := []string{node0.ID, node1.ID, node2.ID} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Filter", func(t *testing.T) { - actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("FilterURI", func(t *testing.T) { - actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() - expected := []URI{uri0, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("Contains", func(t *testing.T) { - actualTrue := Nodes(nodes).Contains(node1) - actualFalse := Nodes(nodes).Contains(node3) - if !reflect.DeepEqual(actualTrue, true) { - t.Errorf("expected: %v, but got: %v", true, actualTrue) - } - if !reflect.DeepEqual(actualFalse, false) { - t.Errorf("expected: %v, but got: %v", false, actualTrue) - } - }) - - t.Run("Clone", func(t *testing.T) { - clone := Nodes(nodes).Clone() - actual := Nodes(clone).URIs() - expected := []URI{uri0, uri1, uri2} - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) -} - -func TestCluster_Coordinator(t *testing.T) { - uri1 := NewTestURIFromHostPort("node1", 0) - uri2 := NewTestURIFromHostPort("node2", 0) - - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - - c1 := *NewCluster() - c1.Node = node1 - c1.Coordinator = node1.ID - c2 := *NewCluster() - c2.Node = node2 - c2.Coordinator = node1.ID - - t.Run("IsCoordinator", func(t *testing.T) { - if !c1.IsCoordinator() { - t.Errorf("!IsCoordinator error: %v", c1.Node) - } else if c2.IsCoordinator() { - t.Errorf("IsCoordinator error: %v", c2.Node) - } - }) -} - -func TestCluster_Topology(t *testing.T) { - c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - - uri0 := NewTestURIFromHostPort("host0", 0) - uri1 := NewTestURIFromHostPort("host1", 0) - uri2 := NewTestURIFromHostPort("host2", 0) - invalid := NewTestURIFromHostPort("invalid", 0) - - node0 := &Node{ID: "node0", URI: uri0} - node1 := &Node{ID: "node1", URI: uri1} - node2 := &Node{ID: "node2", URI: uri2} - nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} - - t.Run("AddNode", func(t *testing.T) { - err := c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - // add the same host. - err = c1.AddNode(node1) - if err != nil { - t.Fatal(err) - } - err = c1.AddNode(node2) - if err != nil { - t.Fatal(err) - } - - actual := c1.NodeIDs() - expected := []string{node0.ID, node1.ID, node2.ID} - - if !reflect.DeepEqual(actual, expected) { - t.Errorf("expected: %v, but got: %v", expected, actual) - } - }) - - t.Run("ContainsID", func(t *testing.T) { - if !c1.Topology.ContainsID(node1.ID) { - t.Errorf("!ContainsHost error: %v", node1.ID) - } else if c1.Topology.ContainsID(nodeinvalid.ID) { - t.Errorf("ContainsHost error: %v", nodeinvalid.ID) - } - }) -} - -// Ensure that general cluster functionality works as expected. -func TestCluster_ResizeStates(t *testing.T) { - - t.Run("Single node, no data", func(t *testing.T) { - tc := NewClusterCluster(1) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - node := tc.Clusters[0] - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected topology: %v, but got: %v", expectedTop.NodeIDs, node.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{node.Node.ID}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node comes up in state NORMAL. - if node.State() != ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Single node, not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - node := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"some-other-host"}, - } - tc.WriteTopology(node.Path, top) - - // Open TestCluster. - expected := "considerTopology: coordinator node0 is not in topology: [some-other-host]" - err := tc.Open() - if err == nil || err.Error() != expected { - t.Errorf("did not receive expected error: %s", expected) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - tc.AddNode(false) - - node0 := tc.Clusters[0] - node1 := tc.Clusters[1] - - // Ensure that nodes comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // write topology to data file - top := &Topology{ - NodeIDs: []string{"node0", "node2"}, - } - tc.WriteTopology(node0.Path, top) - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Ensure that node is in state STARTING before the other node joins. - if node0.State() != ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) - } - - // Expect an error by adding a node not in the topology. - expectedError := "host is not in topology: node1" - err := tc.AddNode(false) - if err == nil || err.Error() != expectedError { - t.Errorf("did not receive expected error: %s", expectedError) - } - - tc.AddNode(false) - node2 := tc.Clusters[2] - - // Ensure that node comes up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node2.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) - - t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := NewClusterCluster(0) - tc.AddNode(false) - node0 := tc.Clusters[0] - - // Open TestCluster. - if err := tc.Open(); err != nil { - t.Fatal(err) - } - - // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { - t.Fatal(err) - } - tc.SetBit("i", "f", "standard", 1, 101, nil) - tc.SetBit("i", "f", "standard", 1, 1300000, nil) - - // Add Field Data to node0. - if err := tc.CreateFrame("i", "fields", FrameOptions{ - Fields: []*Field{ - { - Name: "fld0", - Type: FieldTypeInt, - Min: -100, - Max: 100, - }, - }, - }); err != nil { - t.Fatal(err) - } - tc.SetFieldValue("i", "fields", 1, "fld0", -10) - tc.SetFieldValue("i", "fields", 1, "fld0", 10) - tc.SetFieldValue("i", "fields", 1300000, "fld0", -99) - tc.SetFieldValue("i", "fields", 1300000, "fld0", 99) - - // Before starting the resize, get the CheckSum to use for - // comparison later. - node0Frame := node0.Holder.Frame("i", "f") - node0View := node0Frame.View("standard") - node0Fragment := node0View.Fragment(1) - node0Checksum := node0Fragment.Checksum() - - node0Frame = node0.Holder.Frame("i", "fields") - node0View = node0Frame.View("field_fld0") - node0Fragment = node0View.Fragment(1) - node0ChecksumFld := node0Fragment.Checksum() - - // AddNode needs to block until the resize process has completed. - tc.AddNode(false) - node1 := tc.Clusters[1] - - // Ensure that nodes come up in state NORMAL. - if node0.State() != ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) - } else if node1.State() != ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) - } - - expectedTop := &Topology{ - NodeIDs: []string{node0.Node.ID, node1.Node.ID}, - } - - // Verify topology file. - if !reflect.DeepEqual(node0.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node0 topology: %v, but got: %v", expectedTop.NodeIDs, node0.Topology.NodeIDs) - } else if !reflect.DeepEqual(node1.Topology.NodeIDs, expectedTop.NodeIDs) { - t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) - } - - // Bits - // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. - node1Frame := node1.Holder.Frame("i", "f") - node1View := node1Frame.View("standard") - node1Fragment := node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0Checksum) { - t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) - } - - // Values - // Verify that node-1 contains the fragment (i/fields/field_fld0/1) transferred from node-0. - node1Frame = node1.Holder.Frame("i", "fields") - node1View = node1Frame.View("field_fld0") - node1Fragment = node1View.Fragment(1) - - // Ensure checksums are the same. - if chksum := node1Fragment.Checksum(); !bytes.Equal(chksum, node0ChecksumFld) { - t.Fatalf("expected checksum to match: %x - %x", chksum, node0ChecksumFld) - } - - // Close TestCluster. - if err := tc.Close(); err != nil { - t.Fatal(err) - } - }) -} - -// Ensures that coordinator can be changed. -func TestCluster_UpdateCoordinator(t *testing.T) { - t.Run("UpdateCoordinator", func(t *testing.T) { - c := NewTestCluster(2) - - oldNode := c.Nodes[0] - newNode := c.Nodes[1] - - // Update coordinator to the same value. - if c.UpdateCoordinator(oldNode) { - t.Errorf("did not expect coordinator to change") - } else if c.Coordinator != oldNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, oldNode.URI) - } - - // Update coordinator to a new value. - if !c.UpdateCoordinator(newNode) { - t.Errorf("expected coordinator to change") - } else if c.Coordinator != newNode.ID { - t.Errorf("expected coordinator: %s, but got: %s", c.Coordinator, newNode.URI) - } - }) -} diff --git a/cmd/bench.go b/cmd/bench.go deleted file mode 100644 index d4b2b8580..000000000 --- a/cmd/bench.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/pilosa/pilosa/ctl" -) - -var Bencher *ctl.BenchCommand - -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Bencher = ctl.NewBenchCommand(os.Stdin, os.Stdout, os.Stderr) - benchCmd := &cobra.Command{ - Use: "bench", - Short: "Benchmark operations.", - Long: ` -Executes a benchmark for a given operation against the index. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Bencher.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := benchCmd.Flags() - flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") - flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") - flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") - flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") - ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify) - - return benchCmd -} - -func init() { - subcommandFns["bench"] = NewBenchCommand -} diff --git a/cmd/bench_test.go b/cmd/bench_test.go deleted file mode 100644 index 4b94d9392..000000000 --- a/cmd/bench_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestBenchHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "bench", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa bench") || err != nil { - t.Fatalf("Command 'bench --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestBenchConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"bench", "--operation", "set-bit"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Bencher.Host, "localhost:12345") - v.Check(cmd.Bencher.Index, "myindex") - v.Check(cmd.Bencher.Frame, "f1") - v.Check(cmd.Bencher.Op, "set-bit") - v.Check(cmd.Bencher.N, 0) - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/cmd/export.go b/cmd/export.go index 951c4a0a9..9613d9544 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -52,7 +52,7 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") - flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") + flags.StringVarP(&Exporter.Field, "field", "f", "", "Field to export") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify) diff --git a/cmd/export_test.go b/cmd/export_test.go index 2b30116c6..6f7a49b8f 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -37,13 +37,13 @@ func TestExportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") - v.Check(cmd.Exporter.Frame, "f1") + v.Check(cmd.Exporter.Field, "f1") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, diff --git a/cmd/import.go b/cmd/import.go index 8dd31181c..db5cebf8d 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The data + Long: `Bulk imports one or more CSV files to a host's index and field. The data of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -54,15 +54,14 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags := importCmd.Flags() flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.") - flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") - flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.") + flags.StringVarP(&Importer.Field, "field", "f", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") - flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") - flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") + flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Time quantum for the field") + flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Cache type for the field; valid values: none, lru, ranked") + flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Cache size for the field") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) return importCmd diff --git a/cmd/import_test.go b/cmd/import_test.go index 8117d46fa..e8e5d61f4 100644 --- a/cmd/import_test.go +++ b/cmd/import_test.go @@ -37,13 +37,13 @@ func TestImportConfig(t *testing.T) { env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" -frame = "f1" +field = "f1" `, validation: func() error { v := validator{} v.Check(cmd.Importer.Host, "localhost:12345") v.Check(cmd.Importer.Index, "myindex") - v.Check(cmd.Importer.Frame, "f1") + v.Check(cmd.Importer.Field, "f1") return v.Error() }, }, diff --git a/ctl/bench.go b/ctl/bench.go deleted file mode 100644 index 4a71169d9..000000000 --- a/ctl/bench.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "fmt" - "io" - "math/rand" - "time" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// BenchCommand represents a command for benchmarking index operations. -type BenchCommand struct { - // Destination host and port. - Host string - - // Name of the index & frame to execute against. - Index string - Frame string - - // Type of operation and number to execute. - Op string - N int - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewBenchCommand returns a new instance of BenchCommand. -func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *BenchCommand { - return &BenchCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the bench command. -func (cmd *BenchCommand) Run(ctx context.Context) error { - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - switch cmd.Op { - case "set-bit": - return cmd.runSetBit(ctx, client) - case "": - return errors.New("op required") - default: - return fmt.Errorf("unknown bench op: %q", cmd.Op) - } -} - -// runSetBit executes a benchmark of random SetBit() operations. -func (cmd *BenchCommand) runSetBit(ctx context.Context, client pilosa.InternalClient) error { - if cmd.N == 0 { - return errors.New("operation count required") - } else if cmd.Index == "" { - return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired - } - - const maxRowID = 1000 - const maxColumnID = 100000 - - startTime := time.Now() - - // Execute operation continuously. - for i := 0; i < cmd.N; i++ { - rowID := rand.Intn(maxRowID) - columnID := rand.Intn(maxColumnID) - - queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`SetBit(row=%d, frame="%s", col=%d)`, rowID, cmd.Frame, columnID), - Remote: false, - } - if _, err := client.Query(ctx, cmd.Index, queryRequest); err != nil { - return err - } - } - - // Print results. - elapsed := time.Since(startTime) - fmt.Fprintf(cmd.Stdout, "Executed %d operations in %s (%0.3f op/sec)\n", cmd.N, elapsed, float64(cmd.N)/elapsed.Seconds()) - - return nil -} - -func (cmd *BenchCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go deleted file mode 100644 index 766ec8a1a..000000000 --- a/ctl/bench_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bufio" - "bytes" - "context" - "fmt" - "io" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pkg/errors" -) - -func TestBenchCommand_InvalidOption(t *testing.T) { - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - cm := NewBenchCommand(stdin, stdout, stderr) - err := cm.Run(context.Background()) - if errors.Cause(err) != pilosa.ErrHostRequired { - t.Fatalf("Expect err: %s, actual err: %s", pilosa.ErrHostRequired, err) - } - - cm.Host = "localhost:10101" - err = cm.Run(context.Background()) - if err.Error() != "op required" { - t.Fatalf("Expect err: %s, actual err: %s", "op required", err) - } - - cm.Op = "test" - err = cm.Run(context.Background()) - if err.Error() != "unknown bench op: \"test\"" { - t.Fatalf("Expect err: %s, actual err: %s", "unknown bench op: test", err) - } - -} - -func TestBenchCommand_Run(t *testing.T) { - rder := []byte{} - stdin := bytes.NewReader(rder) - r, w, _ := os.Pipe() - - cm := NewBenchCommand(stdin, w, w) - cm.Op = "set-bit" - cm.Host = "localhost:10101" - - err := cm.Run(context.Background()) - if err.Error() != "operation count required" { - t.Fatalf("Expect error: %s, actual err: %s", "operation count required", err) - } - - cm.N = 1 - err = cm.Run(context.Background()) - if err != pilosa.ErrIndexRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrIndexRequired, err) - } - - cm.Index = "i" - err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Expect error: %s, actual err: %s", pilosa.ErrFrameRequired, err) - } - - cm.Frame = "f" - err = cm.Run(context.Background()) - w.Close() - var buf bytes.Buffer - io.Copy(&buf, r) - fmt.Println(buf.String()) - if err != nil { - fmt.Println(buf.String()) - } -} - -// declare stdin, stdout, stderr -func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { - rder := []byte{} - stdin := bytes.NewReader(rder) - stdout := bufio.NewWriter(&buf) - stderr := bufio.NewWriter(&buf) - return stdin, stdout, stderr -} diff --git a/ctl/common.go b/ctl/common.go index 11042a704..44e01d406 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -17,7 +17,7 @@ package ctl import ( "crypto/tls" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -37,7 +37,7 @@ func SetTLSConfig(flags *pflag.FlagSet, certificatePath *string, certificateKeyP } // CommandClient returns a pilosa.InternalHTTPClient for the command -func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error) { +func CommandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { tlsConfig := cmd.TLSConfiguration() var TLSConfig *tls.Config if tlsConfig.CertificatePath != "" && tlsConfig.CertificateKeyPath != "" { @@ -50,7 +50,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) + client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(TLSConfig)) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/ctl/export.go b/ctl/export.go index 5ee8d2120..090904807 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -30,9 +30,9 @@ type ExportCommand struct { // Remote host and port. Host string - // Name of the index & frame to export from. + // Name of the index & field to export from. Index string - Frame string + Field string // Filename to export to. Path string @@ -57,8 +57,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Validate arguments. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + } else if cmd.Field == "" { + return pilosa.ErrFieldRequired } // Use output file, if specified. @@ -89,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Field, slice, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/export_test.go b/ctl/export_test.go index 469ca7d75..5e87334d9 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -38,8 +38,8 @@ func TestExportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } } @@ -59,10 +59,10 @@ func TestExportCommand_Run(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(""))) cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } diff --git a/ctl/generate_config.go b/ctl/generate_config.go index a64678475..a9429ec80 100644 --- a/ctl/generate_config.go +++ b/ctl/generate_config.go @@ -18,9 +18,11 @@ import ( "context" "fmt" "io" - "strings" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pkg/errors" ) // GenerateConfigCommand represents a command for printing a default config. @@ -37,28 +39,11 @@ func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *Genera // Run prints out the default config. func (cmd *GenerateConfigCommand) Run(ctx context.Context) error { - fmt.Fprintln(cmd.Stdout, strings.TrimSpace(` -data-dir = "~/.pilosa" -bind = "localhost:10101" -max-writes-per-request = 5000 - -[cluster] - replicas = 1 - hosts = [ - "localhost:10101", - ] - -[anti-entropy] - interval = "10m0s" - -[profile] - cpu = "" - cpu-time = "30s" - -[metric] - service = "statsd" - host = "127.0.0.1:8125" - poll-interval = "0m15s" -`)+"\n") + conf := server.NewConfig() + ret, err := toml.Marshal(*conf) + if err != nil { + return errors.Wrap(err, "unmarshaling default config") + } + fmt.Fprintf(cmd.Stdout, "%s\n", ret) return nil } diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 56f392bba..26b531e8f 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -34,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), "localhost:10101") { + } else if !strings.Contains(buf.String(), ":10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import.go b/ctl/import.go index a9d70fd7c..c4d65f232 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" ) @@ -35,20 +36,17 @@ type ImportCommand struct { // Destination host and port. Host string `json:"host"` - // Name of the index & frame to import into. + // Name of the index & field to import into. Index string `json:"index"` - Frame string `json:"frame"` + Field string `json:"field"` - // Options for index & frame to be created if they don't exist + // Options for index & field to be created if they don't exist IndexOptions pilosa.IndexOptions - FrameOptions pilosa.FrameOptions + FieldOptions pilosa.FieldOptions // CreateSchema ensures the schema exists before import CreateSchema bool - // For Range-Encoded fields, name of the Field to import into. - Field string `json:"field"` - // Indicates that the payload should be treated as string keys. StringKeys bool `json:"StringKeys"` @@ -83,11 +81,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Validate arguments. - // Index and frame are validated early before the files are parsed. + // Index and field are validated early before the files are parsed. if cmd.Index == "" { return pilosa.ErrIndexRequired - } else if cmd.Frame == "" { - return pilosa.ErrFrameRequired + } else if cmd.Field == "" { + return pilosa.ErrFieldRequired } else if len(cmd.Paths) == 0 { return errors.New("path required") } @@ -105,10 +103,26 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } } + // Determine the field type in order to correctly handle the input data. + fieldType := pilosa.DefaultFieldType + schema, err := cmd.Client.Schema(ctx) + if err != nil { + return errors.Wrap(err, "getting schema") + } + for _, index := range schema { + if index.Name == cmd.Index { + for _, field := range index.Fields { + if field.Name == cmd.Field { + fieldType = field.Options.Type + } + } + } + } + // Import each path and import by slice. for _, path := range cmd.Paths { logger.Printf("parsing: %s", path) - if err := cmd.importPath(ctx, path); err != nil { + if err := cmd.importPath(ctx, fieldType, path); err != nil { return err } } @@ -121,18 +135,18 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("Error Creating Index: %s", err) } - err = cmd.Client.EnsureFrame(ctx, cmd.Index, cmd.Frame, cmd.FrameOptions) + err = cmd.Client.EnsureField(ctx, cmd.Index, cmd.Field, cmd.FieldOptions) if err != nil { - return fmt.Errorf("Error Creating Frame: %s", err) + return fmt.Errorf("Error Creating Field: %s", err) } return nil } // importPath parses a path into bits and imports it to the server. -func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { - // If a field is provided, treat the import data as values to be range-encoded. - if cmd.Field != "" { - return cmd.bufferFieldValues(ctx, path) +func (cmd *ImportCommand) importPath(ctx context.Context, fieldType, path string) error { + // If fieldType is `int`, treat the import data as values to be range-encoded. + if fieldType == pilosa.FieldTypeInt { + return cmd.bufferValues(ctx, path) } else { if cmd.StringKeys { return cmd.bufferBitsK(ctx, path) @@ -232,16 +246,16 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err // Group bits by slice. logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := pilosa.Bits(bits).GroupBySlice() + bitsBySlice := http.Bits(bits).GroupBySlice() // Parse path into bits. for slice, chunk := range bitsBySlice { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(chunk)) + sort.Sort(http.BitsByPos(chunk)) } logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil { + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Field, slice, chunk); err != nil { return errors.Wrap(err, "importing") } } @@ -338,15 +352,15 @@ func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) er // TODO: does it help to sort the rowKeys? logger.Printf("importing keys: n=%d", len(bits)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { + if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Field, bits); err != nil { return errors.Wrap(err, "importing keys") } return nil } -// bufferFieldValues buffers slices of fieldValues to be imported as a batch. -func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) error { +// bufferValues buffers slices of FieldValues to be imported as a batch. +func (cmd *ImportCommand) bufferValues(ctx context.Context, path string) error { a := make([]pilosa.FieldValue, 0, cmd.BufferSize) var r *csv.Reader @@ -394,7 +408,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } val.ColumnID = columnID - // Parse field value. + // Parse FieldValue. value, err := strconv.ParseInt(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid value on row %d: %q", rnum, record[1]) @@ -403,9 +417,9 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er a = append(a, val) - // If we've reached the buffer size then import field values. + // If we've reached the buffer size then import FieldValues. if len(a) == cmd.BufferSize { - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } a = a[:0] @@ -413,29 +427,29 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } // If there are still values in the buffer then flush them. - if err := cmd.importFieldValues(ctx, a); err != nil { + if err := cmd.importValues(ctx, a); err != nil { return err } return nil } -// importFieldValues sends batches of fieldValues to the server. -func (cmd *ImportCommand) importFieldValues(ctx context.Context, vals []pilosa.FieldValue) error { +// importValues sends batches of FieldValues to the server. +func (cmd *ImportCommand) importValues(ctx context.Context, vals []pilosa.FieldValue) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Group vals by slice. logger.Printf("grouping %d vals", len(vals)) - valsBySlice := pilosa.FieldValues(vals).GroupBySlice() + valsBySlice := http.FieldValues(vals).GroupBySlice() - // Parse path into field values. + // Parse path into FieldValues. for slice, vals := range valsBySlice { if cmd.Sort { - sort.Sort(pilosa.FieldValues(vals)) + sort.Sort(http.FieldValues(vals)) } logger.Printf("importing slice: %d, n=%d", slice, len(vals)) - if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Frame, cmd.Field, slice, vals); err != nil { + if err := cmd.Client.ImportValue(ctx, cmd.Index, cmd.Field, slice, vals); err != nil { return errors.Wrap(err, "importing values") } } diff --git a/ctl/import_test.go b/ctl/import_test.go index 08eea8323..ba8a9dc6e 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -15,6 +15,7 @@ package ctl import ( + "bufio" "bytes" "context" "io" @@ -38,11 +39,11 @@ func TestImportCommand_Validation(t *testing.T) { cm.Index = "i" err = cm.Run(context.Background()) - if err != pilosa.ErrFrameRequired { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) + if err != pilosa.ErrFieldRequired { + t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFieldRequired, err) } - cm.Frame = "f" + cm.Field = "f" err = cm.Run(context.Background()) if err.Error() != "path required" { t.Fatalf("Command not working, expect: %s, actual: '%s'", "path required", err) @@ -72,7 +73,7 @@ func TestImportCommand_Run(t *testing.T) { cm.Host = s.Host() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" cm.CreateSchema = true cm.Paths = []string{file.Name()} err = cm.Run(ctx) @@ -81,8 +82,7 @@ func TestImportCommand_Run(t *testing.T) { } } -// Ensure that the ImportValue path runs (note: we have specified a value -// for cm.Field.) +// Ensure that the ImportValue path runs. func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} @@ -106,11 +106,10 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`))) cm.Index = "i" - cm.Frame = "f" - cm.Field = "foo" + cm.Field = "f" cm.Paths = []string{file.Name()} err = cm.Run(ctx) if err != nil { @@ -120,12 +119,21 @@ func TestImportCommand_RunValue(t *testing.T) { func TestImportCommand_InvalidFile(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + s := test.NewServer() + defer s.Close() + + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder + buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = "anyhost" + cm.Host = s.Host() cm.Index = "i" - cm.Frame = "f" + cm.Field = "f" file, err := ioutil.TempFile("", "import.csv") file.Write([]byte("a,2\n3,5\n5,6")) if err != nil { @@ -180,3 +188,12 @@ func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { } return req } + +// declare stdin, stdout, stderr +func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { + rder := []byte{} + stdin := bytes.NewReader(rder) + stdout := bufio.NewWriter(&buf) + stderr := bufio.NewWriter(&buf) + return stdin, stdout, stderr +} diff --git a/ctl/server.go b/ctl/server.go index a4c272a46..ad1ac5df5 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -43,6 +43,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + // Translation + flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "URL for primary translation node for replication.") + // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") flags.StringSliceVarP(&srv.Config.Gossip.Seeds, "gossip.seeds", "", srv.Config.Gossip.Seeds, "Host with which to seed the gossip membership.") diff --git a/diagnostics.go b/diagnostics.go index 9219c01ea..e4067c6e3 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -217,7 +217,7 @@ func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { // EnrichWithSchemaProperties adds schema info to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { var numSlices uint64 - numFrames := 0 + numFields := 0 numIndexes := 0 bsiFieldCount := 0 timeQuantumEnabled := false @@ -225,19 +225,19 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { for _, index := range d.server.Holder.Indexes() { numSlices += index.MaxSlice() + 1 numIndexes += 1 - for _, frame := range index.Frames() { - numFrames += 1 - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) + for _, field := range index.Fields() { + numFields += 1 + if field.Type() == FieldTypeInt { + bsiFieldCount += 1 } - if frame.TimeQuantum() != "" { + if field.TimeQuantum() != "" { timeQuantumEnabled = true } } } d.Set("NumIndexes", numIndexes) - d.Set("NumFrames", numFrames) + d.Set("NumFields", numFields) d.Set("NumSlices", numSlices) d.Set("BSIFieldCount", bsiFieldCount) d.Set("TimeQuantumEnabled", timeQuantumEnabled) diff --git a/docs/getting-started.md b/docs/getting-started.md index c5383e81c..276b0bff9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -112,10 +112,6 @@ Note that both the user IDs and the repository IDs were remapped to sequential i #### Make Some Queries -
-

Note the Pilosa server comes with a WebUI for constructing queries in a browser. In local development, it is available at localhost:10101.

-
- Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ diff --git a/docs/installation.md b/docs/installation.md index 039ea370d..639293338 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -108,7 +107,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -144,11 +142,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` @@ -173,7 +170,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -262,7 +258,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. @@ -298,11 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende git clone https://github.com/pilosa/pilosa.git ``` -3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): +3. Build the Pilosa repo: ``` cd $GOPATH/src/github.com/pilosa/pilosa make install-build-deps - make generate-statik make install ``` @@ -327,7 +321,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. export Export data from pilosa. diff --git a/docs/webui.md b/docs/webui.md index 07acd2ab4..94377fbe8 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -9,13 +9,27 @@ nav = [ ## WebUI -The Pilosa server comes packaged with in-browser WebUI. When you run a local Pilosa server on the default host, you can access it at [localhost:10101](http://localhost:10101). +A web-based app called Pilosa WebUI is available in a separate package. This can be used for constructing queries and viewing the cluster status. -This can be used for constructing queries and viewing the cluster status. +### Installation + +Releases are [available on Github](https://github.com/pilosa/webui/releases) as well as on [Homebrew](https://brew.sh/) for Mac. + +Installing on a Mac with Homebrew is simple; just run: + +``` +brew install pilosa-webui +``` + +You may also build from source by checking out the [repo on Github](https://github.com/pilosa/webui) and running: + +``` +make install +``` ### Console -The [Console view](http://localhost:10101/#console) allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. +The Console view allows you to enter [PQL](../query-language/) queries and run them against your locally running server. First you must select an Index with the Select index dropdown. Each query's result will be displayed in the Output section along with the query time. @@ -39,4 +53,4 @@ Frame creation also supports options like `timeQuantum`. When creating a new fra ### Cluster Admin -Use the [Cluster Admin tab](http://localhost:10101/#admin) to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. +Use the Cluster Admin tab to view the current status of your cluster. This contains information on each node in the cluster, plus the list of Indexes and Frames. diff --git a/executor.go b/executor.go index 97f42acf1..a99753023 100644 --- a/executor.go +++ b/executor.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "fmt" - "net/http" "sort" "time" @@ -26,13 +25,13 @@ import ( "github.com/pkg/errors" ) -// DefaultFrame is the frame used if one is not specified. +// defaultField is the field used if one is not specified. const ( - DefaultFrame = "general" + defaultField = "general" - // MinThreshold is the lowest count to use in a Top-N operation when + // defaultMinThreshold is the lowest count to use in a Top-N operation when // looking for additional id/count pairs. - MinThreshold = 1 + defaultMinThreshold = 1 columnLabel = "col" rowLabel = "row" @@ -47,17 +46,37 @@ type Executor struct { Cluster *Cluster // Client used for remote requests. - client InternalClient + client InternalQueryClient // Maximum number of SetBit() or ClearBit() commands per request. MaxWritesPerRequest int + + // Stores key/id translation data. + TranslateStore TranslateStore +} + +// ExecutorOption is a functional option type for pilosa.Executor +type ExecutorOption func(e *Executor) error + +func OptExecutorInternalQueryClient(c InternalQueryClient) ExecutorOption { + return func(e *Executor) error { + e.client = c + return nil + } } // NewExecutor returns a new instance of Executor. -func NewExecutor(remoteClient *http.Client) *Executor { - return &Executor{ - client: NewInternalHTTPClientFromURI(nil, remoteClient), +func NewExecutor(opts ...ExecutorOption) *Executor { + e := &Executor{ + client: NewNopInternalQueryClient(), } + for _, opt := range opts { + err := opt(e) + if err != nil { + panic(err) + } + } + return e } // Execute executes a PQL query. @@ -67,6 +86,11 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic return nil, ErrIndexRequired } + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Verify that the number of writes do not exceed the maximum. if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest { return nil, ErrTooManyWrites @@ -77,6 +101,29 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic opt = &ExecOptions{} } + // Translate query keys to ids, if necessary. + for i := range q.Calls { + if err := e.translateCall(index, idx, q.Calls[i]); err != nil { + return nil, err + } + } + + results, err := e.execute(ctx, index, q, slices, opt) + if err != nil { + return nil, err + } + + // Translate response objects from ids to keys, if necessary. + for i := range results { + results[i], err = e.translateResult(index, idx, q.Calls[i], results[i]) + if err != nil { + return nil, err + } + } + return results, nil +} + +func (e *Executor) execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) { // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) @@ -127,10 +174,10 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeSum(ctx, index, c, slices, opt) case "Min": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMin(ctx, index, c, slices, opt) + return e.executeMin(ctx, index, c, slices, opt) case "Max": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeFieldMax(ctx, index, c, slices, opt) + return e.executeMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -138,8 +185,8 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeCount(ctx, index, c, slices, opt) case "SetBit": return e.executeSetBit(ctx, index, c, opt) - case "SetFieldValue": - return nil, e.executeSetFieldValue(ctx, index, c, opt) + case "SetValue": + return nil, e.executeSetValue(ctx, index, c, opt) case "SetRowAttrs": return nil, e.executeSetRowAttrs(ctx, index, c, opt) case "SetColumnAttrs": @@ -174,9 +221,7 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { // executeSum executes a Sum() call. func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Sum(): frame required") - } else if field := c.Args["field"]; field == "" { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Sum(): field required") } @@ -207,11 +252,9 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other, nil } -// executeFieldMin executes a Min() call. -func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Min(): frame required") - } else if field := c.Args["field"]; field == "" { +// executeMin executes a Min() call. +func (e *Executor) executeMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Min(): field required") } @@ -221,7 +264,7 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMinSlice(ctx, index, c, slice) + return e.executeMinSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -242,11 +285,9 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal return other, nil } -// executeFieldMax executes a Max() call. -func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { - if frame := c.Args["frame"]; frame == "" { - return ValCount{}, errors.New("Max(): frame required") - } else if field := c.Args["field"]; field == "" { +// executeMax executes a Max() call. +func (e *Executor) executeMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { + if field := c.Args["field"]; field == "" { return ValCount{}, errors.New("Max(): field required") } @@ -256,7 +297,7 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal // Execute calls in bulk on each remote node and merge. mapFn := func(slice uint64) (interface{}, error) { - return e.executeFieldMaxSlice(ctx, index, c, slice) + return e.executeMaxSlice(ctx, index, c, slice) } // Merge returned results at coordinating node. @@ -318,8 +359,8 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } else if err != nil { return nil, err } else { - frame, _ := c.Args["frame"].(string) - if fr := idx.Frame(frame); fr != nil { + field, _ := c.Args["field"].(string) + if fr := idx.Field(field); fr != nil { rowID, _, err := c.UintArg(rowLabel) if err != nil { return nil, errors.Wrap(err, "getting row") @@ -362,7 +403,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice calculates the sum and count for fields on a slice. +// executeSumCountSlice calculates the sum and count for bsiGroups on a slice. func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { @@ -373,36 +414,35 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } - vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.sum(filter, bsig.BitDepth()) if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } return ValCount{ - Val: int64(vsum) + (int64(vcount) * field.Min), + Val: int64(vsum) + (int64(vcount) * bsig.Min), Count: int64(vcount), }, nil } -// executeFieldMinSlice calculates the min for fields on a slice. -func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMinSlice calculates the min for bsiGroups on a slice. +func (e *Executor) executeMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) @@ -412,36 +452,35 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } - fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + fmin, fcount, err := fragment.min(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmin) + field.Min, + Val: int64(fmin) + bsig.Min, Count: int64(fcount), }, nil } -// executeFieldMaxSlice calculates the max for fields on a slice. -func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { +// executeMaxSlice calculates the max for bsiGroups on a slice. +func (e *Executor) executeMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) @@ -451,30 +490,29 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq filter = row } - frameName, _ := c.Args["frame"].(string) fieldName, _ := c.Args["field"].(string) - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ValCount{}, nil - } - - field := frame.Field(fieldName) + field := e.Holder.Field(index, fieldName) if field == nil { return ValCount{}, nil } - fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + bsig := field.bsiGroup(fieldName) + if bsig == nil { + return ValCount{}, nil + } + + fragment := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if fragment == nil { return ValCount{}, nil } - fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + fmax, fcount, err := fragment.max(filter, bsig.BitDepth()) if err != nil { return ValCount{}, err } return ValCount{ - Val: int64(fmax) + field.Min, + Val: int64(fmax) + bsig.Min, Count: int64(fcount), }, nil } @@ -547,12 +585,12 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { - frame, _ := c.Args["frame"].(string) + field, _ := c.Args["field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } - field, _ := c.Args["field"].(string) + attrName, _ := c.Args["attrName"].(string) rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -561,7 +599,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } - filters, _ := c.Args["filters"].([]interface{}) + attrValues, _ := c.Args["attrValues"].([]interface{}) tanimotoThreshold, _, err := c.UintArg("tanimotoThreshold") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -579,29 +617,29 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca return nil, errors.New("TopN() can only have one input bitmap") } - // Set default frame. - if frame == "" { - frame = DefaultFrame + // Set default field. + if field == "" { + field = defaultField } - f := e.Holder.Fragment(index, frame, ViewStandard, slice) + f := e.Holder.Fragment(index, field, ViewStandard, slice) if f == nil { return nil, nil } if minThreshold <= 0 { - minThreshold = MinThreshold + minThreshold = defaultMinThreshold } if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.Top(TopOptions{ + return f.top(TopOptions{ N: int(n), Src: src, RowIDs: rowIDs, - FilterField: field, - FilterValues: filters, + FilterName: attrName, + FilterValues: attrValues, MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, }) @@ -636,14 +674,14 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, ErrIndexNotFound } - // Fetch frame & row label based on argument. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Fetch field & row label based on argument. + field, _ := c.Args["field"].(string) + if field == "" { + field = defaultField } - f := e.Holder.Frame(index, frame) + f := e.Holder.Field(index, field) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } rowID, rowOK, rowErr := c.UintArg(rowLabel) @@ -654,11 +692,11 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) } - frag := e.Holder.Fragment(index, frame, ViewStandard, slice) + frag := e.Holder.Fragment(index, field, ViewStandard, slice) if frag == nil { return NewRow(), nil } - return frag.Row(rowID), nil + return frag.row(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -685,15 +723,15 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *p // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Handle field ranges differently. + // Handle bsiGroup ranges differently. if c.HasConditionArg() { - return e.executeFieldRangeSlice(ctx, index, c, slice) + return e.executeBSIGroupRangeSlice(ctx, index, c, slice) } - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame + // Parse field, use default if unset. + field, _ := c.Args["field"].(string) + if field == "" { + field = defaultField } // Retrieve column label. @@ -702,10 +740,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return nil, ErrIndexNotFound } - // Retrieve base frame. - f := idx.Frame(frame) + // Retrieve base field. + f := idx.Field(field) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } // Read row & column id. @@ -743,46 +781,32 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C return &Row{}, nil } - // Union bitmaps across all time-based subframes. + // Union bitmaps across all time-based views. row := &Row{} - for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { - f := e.Holder.Fragment(index, frame, view, slice) + for _, view := range viewsByTimeRange(ViewStandard, startTime, endTime, q) { + f := e.Holder.Fragment(index, field, view, slice) if f == nil { continue } - row = row.Union(f.Row(rowID)) + row = row.Union(f.row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil } -// executeFieldRangeSlice executes a range(field) call for a local slice. -func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - f := e.Holder.Frame(index, frame) - if f == nil { - return nil, ErrFrameNotFound - } - - // Remove frame field. - args := pql.CopyArgs(c.Args) - delete(args, "frame") - - // Only one conditional field should remain. - if len(args) == 0 { +// executeBSIGroupRangeSlice executes a range(bsiGroup) call for a local slice. +func (e *Executor) executeBSIGroupRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Row, error) { + // Only one conditional should be present. + if len(c.Args) == 0 { return nil, errors.New("Range(): condition required") - } else if len(args) > 1 { + } else if len(c.Args) > 1 { return nil, errors.New("Range(): too many arguments") } - // Extract condition field. + // Extract conditional. var fieldName string var cond *pql.Condition - for k, v := range args { + for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) @@ -790,28 +814,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * fieldName, cond = k, vv } - // EQ null (not implemented: flip frag.FieldNotNull with max ColumnID) - // NEQ null frag.FieldNotNull() - // BETWEEN a,b(in) BETWEEN/frag.FieldRangeBetween() - // BETWEEN a,b(out) BETWEEN/frag.FieldNotNull() - // EQ frag.FieldRange - // NEQ frag.FieldRange + f := e.Holder.Field(index, fieldName) + if f == nil { + return nil, ErrFieldNotFound + } + + // EQ null (not implemented: flip frag.NotNull with max ColumnID) + // NEQ null frag.NotNull() + // BETWEEN a,b(in) BETWEEN/frag.RangeBetween() + // BETWEEN a,b(out) BETWEEN/frag.NotNull() + // EQ frag.RangeOp + // NEQ frag.RangeOp // Handle `!= null`. if cond.Op == pql.NEQ && cond.Value == nil { - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } - return frag.FieldNotNull(field.BitDepth()) + return frag.notNull(bsig.BitDepth()) } else if cond.Op == pql.BETWEEN { @@ -826,33 +855,33 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * } // The reason we don't just call: - // return f.FieldRangeBetween(fieldName, predicates[0], predicates[1]) + // return f.RangeBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be slice-specific. - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicates[0], predicates[1]) + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicates[0], predicates[1]) if outOfRange { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } // If the query is asking for the entire valid range, just return - // the not-null bitmap for the field. - if predicates[0] <= field.Min && predicates[1] >= field.Max { - return frag.FieldNotNull(field.BitDepth()) + // the not-null bitmap for the bsiGroup. + if predicates[0] <= bsig.Min && predicates[1] >= bsig.Max { + return frag.notNull(bsig.BitDepth()) } - return frag.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) + return frag.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) } else { @@ -862,36 +891,36 @@ func (e *Executor) executeFieldRangeSlice(ctx context.Context, index string, c * return nil, errors.New("Range(): conditions only support integer values") } - // Find field. - field := f.Field(fieldName) - if field == nil { - return nil, ErrFieldNotFound + // Find bsiGroup. + bsig := f.bsiGroup(fieldName) + if bsig == nil { + return nil, ErrBSIGroupNotFound } - baseValue, outOfRange := field.BaseValue(cond.Op, value) + baseValue, outOfRange := bsig.baseValue(cond.Op, value) if outOfRange && cond.Op != pql.NEQ { return NewRow(), nil } // Retrieve fragment. - frag := e.Holder.Fragment(index, frame, ViewFieldPrefix+fieldName, slice) + frag := e.Holder.Fragment(index, fieldName, viewBSIGroupPrefix+fieldName, slice) if frag == nil { return NewRow(), nil } - // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid field range. - if (cond.Op == pql.LT && value > field.Max) || (cond.Op == pql.LTE && value >= field.Max) || - (cond.Op == pql.GT && value < field.Min) || (cond.Op == pql.GTE && value <= field.Min) { - return frag.FieldNotNull(field.BitDepth()) + // LT[E] and GT[E] should return all not-null if selected range fully encompasses valid bsiGroup range. + if (cond.Op == pql.LT && value > bsig.Max) || (cond.Op == pql.LTE && value >= bsig.Max) || + (cond.Op == pql.GT && value < bsig.Min) || (cond.Op == pql.GTE && value <= bsig.Min) { + return frag.notNull(bsig.BitDepth()) } // outOfRange for NEQ should return all not-null. if outOfRange && cond.Op == pql.NEQ { - return frag.FieldNotNull(field.BitDepth()) + return frag.notNull(bsig.BitDepth()) } - f.Stats.Count("range:field", 1, 1.0) - return frag.FieldRange(cond.Op, field.BitDepth(), baseValue) + f.Stats.Count("range:bsigroup", 1, 1.0) + return frag.rangeOp(cond.Op, bsig.BitDepth(), baseValue) } } @@ -967,19 +996,19 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, // executeClearBit executes a ClearBit() call. func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return false, errors.New("ClearBit() frame required") + return false, errors.New("ClearBit() field required") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(field) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } // Read fields using labels. @@ -1001,10 +1030,10 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // executeClearBitView executes a ClearBit() call for a single view. -func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { +func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) @@ -1032,19 +1061,19 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql // executeSetBit executes a SetBit() call. func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return false, errors.New("SetBit() field required: frame") + return false, errors.New("SetBit() field required: field") } - // Retrieve frame. + // Retrieve field. idx := e.Holder.Index(index) if idx == nil { return false, ErrIndexNotFound } - f := idx.Frame(frame) + f := idx.Field(field) if f == nil { - return false, ErrFrameNotFound + return false, ErrFieldNotFound } // Read fields using labels. @@ -1076,11 +1105,11 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // executeSetBitView executes a SetBit() call for a specific view. -func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { +func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Field, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -1107,46 +1136,40 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C return ret, nil } -// executeSetFieldValue executes a SetFieldValue() call. -func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) - if !ok { - return errors.New("SetFieldValue() frame required") - } - - // Retrieve frame. - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ErrFrameNotFound - } - +// executeSetValue executes a SetValue() call. +func (e *Executor) executeSetValue(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { // Parse labels. columnID, ok, err := c.UintArg(columnLabel) if err != nil { - return fmt.Errorf("reading SetFieldValue() column: %v", err) + return fmt.Errorf("reading SetValue() column: %v", err) } else if !ok { - return fmt.Errorf("SetFieldValue() column field '%v' required", columnLabel) + return fmt.Errorf("SetValue() column field '%v' required", columnLabel) } // Copy args and remove reserved fields. args := pql.CopyArgs(c.Args) - delete(args, "frame") - // While frame could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. - // Also, if we ever need to make ColumnAttrs frame-specific, then having this reserved word prevents backward incompatibility. + // While field could technically work as a ColumnAttr argument, we are treating it as a reserved word primarily to avoid confusion. + // Also, if we ever need to make ColumnAttrs field-specific, then having this reserved word prevents backward incompatibility. delete(args, columnLabel) // Set values. for name, value := range args { + // Retrieve field. + field := e.Holder.Field(index, name) + if field == nil { + return ErrFieldNotFound + } + switch value := value.(type) { case int64: - if _, err := frame.SetFieldValue(columnID, name, value); err != nil { + if _, err := field.SetValue(columnID, value); err != nil { return err } default: - return ErrInvalidFieldValueType + return ErrInvalidBSIGroupValueType } + field.Stats.Count("SetValue", 1, 1.0) } - frame.Stats.Count("SetFieldValue", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1175,15 +1198,15 @@ func (e *Executor) executeSetFieldValue(ctx context.Context, index string, c *pq // executeSetRowAttrs executes a SetRowAttrs() call. func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error { - frameName, ok := c.Args["frame"].(string) + fieldName, ok := c.Args["field"].(string) if !ok { - return errors.New("SetRowAttrs() frame required") + return errors.New("SetRowAttrs() field required") } - // Retrieve frame. - frame := e.Holder.Frame(index, frameName) - if frame == nil { - return ErrFrameNotFound + // Retrieve field. + field := e.Holder.Field(index, fieldName) + if field == nil { + return ErrFieldNotFound } // Parse labels. @@ -1196,14 +1219,14 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") + delete(attrs, "field") delete(attrs, rowLabel) // Set attributes. - if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { + if err := field.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1232,18 +1255,18 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. // executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { - // Collect attributes by frame/id. + // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { - frame, ok := c.Args["frame"].(string) + field, ok := c.Args["field"].(string) if !ok { - return nil, errors.New("SetRowAttrs() frame required") + return nil, errors.New("SetRowAttrs() field required") } - // Retrieve frame. - f := e.Holder.Frame(index, frame) + // Retrieve field. + f := e.Holder.Field(index, field) if f == nil { - return nil, ErrFrameNotFound + return nil, ErrFieldNotFound } rowID, ok, err := c.UintArg(rowLabel) @@ -1255,20 +1278,20 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) - delete(attrs, "frame") + delete(attrs, "field") delete(attrs, rowLabel) - // Create frame group, if not exists. - frameMap := m[frame] - if frameMap == nil { - frameMap = make(map[uint64]map[string]interface{}) - m[frame] = frameMap + // Create field group, if not exists. + fieldMap := m[field] + if fieldMap == nil { + fieldMap = make(map[uint64]map[string]interface{}) + m[field] = fieldMap } // Set or merge attributes. - attr := frameMap[rowID] + attr := fieldMap[rowID] if attr == nil { - frameMap[rowID] = cloneAttrs(attrs) + fieldMap[rowID] = cloneAttrs(attrs) } else { for k, v := range attrs { attr[k] = v @@ -1276,19 +1299,19 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal } } - // Bulk insert attributes by frame. - for name, frameMap := range m { - // Retrieve frame. - frame := e.Holder.Frame(index, name) - if frame == nil { - return nil, ErrFrameNotFound + // Bulk insert attributes by field. + for name, fieldMap := range m { + // Retrieve field. + field := e.Holder.Field(index, name) + if field == nil { + return nil, ErrFieldNotFound } // Set attributes. - if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { + if err := field.RowAttrStore().SetBulkAttrs(fieldMap); err != nil { return nil, err } - frame.Stats.Count("SetRowAttrs", 1, 1.0) + field.Stats.Count("SetRowAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1333,7 +1356,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *p // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, columnLabel) - delete(attrs, "frame") + delete(attrs, "field") // Set attributes. if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil { @@ -1404,7 +1427,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * case "SetRowAttrs": case "SetColumnAttrs": default: - v, err = decodeRow(pb.Results[i].GetRow()), nil + v, err = DecodeRow(pb.Results[i].GetRow()), nil } if err != nil { return nil, err @@ -1422,7 +1445,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.SliceNodes(index, slice) { + for _, node := range e.Cluster.sliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop @@ -1452,7 +1475,7 @@ func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, if !opt.Remote { nodes = Nodes(e.Cluster.Nodes).Clone() } else { - nodes = []*Node{e.Cluster.nodeByID(e.Node.ID)} + nodes = []*Node{e.Cluster.unprotectedNodeByID(e.Node.ID)} } // Start mapping across all primary owners. @@ -1567,6 +1590,78 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } } +func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error { + // Translate column key. + if idx.Keys() { + if value := callArgString(c, "col"); value != "" { + ids, err := e.TranslateStore.TranslateColumnsToUint64(index, []string{value}) + if err != nil { + return err + } + c.Args["col"] = ids[0] + } + } + + // Translate row key, if field is specified & key exists. + if fieldName := callArgString(c, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + if value := callArgString(c, "row"); value != "" { + ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value}) + if err != nil { + return err + } + c.Args["row"] = ids[0] + } + } + } + + // Translate child calls. + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return err + } + } + + return nil +} + +func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { + switch result := result.(type) { + case *Row: + if idx.Keys() { + other := &Row{Attrs: result.Attrs} + for _, segment := range result.Segments() { + for _, col := range segment.Columns() { + key, err := e.TranslateStore.TranslateColumnToString(index, col) + if err != nil { + return nil, err + } + other.Keys = append(other.Keys, key) + } + } + return other, nil + } + + case []Pair: + if fieldName := callArgString(call, "field"); fieldName != "" { + field := idx.Field(fieldName) + if field.Keys() { + other := make([]Pair, len(result)) + for i := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, result[i].ID) + if err != nil { + return nil, err + } + other[i] = Pair{Key: key, Count: result[i].Count} + } + return other, nil + } + } + } + return result, nil +} + // errSliceUnavailable is a marker error if no nodes are available. var errSliceUnavailable = errors.New("slice unavailable") @@ -1643,7 +1738,7 @@ func (vc *ValCount) Add(other ValCount) ValCount { } } -func encodeValCount(vc ValCount) *internal.ValCount { +func EncodeValCount(vc ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, Count: vc.Count, @@ -1678,3 +1773,12 @@ func (vc *ValCount) Larger(other ValCount) ValCount { Count: vc.Count, } } + +func callArgString(call *pql.Call, key string) string { + value, ok := call.Args[key] + if !ok { + return "" + } + s, _ := value.(string) + return s +} diff --git a/executor_test.go b/executor_test.go index 0807228ac..164133ecb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -22,6 +22,8 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/test" @@ -33,18 +35,18 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) + f, err := index.CreateField("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } @@ -52,7 +54,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) @@ -61,7 +63,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit column attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) @@ -70,7 +72,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhibit row attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) @@ -83,7 +85,30 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + + // Set bits. + if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(field=f, row=%d, col=%d)\n", 20, SliceWidth+1), + ), nil, nil); err != nil { + t.Fatal(err) + } + if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + t.Fatal(err) + } + }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + if _, err := index.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { t.Fatal(err) } @@ -91,14 +116,20 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 20, SliceWidth+1), + `SetBit(field=f, row="bar", col="foo")`+"\n"+ + `SetBit(field=f, row="baz", col="foo")`+"\n"+ + `SetBit(field=f, row="bar", col="bat")`+"\n"+ + `SetBit(field=f, row="bbb", col="aaa")`+"\n", ), nil, nil); err != nil { t.Fatal(err) } - if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + + if results, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row="bar", field=f)`), nil, nil); err != nil { t.Fatal(err) + } else if diff := cmp.Diff(results, []interface{}{ + &pilosa.Row{Keys: []string{"foo", "bat"}, Attrs: map[string]interface{}{}}, + }, cmpopts.IgnoreUnexported(pilosa.Row{})); diff != "" { + t.Fatal(diff) } }) } @@ -107,13 +138,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, 2) + hldr.SetBit("i", "general", 10, 3) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, 4) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { @@ -125,9 +156,9 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.SetBit("i", "general", 10, 1) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } @@ -137,15 +168,15 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { @@ -158,7 +189,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } @@ -168,14 +199,14 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { @@ -187,9 +218,9 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.SetBit("i", "general", 10, 0) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { @@ -201,14 +232,14 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, SliceWidth+1) + hldr.SetBit("i", "general", 10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { @@ -220,12 +251,12 @@ func TestExecutor_Execute_Xor(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.SetBit("i", "f", 10, 3) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 10, SliceWidth+2) - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(3) { t.Fatalf("unexpected n: %d", res[0]) @@ -237,13 +268,15 @@ func TestExecutor_Execute_SetBit(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if n := f.Row(11).Count(); n != 0 { + // set a bit so the view gets created. + hldr.SetBit("i", "f", 1, 0) + + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -251,10 +284,10 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := f.Row(11).Count(); n != 1 { + if n := hldr.Row("i", "f", 11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, frame=f, col=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=11, field=f, col=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -263,35 +296,34 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } -// Ensure a SetFieldValue() query can be executed. -func TestExecutor_Execute_SetFieldValue(t *testing.T) { +// Ensure a SetValue() query can be executed. +func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Create frames. + // Create felds. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, - }, + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 50, }); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Set field values. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0=25, field1=2)`), nil, nil); err != nil { + // Set bsiGroup values. + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f=25)`), nil, nil); err != nil { t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=100, frame=f, field0=10)`), nil, nil); err != nil { + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=100, f=10)`), nil, nil); err != nil { t.Fatal(err) } - f := hldr.Frame("i", "f") - if value, exists, err := f.FieldValue(10, "field0"); err != nil { + f := hldr.Field("i", "f") + if value, exists, err := f.Value(10); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -299,15 +331,7 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { t.Fatalf("unexpected value: %v", value) } - if value, exists, err := f.FieldValue(10, "field1"); err != nil { - t.Fatal(err) - } else if !exists { - t.Fatal("expected value to exist") - } else if value != 2 { - t.Fatalf("unexpected value: %v", value) - } - - if value, exists, err := f.FieldValue(100, "field0"); err != nil { + if value, exists, err := f.Value(100); err != nil { t.Fatal(err) } else if !exists { t.Fatal("expected value to exist") @@ -320,38 +344,31 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, - }, + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 100, }); err != nil { t.Fatal(err) } - t.Run("ErrFrameRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() frame required` { + t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name=10, f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrColumnFieldRequired", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name=10, frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { + t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(invalid_column_name="bad_column", f=100)`), nil, nil); err == nil || err.Error() != `SetValue() column field 'col' required` { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrColumnFieldValue", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(invalid_column_name="bad_column", frame=f, field0=100)`), nil, nil); err == nil || err.Error() != `SetFieldValue() column field 'col' required` { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrInvalidFieldValueType", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetFieldValue(col=10, frame=f, field0="hello")`), nil, nil); err == nil || err.Error() != `invalid field value type` { + t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetValue(col=10, f="hello")`), nil, nil); err == nil || err != pilosa.ErrInvalidBSIGroupValueType { t.Fatalf("unexpected error: %s", err) } }) @@ -363,31 +380,31 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Create frames. + // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + } else if _, err := index.CreateFieldIfNotExists("xxx", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Set two fields on f/10. - // Also set fields on other bitmaps and frames to test isolation. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + // Set two attrs on f/10. + // Also set attrs on other bitmaps and fields to test isolation. + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=200, field=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(row=10, field=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } - f := hldr.Frame("i", "f") + f := hldr.Field("i", "f") if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { @@ -397,37 +414,37 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { // Ensure a TopN() query can be executed. func TestExecutor_Execute_TopN(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + t.Run("ID", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // Set columns for rows 0, 10, & 20 across two slices. - if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=1) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) - SetBit(frame=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetBit(frame=f, row=10, col=0) - SetBit(frame=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) - SetBit(frame=other, row=0, col=0) - `), nil, nil); err != nil { - t.Fatal(err) - } + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=1) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=f, row=0, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetBit(field=f, row=10, col=0) + SetBit(field=f, row=10, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=f, row=20, col=`+strconv.Itoa(SliceWidth)+`) + SetBit(field=other, row=0, col=0) + `), nil, nil); err != nil { + t.Fatal(err) + } - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() - t.Run("Standard", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ {ID: 0, Count: 5}, @@ -436,6 +453,46 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) + + t.Run("Keys", func(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + // Set columns for rows 0, 10, & 20 across two slices. + if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("f", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := idx.CreateField("other", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(field=f, row="foo", col="a") + SetBit(field=f, row="foo", col="b") + SetBit(field=f, row="foo", col="c") + SetBit(field=f, row="foo", col="d") + SetBit(field=f, row="foo", col="e") + SetBit(field=f, row="bar", col="a") + SetBit(field=f, row="bar", col="b") + SetBit(field=f, row="baz", col="b") + SetBit(field=other, row="foo", col="a") + `), nil, nil); err != nil { + t.Fatal(err) + } + + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() + + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(result, []interface{}{ + []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + }); diff != "" { + t.Fatal(diff) + } + }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -443,16 +500,16 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, 2) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 1, SliceWidth+2) + hldr.SetBit("i", "f", 1, SliceWidth) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 4}, @@ -466,27 +523,27 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 0, 2*SliceWidth) + hldr.SetBit("i", "f", 0, 3*SliceWidth) + hldr.SetBit("i", "f", 0, 4*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1) + hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1) + hldr.SetBit("i", "f", 2, SliceWidth) + hldr.SetBit("i", "f", 2, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1) + hldr.SetBit("i", "f", 3, 2*SliceWidth) + hldr.SetBit("i", "f", 3, 2*SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1) + hldr.SetBit("i", "f", 4, 3*SliceWidth) + hldr.SetBit("i", "f", 4, 3*SliceWidth+1) // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=1)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -501,27 +558,27 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer hldr.Close() // Set columns for rows 0, 10, & 20 across two slices. - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 0, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth) + hldr.SetBit("i", "f", 10, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth) + hldr.SetBit("i", "f", 20, SliceWidth+1) + hldr.SetBit("i", "f", 20, SliceWidth+2) // Create an intersecting row. - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) + hldr.SetBit("i", "other", 100, SliceWidth) + hldr.SetBit("i", "other", 100, SliceWidth+1) + hldr.SetBit("i", "other", 100, SliceWidth+2) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache() // Execute query. - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=100, field=other), field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 20, Count: 3}, @@ -537,15 +594,15 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -560,15 +617,15 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { // hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) + hldr.SetBit("i", "f", 0, 0) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", 10, SliceWidth) - if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil { + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(row=10,field=f),field="f", n=1, attrName="category", attrValues=[123])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{ {ID: 10, Count: 1}, @@ -581,36 +638,40 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { func TestExecutor_Execute_MinMax(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, - }, + if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: -10, + Max: 100, }); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=3) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetBit(frame=f, row=1, col=1) - SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + SetBit(field=x, row=0, col=0) + SetBit(field=x, row=0, col=3) + SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=x, row=1, col=1) + SetBit(field=x, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, col=0) - SetFieldValue(frame=f, foo=-5, col=1) - SetFieldValue(frame=f, foo=-5, col=2) - SetFieldValue(frame=f, foo=10, col=3) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(f=20, col=0) + SetValue(f=-5, col=1) + SetValue(f=-5, col=2) + SetValue(f=10, col=3) + SetValue(f=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(f=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(f=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(f=60, col=`+strconv.Itoa(SliceWidth+1)+`) `), nil, nil); err != nil { t.Fatal(err) } @@ -622,16 +683,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: -5, cnt: 2}, - {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(field=x, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Min(frame=f, field=foo)` + pql = `Min(field=f)` } else { - pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -648,16 +709,16 @@ func TestExecutor_Execute_MinMax(t *testing.T) { cnt int64 }{ {filter: ``, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, - {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, - {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + {filter: `Bitmap(field=x, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(field=x, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(field=x, row=2)`, exp: 40, cnt: 1}, } for i, tt := range tests { var pql string if tt.filter == "" { - pql = `Max(frame=f, field=foo)` + pql = `Max(field=f)` } else { - pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) @@ -672,46 +733,58 @@ func TestExecutor_Execute_MinMax(t *testing.T) { func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, + if _, err := idx.CreateField("x", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 10, + Max: 100, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, + if _, err := idx.CreateField("bar", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 100000, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 1000, }); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=x, row=0, col=0) + SetBit(field=x, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, col=0) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) + SetValue(foo=20, col=0) + SetValue(bar=2000, col=0) + SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(other=1000, col=0) `), nil, nil); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -719,7 +792,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(field=x, row=0), field=foo)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -728,16 +801,17 @@ func TestExecutor_Execute_Sum(t *testing.T) { } // Ensure a range query can be executed. -func TestExecutor_Execute_Range(t *testing.T) { +func TestExecutor_Execute_BSIGroupRange(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) // Create index. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { t.Fatal(err) @@ -745,84 +819,95 @@ func TestExecutor_Execute_Range(t *testing.T) { // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") - SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") - SetBit(frame=f, row=1, col=4, timestamp="2000-01-02T00:00") - SetBit(frame=f, row=1, col=5, timestamp="2000-02-01T00:00") - SetBit(frame=f, row=1, col=6, timestamp="2001-01-01T00:00") - SetBit(frame=f, row=1, col=7, timestamp="2002-01-01T02:00") + SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00") + SetBit(field=f, row=1, col=3, timestamp="2000-01-01T00:00") + SetBit(field=f, row=1, col=4, timestamp="2000-01-02T00:00") + SetBit(field=f, row=1, col=5, timestamp="2000-02-01T00:00") + SetBit(field=f, row=1, col=6, timestamp="2001-01-01T00:00") + SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00") - SetBit(frame=f, row=1, col=2, timestamp="1999-12-30T00:00") - SetBit(frame=f, row=1, col=2, timestamp="2002-02-01T00:00") - SetBit(frame=f, row=10, col=2, timestamp="2001-01-01T00:00") + SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00") + SetBit(field=f, row=1, col=2, timestamp="2002-02-01T00:00") + SetBit(field=f, row=10, col=2, timestamp="2001-01-01T00:00") `), nil, nil); err != nil { t.Fatal(err) } t.Run("Standard", func(t *testing.T) { - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) - } -// Ensure a Range(field) query can be executed. -func TestExecutor_Execute_FieldRange(t *testing.T) { +// Ensure a Range(bsiGroup) query can be executed. +func TestExecutor_Execute_Range(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, - {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, - }, + if _, err := idx.CreateField("f", pilosa.FieldOptions{}); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 10, + Max: 100, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, - }, + if _, err := idx.CreateField("bar", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 100000, }); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, - }, + if _, err := idx.CreateField("other", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 1000, + }); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("edge", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: -100, + Max: 100, }); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "i", test.MustParse(` - SetBit(frame=f, row=0, col=0) - SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(field=f, row=0, col=0) + SetBit(field=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=f, foo=20, bar=2000, col=50) - SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) - SetFieldValue(frame=f, foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) - SetFieldValue(frame=f, foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) - SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) - SetFieldValue(frame=other, foo=1000, col=0) - SetFieldValue(frame=edge, foo=100, col=0) - SetFieldValue(frame=edge, foo=-100, col=1) + SetValue(foo=20, col=50) + SetValue(bar=2000, col=50) + SetValue(foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetValue(foo=10, col=`+strconv.Itoa(SliceWidth+2)+`) + SetValue(foo=20, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetValue(foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + SetValue(other=1000, col=0) + SetValue(edge=100, col=0) + SetValue(edge=-100, col=1) `), nil, nil); err != nil { t.Fatal(err) } t.Run("EQ", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -831,19 +916,19 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != null)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo != 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other != -20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -852,7 +937,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo < 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -860,7 +945,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo <= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -868,7 +953,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo > 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -876,7 +961,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GTE", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo >= 20)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -884,16 +969,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [1, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) - // Ensure that the FieldNotNull code path gets run. - t.Run("FieldNotNull", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { + // Ensure that the NotNull code path gets run. + t.Run("NotNull", func(t *testing.T) { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(other >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -901,7 +986,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("BelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 0)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -909,7 +994,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("AboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(foo == 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -917,7 +1002,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge < 200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) @@ -925,21 +1010,15 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil { + if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(edge > -200)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) } }) - t.Run("ErrFrameNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=bad_frame, foo >= 20)`), nil, nil); err != pilosa.ErrFrameNotFound { - t.Fatal(err) - } - }) - t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`Range(bad_field >= 20)`), nil, nil); err != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -947,7 +1026,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { // Ensure a remote query can return a row. func TestExecutor_Execute_Remote_Row(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -963,7 +1042,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "i" { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `Bitmap(frame="f", row=10)` { + } else if query.String() != `Bitmap(field="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } else if !reflect.DeepEqual(slices, []uint64{1}) { t.Fatalf("unexpected slices: %+v", slices) @@ -983,10 +1062,10 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.SetBit("i", "f", 10, SliceWidth+1) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, field=f)`), nil, nil); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { t.Fatalf("unexpected columns: %+v", columns) @@ -995,7 +1074,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { // Ensure a remote query can return a count. func TestExecutor_Execute_Remote_Count(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1017,11 +1096,11 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, field=f))`), nil, nil); err != nil { t.Fatal(err) } else if res[0] != uint64(12) { t.Fatalf("unexpected n: %d", res[0]) @@ -1030,7 +1109,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1049,7 +1128,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10)` { + } else if query.String() != `SetBit(col=2, field="f", row=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1061,18 +1140,18 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { defer hldr.Close() s.Handler.API.Holder = hldr.Holder - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + // Create field. + if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2)`), nil, nil); err != nil { t.Fatal(err) } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { + if n := hldr.Row("i", "f", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1082,7 +1161,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) c.ReplicaN = 2 // Create secondary server and update second cluster node. @@ -1101,7 +1180,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != `i` { t.Fatalf("unexpected index: %s", index) - } else if query.String() != `SetBit(col=2, frame="f", row=10, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(col=2, field="f", row=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -1113,20 +1192,20 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { defer hldr.Close() s.Handler.API.Holder = hldr.Holder - // Create frame. - if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + // Create field. + if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateField("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := f.SetTimeQuantum("Y"); err != nil { t.Fatal(err) } e := test.NewExecutor(hldr.Holder, c) - if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, frame=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(row=10, field=f, col=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } // Verify that one column is set on both node's holder. - if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { + if n := hldr.ViewRow("i", "f", "standard_2016", 10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -1136,7 +1215,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Ensure a remote query can return a top-n query. func TestExecutor_Execute_Remote_TopN(t *testing.T) { - c := test.NewCluster(2) + c := pilosa.NewTestCluster(2) // Create secondary server and update second cluster node. s := test.NewServer() @@ -1162,11 +1241,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // slices and a second time to get the counts for a set of bitmaps. switch remoteExecN { case 0: - if query.String() != `TopN(frame="f", n=3)` { + if query.String() != `TopN(field="f", n=3)` { t.Fatalf("unexpected query(0): %s", query.String()) } case 1: - if query.String() != `TopN(frame="f", ids=[0,10,30], n=3)` { + if query.String() != `TopN(field="f", ids=[0,10,30], n=3)` { t.Fatalf("unexpected query(1): %s", query.String()) } default: @@ -1186,11 +1265,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) + hldr.SetBit("i", "f", 30, (2*SliceWidth)+1) + hldr.SetBit("i", "f", 30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(field=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{ {ID: 0, Count: 5}, @@ -1205,30 +1284,31 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.MaxWritesPerRequest = 3 if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } -// Ensure SetColumnAttrs doesn't save `frame` as an attribute -func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { +// Ensure SetColumnAttrs doesn't save `field` as an attribute +func TestExectutor_SetColumnAttrs_ExcludeField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FrameOptions{}) + index.CreateField("f", pilosa.FieldOptions{}) targetAttrs := map[string]interface{}{ "foo": "bar", } - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) - // SetColumnAttrs call should exclude the frame attribute - _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=10)"), nil, nil) + // SetColumnAttrs call should exclude the field attribute + _, err := e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=10)"), nil, nil) if err != nil { t.Fatal(err) } - _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(frame='f', col=10, foo='bar')"), nil, nil) + _, err = e.Execute(context.Background(), "i", test.MustParse("SetColumnAttrs(field='f', col=10, foo='bar')"), nil, nil) if err != nil { t.Fatal(err) } @@ -1240,8 +1320,8 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { t.Fatalf("%#v != %#v", targetAttrs, attrs) } - // SetColumnAttrs call should not break if frame is not specified - _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(frame='f', row=1, col=20)"), nil, nil) + // SetColumnAttrs call should not break if field is not specified + _, err = e.Execute(context.Background(), "i", test.MustParse("SetBit(field='f', row=1, col=20)"), nil, nil) if err != nil { t.Fatal(err) } diff --git a/field.go b/field.go new file mode 100644 index 000000000..d9c88c924 --- /dev/null +++ b/field.go @@ -0,0 +1,1258 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" +) + +// Default field settings. +const ( + DefaultFieldType = FieldTypeSet + + defaultCacheType = CacheTypeRanked + + // Default ranked field cache + defaultCacheSize = 50000 +) + +// Field types. +const ( + FieldTypeSet = "set" + FieldTypeInt = "int" + FieldTypeTime = "time" +) + +// Field represents a container for views. +type Field struct { + mu sync.RWMutex + path string + index string + name string + + views map[string]*View + + // Row attribute storage and cache + rowAttrStore AttrStore + + broadcaster Broadcaster + Stats StatsClient + + // Field options. + options FieldOptions + + bsiGroups []*bsiGroup + + Logger Logger +} + +// FieldOption is a functional option type for pilosa.Fielde. +type FieldOption func(f *Field) error + +// TODO: break these out into separate Options (not a FieldOptions object) +func OptFieldFieldOptions(o FieldOptions) FieldOption { + return func(f *Field) error { + f.options = o + return nil + } +} + +// NewField returns a new instance of field. +func NewField(path, index, name string, opts ...FieldOption) (*Field, error) { + err := validateName(name) + if err != nil { + return nil, err + } + + f := &Field{ + path: path, + index: index, + name: name, + + views: make(map[string]*View), + + rowAttrStore: NopAttrStore, + + broadcaster: NopBroadcaster, + Stats: NopStatsClient, + + options: FieldOptions{ + Type: DefaultFieldType, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, + }, + + Logger: NopLogger, + } + + for _, opt := range opts { + err := opt(f) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return f, nil +} + +// Name returns the name the field was initialized with. +func (f *Field) Name() string { return f.name } + +// Index returns the index name the field was initialized with. +func (f *Field) Index() string { return f.index } + +// Path returns the path the field was initialized with. +func (f *Field) Path() string { return f.path } + +// RowAttrStore returns the attribute storage. +func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } + +// MaxSlice returns the max slice in the field. +func (f *Field) MaxSlice() uint64 { + f.mu.RLock() + defer f.mu.RUnlock() + + var max uint64 + for _, view := range f.views { + if viewMaxSlice := view.calculateMaxSlice(); viewMaxSlice > max { + max = viewMaxSlice + } + } + return max +} + +// Type returns the field type. +func (f *Field) Type() string { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Type +} + +// CacheType returns the caching mode for the field. +func (f *Field) CacheType() string { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.CacheType +} + +// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. +// defaults to DefaultCacheSize 50000 +func (f *Field) SetCacheSize(v uint32) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == 0 || f.options.CacheSize == v { + return nil + } + + // Persist meta data to disk on change. + f.options.CacheSize = v + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving") + } + + return nil +} + +// CacheSize returns the ranked field cache size. +func (f *Field) CacheSize() uint32 { + f.mu.RLock() + v := f.options.CacheSize + f.mu.RUnlock() + return v +} + +// Options returns all options for this field. +func (f *Field) Options() FieldOptions { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options +} + +// Open opens and initializes the field. +func (f *Field) Open() error { + if err := func() error { + // Ensure the field's path exists. + if err := os.MkdirAll(f.path, 0777); err != nil { + return errors.Wrap(err, "creating field dir") + } + + if err := f.loadMeta(); err != nil { + return errors.Wrap(err, "loading meta") + } + + // Apply the field options loaded from meta. + if err := f.applyOptions(f.options); err != nil { + return errors.Wrap(err, "applying options") + } + + if err := f.openViews(); err != nil { + return errors.Wrap(err, "opening views") + } + + if err := f.rowAttrStore.Open(); err != nil { + return errors.Wrap(err, "opening attrstore") + } + + return nil + }(); err != nil { + f.Close() + return err + } + + return nil +} + +// openViews opens and initializes the views inside the field. +func (f *Field) openViews() error { + file, err := os.Open(filepath.Join(f.path, "views")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "opening view directory") + } + defer file.Close() + + fis, err := file.Readdir(0) + if err != nil { + return errors.Wrap(err, "reading directory") + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + + name := filepath.Base(fi.Name()) + view := f.newView(f.ViewPath(name), name) + if err := view.open(); err != nil { + return fmt.Errorf("opening view: view=%s, err=%s", view.name, err) + } + view.RowAttrStore = f.rowAttrStore + f.views[view.name] = view + } + + return nil +} + +// loadMeta reads meta data for the field, if any. +func (f *Field) loadMeta() error { + var pb internal.FieldOptions + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return errors.Wrap(err, "reading meta") + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return errors.Wrap(err, "unmarshaling") + } + } + + // Copy metadata fields. + f.options.Type = pb.Type + f.options.CacheType = pb.CacheType + f.options.CacheSize = pb.CacheSize + f.options.Min = pb.Min + f.options.Max = pb.Max + f.options.TimeQuantum = TimeQuantum(pb.TimeQuantum) + f.options.Keys = pb.Keys + + return nil +} + +// saveMeta writes meta data for the field. +func (f *Field) saveMeta() error { + // Marshal metadata. + fo := f.options + buf, err := proto.Marshal(fo.Encode()) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { + return errors.Wrap(err, "writing meta") + } + + return nil +} + +// applyOptions configures the field based on opt. +func (f *Field) applyOptions(opt FieldOptions) error { + switch opt.Type { + case FieldTypeSet, "": + f.options.Type = FieldTypeSet + if opt.CacheType != "" { + f.options.CacheType = opt.CacheType + } + if opt.CacheSize != 0 { + f.options.CacheSize = opt.CacheSize + } + f.options.Min = 0 + f.options.Max = 0 + f.options.TimeQuantum = "" + f.options.Keys = opt.Keys + case FieldTypeInt: + f.options.Type = opt.Type + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = opt.Min + f.options.Max = opt.Max + f.options.TimeQuantum = "" + f.options.Keys = opt.Keys + + // Create new bsiGroup. + bsig := &bsiGroup{ + Name: f.name, + Type: bsiGroupTypeInt, + Min: opt.Min, + Max: opt.Max, + } + // Validate bsiGroup. + if err := bsig.validate(); err != nil { + return err + } + if err := f.createBSIGroup(bsig); err != nil { + return errors.Wrap(err, "creating bsigroup") + } + case FieldTypeTime: + f.options.Type = opt.Type + f.options.CacheType = CacheTypeNone + f.options.CacheSize = 0 + f.options.Min = 0 + f.options.Max = 0 + f.options.Keys = opt.Keys + // Set the time quantum. + if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + f.Close() + return errors.Wrap(err, "setting time quantum") + } + default: + return errors.New("invalid field type") + } + + return nil +} + +// Close closes the field and its views. +func (f *Field) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + + // Close the attribute store. + if f.rowAttrStore != nil { + _ = f.rowAttrStore.Close() + } + + // Close all views. + for _, view := range f.views { + if err := view.close(); err != nil { + return err + } + } + f.views = make(map[string]*View) + + return nil +} + +// Keys returns true if the field uses string keys. +func (f *Field) Keys() bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.options.Keys +} + +// bsiGroup returns a bsiGroup by name. +func (f *Field) bsiGroup(name string) *bsiGroup { + f.mu.RLock() + defer f.mu.RUnlock() + for _, bsig := range f.bsiGroups { + if bsig.Name == name { + return bsig + } + } + return nil +} + +// hasBSIGroup returns true if a bsiGroup exists on the field. +func (f *Field) hasBSIGroup(name string) bool { + for _, bsig := range f.bsiGroups { + if bsig.Name == name { + return true + } + } + return false +} + +// createBSIGroup creates a new bsiGroup on the field. +func (f *Field) createBSIGroup(bsig *bsiGroup) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Append bsiGroup. + if err := f.addBSIGroup(bsig); err != nil { + return err + } + f.saveMeta() + return nil +} + +// addBSIGroup adds a single bsiGroup to bsiGroups. +func (f *Field) addBSIGroup(bsig *bsiGroup) error { + if err := bsig.validate(); err != nil { + return errors.Wrap(err, "validating bsigroup") + } else if f.hasBSIGroup(bsig.Name) { + return ErrBSIGroupExists + } + + // Add bsiGroup to list. + f.bsiGroups = append(f.bsiGroups, bsig) + + // Sort bsiGroups by name. + sort.Slice(f.bsiGroups, func(i, j int) bool { + return f.bsiGroups[i].Name < f.bsiGroups[j].Name + }) + + return nil +} + +// deleteBSIGroupAndView deletes an existing bsiGroup on the schema. +func (f *Field) deleteBSIGroupAndView(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Remove bsiGroup. + if err := f.deleteBSIGroup(name); err != nil { + return err + } + + // Remove views. + viewName := viewBSIGroupPrefix + name + if view := f.views[viewName]; view != nil { + delete(f.views, viewName) + + if err := view.close(); err != nil { + return errors.Wrap(err, "closing view") + } else if err := os.RemoveAll(view.path); err != nil { + return errors.Wrap(err, "deleting directory") + } + } + + return nil +} + +// deleteBSIGroup removes a single bsiGroup from bsiGroups. +func (f *Field) deleteBSIGroup(name string) error { + for i, bsig := range f.bsiGroups { + if bsig.Name == name { + copy(f.bsiGroups[i:], f.bsiGroups[i+1:]) + f.bsiGroups, f.bsiGroups[len(f.bsiGroups)-1] = f.bsiGroups[:len(f.bsiGroups)-1], nil + return nil + } + } + return ErrBSIGroupNotFound +} + +// TimeQuantum returns the time quantum for the field. +func (f *Field) TimeQuantum() TimeQuantum { + f.mu.Lock() + defer f.mu.Unlock() + return f.options.TimeQuantum +} + +// SetTimeQuantum sets the time quantum for the field. +func (f *Field) SetTimeQuantum(q TimeQuantum) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on field. + f.options.TimeQuantum = q + + // Persist meta data to disk. + if err := f.saveMeta(); err != nil { + return errors.Wrap(err, "saving meta") + } + + return nil +} + +// ViewPath returns the path to a view in the field. +func (f *Field) ViewPath(name string) string { + return filepath.Join(f.path, "views", name) +} + +// View returns a view in the field by name. +func (f *Field) View(name string) *View { + f.mu.RLock() + defer f.mu.RUnlock() + return f.view(name) +} + +func (f *Field) view(name string) *View { return f.views[name] } + +// Views returns a list of all views in the field. +func (f *Field) Views() []*View { + f.mu.RLock() + defer f.mu.RUnlock() + + other := make([]*View, 0, len(f.views)) + for _, view := range f.views { + other = append(other, view) + } + return other +} + +// viewNames returns a list of all views (as a string) in the field. +func (f *Field) viewNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + + other := make([]string, 0, len(f.views)) + for viewName, _ := range f.views { + other = append(other, viewName) + } + return other +} + +// RecalculateCaches recalculates caches on every view in the field. +func (f *Field) RecalculateCaches() { + for _, view := range f.Views() { + view.recalculateCaches() + } +} + +// CreateViewIfNotExists returns the named view, creating it if necessary. +// Additionally, a CreateViewMessage is sent to the cluster. +func (f *Field) CreateViewIfNotExists(name string) (*View, error) { + + view, created, err := f.createViewIfNotExistsBase(name) + if err != nil { + return nil, err + } + + if created { + // Broadcast view creation to the cluster. + err = f.broadcaster.SendSync( + &internal.CreateViewMessage{ + Index: f.index, + Field: f.name, + View: name, + }) + if err != nil { + return nil, errors.Wrap(err, "sending CreateView message") + } + } + + return view, nil +} + +// createViewIfNotExistsBase returns the named view, creating it if necessary. +// The returned bool indicates whether the view was created or not. +func (f *Field) createViewIfNotExistsBase(name string) (*View, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if view := f.views[name]; view != nil { + return view, false, nil + } + + view := f.newView(f.ViewPath(name), name) + + if err := view.open(); err != nil { + return nil, false, errors.Wrap(err, "opening view") + } + view.RowAttrStore = f.rowAttrStore + f.views[view.name] = view + + return view, true, nil +} + +func (f *Field) newView(path, name string) *View { + view := NewView(path, f.index, f.name, name, f.options.CacheSize) + view.cacheType = f.options.CacheType + view.Logger = f.Logger + view.RowAttrStore = f.rowAttrStore + view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) + view.broadcaster = f.broadcaster + return view +} + +// DeleteView removes the view from the field. +func (f *Field) DeleteView(name string) error { + view := f.views[name] + if view == nil { + return ErrInvalidView + } + + // Close data files before deletion. + if err := view.close(); err != nil { + return errors.Wrap(err, "closing view") + } + + // Delete view directory. + if err := os.RemoveAll(view.path); err != nil { + return errors.Wrap(err, "deleting directory") + } + + delete(f.views, name) + + return nil +} + +// Row returns a row of the standard view. +func (f *Field) Row(rowID uint64) (*Row, error) { + if f.Type() != FieldTypeSet { + return nil, errors.Errorf("row method unsupported for field type: %s", f.Type()) + } + view := f.View(ViewStandard) + if view == nil { + return nil, ErrInvalidView + } + return view.row(rowID), nil +} + +// ViewRow returns a row for a view and slice. +// TODO: unexport this with views (it's only used in tests). +func (f *Field) ViewRow(viewName string, rowID uint64) (*Row, error) { + view := f.View(viewName) + if view == nil { + return nil, ErrInvalidView + } + return view.row(rowID), nil +} + +// SetBit sets a bit on a view within the field. +func (f *Field) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Validate view name. + if !isValidView(name) { + return false, ErrInvalidView + } + + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) + if err != nil { + return changed, errors.Wrap(err, "creating view") + } + + // Set non-time bit. + if v, err := view.setBit(rowID, colID); err != nil { + return changed, errors.Wrap(err, "setting on view") + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then set bits across all views for the quantum. + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, errors.Wrapf(err, "creating view %s", subname) + } + + if c, err := view.setBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "setting on view %s", subname) + } else if c { + changed = true + } + } + + return changed, nil +} + +// ClearBit clears a bit within the field. +func (f *Field) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { + // Validate view name. + if !isValidView(name) { + return false, ErrInvalidView + } + + // Retrieve view. Exit if it doesn't exist. + view, err := f.CreateViewIfNotExists(name) + if err != nil { + return changed, errors.Wrap(err, "creating view") + } + + // Clear non-time bit. + if v, err := view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrap(err, "clearing on view") + } else if v { + changed = v + } + + // Exit early if no timestamp is specified. + if t == nil { + return changed, nil + } + + // If a timestamp is specified then clear bits across all views for the quantum. + for _, subname := range viewsByTime(name, *t, f.TimeQuantum()) { + view, err := f.CreateViewIfNotExists(subname) + if err != nil { + return changed, errors.Wrapf(err, "creating view %s", subname) + } + + if c, err := view.clearBit(rowID, colID); err != nil { + return changed, errors.Wrapf(err, "clearing on view %s", subname) + } else if c { + changed = true + } + } + + return changed, nil +} + +// Value reads a field value for a column. +func (f *Field) Value(columnID uint64) (value int64, exists bool, err error) { + bsig := f.bsiGroup(f.name) + if bsig == nil { + return 0, false, ErrBSIGroupNotFound + } + + // Fetch target view. + view := f.View(viewBSIGroupPrefix + f.name) + if view == nil { + return 0, false, nil + } + + v, exists, err := view.value(columnID, bsig.BitDepth()) + if err != nil { + return 0, false, err + } else if !exists { + return 0, false, nil + } + return int64(v) + bsig.Min, true, nil +} + +// SetValue sets a field value for a column. +func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error) { + // Fetch bsiGroup and validate value. + bsig := f.bsiGroup(f.name) + if bsig == nil { + return false, ErrBSIGroupNotFound + } else if value < bsig.Min { + return false, ErrBSIGroupValueTooLow + } else if value > bsig.Max { + return false, ErrBSIGroupValueTooHigh + } + + // Fetch target view. + view, err := f.CreateViewIfNotExists(viewBSIGroupPrefix + f.name) + if err != nil { + return false, errors.Wrap(err, "creating view") + } + + // Determine base value to store. + baseValue := uint64(value - bsig.Min) + + return view.setValue(columnID, bsig.BitDepth(), baseValue) +} + +// Sum returns the sum and count for a field. +// An optional filtering row can be provided. +func (f *Field) Sum(filter *Row, name string) (sum, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.View(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vsum, vcount, err := view.sum(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vsum) + (int64(vcount) * bsig.Min), int64(vcount), nil +} + +// Min returns the min for a field. +// An optional filtering row can be provided. +func (f *Field) Min(filter *Row, name string) (min, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.View(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.min(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + bsig.Min, int64(vcount), nil +} + +// Max returns the max for a field. +// An optional filtering row can be provided. +func (f *Field) Max(filter *Row, name string) (max, count int64, err error) { + bsig := f.bsiGroup(name) + if bsig == nil { + return 0, 0, ErrBSIGroupNotFound + } + + view := f.View(viewBSIGroupPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.max(filter, bsig.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + bsig.Min, int64(vcount), nil +} + +func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error) { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { + return nil, ErrBSIGroupNotFound + } else if predicate < bsig.Min || predicate > bsig.Max { + return nil, nil + } + + // Retrieve bsiGroup's view. + view := f.View(viewBSIGroupPrefix + name) + if view == nil { + return nil, nil + } + + baseValue, outOfRange := bsig.baseValue(op, predicate) + if outOfRange { + return NewRow(), nil + } + + return view.rangeOp(op, bsig.BitDepth(), baseValue) +} + +func (f *Field) RangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { + // Retrieve and validate bsiGroup. + bsig := f.bsiGroup(name) + if bsig == nil { + return nil, ErrBSIGroupNotFound + } else if predicateMin > predicateMax { + return nil, ErrInvalidBetweenValue + } + + // Retrieve bsiGroup's view. + view := f.View(viewBSIGroupPrefix + name) + if view == nil { + return nil, nil + } + + baseValueMin, baseValueMax, outOfRange := bsig.baseValueBetween(predicateMin, predicateMax) + if outOfRange { + return NewRow(), nil + } + + return view.rangeBetween(bsig.BitDepth(), baseValueMin, baseValueMax) +} + +// Import bulk imports data. +func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { + // Determine quantum if timestamps are set. + q := f.TimeQuantum() + if hasTime(timestamps) && q == "" { + return errors.New("time quantum not set in field") + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] + var timestamp *time.Time + if len(timestamps) > i { + timestamp = timestamps[i] + } + + var standard []string + if timestamp == nil { + standard = []string{ViewStandard} + } else { + standard = viewsByTime(ViewStandard, *timestamp, q) + // In order to match the logic of `SetBit()`, we want bits + // with timestamps to write to both time and standard views. + standard = append(standard, ViewStandard) + } + + // Attach bit to each standard view. + for _, name := range standard { + key := importKey{View: name, Slice: columnID / SliceWidth} + data := dataByFragment[key] + data.RowIDs = append(data.RowIDs, rowID) + data.ColumnIDs = append(data.ColumnIDs, columnID) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + view, err := f.CreateViewIfNotExists(key.View) + if err != nil { + return errors.Wrap(err, "creating view") + } + + frag, err := view.CreateFragmentIfNotExists(key.Slice) + if err != nil { + return errors.Wrap(err, "creating view") + } + + if err := frag.bulkImport(data.RowIDs, data.ColumnIDs); err != nil { + return err + } + } + + return nil +} + +// ImportValue bulk imports range-encoded value data. +func (f *Field) ImportValue(columnIDs []uint64, values []int64) error { + viewName := viewBSIGroupPrefix + f.name + // Get the bsiGroup so we know bitDepth. + bsig := f.bsiGroup(f.name) + if bsig == nil { + return errors.Wrap(ErrBSIGroupNotFound, f.name) + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importValueData) + for i := range columnIDs { + columnID, value := columnIDs[i], values[i] + if int64(value) > bsig.Max { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value) + } else if int64(value) < bsig.Min { + return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value) + } + + // Attach value to each bsiGroup view. + for _, name := range []string{viewName} { + key := importKey{View: name, Slice: columnID / SliceWidth} + data := dataByFragment[key] + data.ColumnIDs = append(data.ColumnIDs, columnID) + data.Values = append(data.Values, value) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + + // The view must already exist (i.e. we can't create it) + // because we need to know bitDepth (based on min/max value). + view, err := f.CreateViewIfNotExists(key.View) + if err != nil { + return errors.Wrap(err, "creating view") + } + + frag, err := view.CreateFragmentIfNotExists(key.Slice) + if err != nil { + return errors.Wrap(err, "creating fragment") + } + + baseValues := make([]uint64, len(data.Values)) + for i, value := range data.Values { + baseValues[i] = uint64(value - bsig.Min) + } + + if err := frag.importValue(data.ColumnIDs, baseValues, bsig.BitDepth()); err != nil { + return err + } + } + + return nil +} + +// encodeFields converts a into its internal representation. +func encodeFields(a []*Field) []*internal.Field { + other := make([]*internal.Field, len(a)) + for i := range a { + other[i] = encodeField(a[i]) + } + return other +} + +// encodeField converts f into its internal representation. +func encodeField(f *Field) *internal.Field { + fo := f.options + return &internal.Field{ + Name: f.name, + Meta: fo.Encode(), + Views: f.viewNames(), + } +} + +type fieldSlice []*Field + +func (p fieldSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldSlice) Len() int { return len(p) } +func (p fieldSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } + +// FieldInfo represents schema information for a field. +type FieldInfo struct { + Name string `json:"name"` + Options FieldOptions `json:"options"` + Views []*ViewInfo `json:"views,omitempty"` +} + +type fieldInfoSlice []*FieldInfo + +func (p fieldInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p fieldInfoSlice) Len() int { return len(p) } +func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + +// FieldOptions represents options to set when initializing a field. +type FieldOptions struct { + Type string `json:"type,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Keys bool `json:"keys,omitempty"` +} + +// Validate ensures that FieldOption values are valid. +func (o *FieldOptions) Validate() error { + switch o.Type { + case FieldTypeSet, "": + // TODO: cacheType, cacheSize validation + case FieldTypeInt: + if o.Min > o.Max { + return ErrInvalidBSIGroupRange + } + case FieldTypeTime: + if o.TimeQuantum == "" || !o.TimeQuantum.Valid() { + return ErrInvalidTimeQuantum + } + default: + return errors.New("invalid field type") + } + return nil +} + +// Encode converts o into its internal representation. +func (o *FieldOptions) Encode() *internal.FieldOptions { + return encodeFieldOptions(o) +} + +func encodeFieldOptions(o *FieldOptions) *internal.FieldOptions { + if o == nil { + return nil + } + return &internal.FieldOptions{ + Type: o.Type, + CacheType: o.CacheType, + CacheSize: o.CacheSize, + Min: o.Min, + Max: o.Max, + TimeQuantum: string(o.TimeQuantum), + Keys: o.Keys, + } +} + +func decodeFieldOptions(options *internal.FieldOptions) *FieldOptions { + if options == nil { + return nil + } + return &FieldOptions{ + Type: options.Type, + CacheType: options.CacheType, + CacheSize: options.CacheSize, + Min: options.Min, + Max: options.Max, + TimeQuantum: TimeQuantum(options.TimeQuantum), + Keys: options.Keys, + } +} + +// List of bsiGroup types. +const ( + bsiGroupTypeInt = "int" +) + +func isValidBSIGroupType(v string) bool { + switch v { + case bsiGroupTypeInt: + return true + default: + return false + } +} + +// bsiGroup represents a group of range-encoded rows on a field. +type bsiGroup struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` +} + +// BitDepth returns the number of bits required to store a value between min & max. +func (b *bsiGroup) BitDepth() uint { + for i := uint(0); i < 63; i++ { + if b.Max-b.Min < (1 << i) { + return i + } + } + return 63 +} + +// baseValue adjusts the value to align with the range for Field for a certain +// operation type. +// Note: There is an edge case for GT and LT where this returns a baseValue +// that does not fully encompass the range. +// ex: Field.Min = 0, Field.Max = 1023 +// baseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively +// exclude any columns with value = 1023. +// Note that in this case (because the range uses the full BitDepth 0 to 1023), +// we can't simply return 1024. +// In order to make this work, we effectively need to change the operator to LTE. +// Executor.executeBSIGroupRangeSlice() takes this into account and returns +// `frag.FieldNotNull(bsig.BitDepth())` in such instances. +func (b *bsiGroup) baseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { + if op == pql.GT || op == pql.GTE { + if value > b.Max { + return baseValue, true + } else if value > b.Min { + baseValue = uint64(value - b.Min) + } + } else if op == pql.LT || op == pql.LTE { + if value < b.Min { + return baseValue, true + } else if value > b.Max { + baseValue = uint64(b.Max - b.Min) + } else { + baseValue = uint64(value - b.Min) + } + } else if op == pql.EQ || op == pql.NEQ { + if value < b.Min || value > b.Max { + return baseValue, true + } + baseValue = uint64(value - b.Min) + } + return baseValue, false +} + +// baseValueBetween adjusts the min/max value to align with the range for Field. +func (b *bsiGroup) baseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { + if max < b.Min || min > b.Max { + return baseValueMin, baseValueMax, true + } + // Adjust min/max to range. + if min > b.Min { + baseValueMin = uint64(min - b.Min) + } + // Make sure the high value of the BETWEEN does not exceed BitDepth. + if max > b.Max { + baseValueMax = uint64(b.Max - b.Min) + } else if max > b.Min { + baseValueMax = uint64(max - b.Min) + } + return baseValueMin, baseValueMax, false +} + +func (b *bsiGroup) validate() error { + if b.Name == "" { + return ErrBSIGroupNameRequired + } else if !isValidBSIGroupType(b.Type) { + return ErrInvalidBSIGroupType + } else if b.Min > b.Max { + return ErrInvalidBSIGroupRange + } + return nil +} + +func encodeBSIGroup(b *bsiGroup) *internal.BSIGroup { + if b == nil { + return nil + } + return &internal.BSIGroup{ + Name: b.Name, + Type: b.Type, + Min: int64(b.Min), + Max: int64(b.Max), + } +} + +func decodeBSIGroup(b *internal.BSIGroup) *bsiGroup { + if b == nil { + return nil + } + return &bsiGroup{ + Name: b.Name, + Type: b.Type, + Min: b.Min, + Max: b.Max, + } +} + +// importBitSet represents slices of row and column ids. +// This is used to sort data during import. +type importBitSet struct { + rowIDs, columnIDs []uint64 +} + +func (p importBitSet) Swap(i, j int) { + p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i] + p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i] +} +func (p importBitSet) Len() int { return len(p.rowIDs) } +func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } + +// Cache types. +const ( + CacheTypeLRU = "lru" + CacheTypeRanked = "ranked" + CacheTypeNone = "none" +) + +// isValidCacheType returns true if v is a valid cache type. +func isValidCacheType(v string) bool { + switch v { + case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: + return true + default: + return false + } +} diff --git a/field_internal_test.go b/field_internal_test.go new file mode 100644 index 000000000..91b1af02c --- /dev/null +++ b/field_internal_test.go @@ -0,0 +1,149 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "reflect" + "testing" + + "github.com/pilosa/pilosa/pql" +) + +// Ensure a bsiGroup can adjust to its baseValue. +func TestBSIGroup_BaseValue(t *testing.T) { + b0 := &bsiGroup{ + Name: "b0", + Type: bsiGroupTypeInt, + Min: -100, + Max: 900, + } + b1 := &bsiGroup{ + Name: "b1", + Type: bsiGroupTypeInt, + Min: 0, + Max: 1000, + } + + b2 := &bsiGroup{ + Name: "b2", + Type: bsiGroupTypeInt, + Min: 100, + Max: 1100, + } + + t.Run("Normal Condition", func(t *testing.T) { + + for _, tt := range []struct { + f *bsiGroup + op pql.Token + val int64 + expBaseValue uint64 + expOutOfRange bool + }{ + // LT + {b0, pql.LT, 5, 105, false}, + {b0, pql.LT, -8, 92, false}, + {b0, pql.LT, -108, 0, true}, + {b0, pql.LT, 1005, 1000, false}, + {b0, pql.LT, 0, 100, false}, + + {b1, pql.LT, 5, 5, false}, + {b1, pql.LT, -8, 0, true}, + {b1, pql.LT, 1005, 1000, false}, + {b1, pql.LT, 0, 0, false}, + + {b2, pql.LT, 5, 0, true}, + {b2, pql.LT, -8, 0, true}, + {b2, pql.LT, 105, 5, false}, + {b2, pql.LT, 1105, 1000, false}, + + // GT + {b0, pql.GT, -105, 0, false}, + {b0, pql.GT, 5, 105, false}, + {b0, pql.GT, 905, 0, true}, + {b0, pql.GT, 0, 100, false}, + + {b1, pql.GT, 5, 5, false}, + {b1, pql.GT, -8, 0, false}, + {b1, pql.GT, 1005, 0, true}, + {b1, pql.GT, 0, 0, false}, + + {b2, pql.GT, 5, 0, false}, + {b2, pql.GT, -8, 0, false}, + {b2, pql.GT, 105, 5, false}, + {b2, pql.GT, 1105, 0, true}, + + // EQ + {b0, pql.EQ, -105, 0, true}, + {b0, pql.EQ, 5, 105, false}, + {b0, pql.EQ, 905, 0, true}, + {b0, pql.EQ, 0, 100, false}, + + {b1, pql.EQ, 5, 5, false}, + {b1, pql.EQ, -8, 0, true}, + {b1, pql.EQ, 1005, 0, true}, + {b1, pql.EQ, 0, 0, false}, + + {b2, pql.EQ, 5, 0, true}, + {b2, pql.EQ, -8, 0, true}, + {b2, pql.EQ, 105, 5, false}, + {b2, pql.EQ, 1105, 0, true}, + } { + bv, oor := tt.f.baseValue(tt.op, tt.val) + if oor != tt.expOutOfRange { + t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(bv, tt.expBaseValue) { + t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) + } + } + }) + + t.Run("Betwween Condition", func(t *testing.T) { + for _, tt := range []struct { + f *bsiGroup + predMin int64 + predMax int64 + expBaseValueMin uint64 + expBaseValueMax uint64 + expOutOfRange bool + }{ + + {b0, -205, -105, 0, 0, true}, + {b0, -105, 80, 0, 180, false}, + {b0, 5, 20, 105, 120, false}, + {b0, 20, 1005, 120, 1000, false}, + {b0, 1005, 2000, 0, 0, true}, + + {b1, -105, -5, 0, 0, true}, + {b1, -5, 20, 0, 20, false}, + {b1, 5, 20, 5, 20, false}, + {b1, 20, 1005, 20, 1000, false}, + {b1, 1005, 2000, 0, 0, true}, + + {b2, 5, 95, 0, 0, true}, + {b2, 95, 120, 0, 20, false}, + {b2, 105, 120, 5, 20, false}, + {b2, 120, 1105, 20, 1000, false}, + {b2, 1105, 2000, 0, 0, true}, + } { + min, max, oor := tt.f.baseValueBetween(tt.predMin, tt.predMax) + if oor != tt.expOutOfRange { + t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) + } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { + t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) + } + } + }) +} diff --git a/field_test.go b/field_test.go new file mode 100644 index 000000000..10e4d9a97 --- /dev/null +++ b/field_test.go @@ -0,0 +1,287 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "io/ioutil" + "testing" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// Ensure field can open and retrieve a view. +func TestField_CreateViewIfNotExists(t *testing.T) { + f := test.MustOpenField() + defer f.Close() + + // Create view. + view, err := f.CreateViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + // Retrieve existing view. + view2, err := f.CreateViewIfNotExists("v") + if err != nil { + t.Fatal(err) + } else if view != view2 { + t.Fatal("view mismatch") + } + + if view != f.View("v") { + t.Fatal("view mismatch") + } +} + +// Ensure field can set its time quantum. +func TestField_SetTimeQuantum(t *testing.T) { + fo := pilosa.FieldOptions{ + Type: "time", + } + f := test.MustOpenField(pilosa.OptFieldFieldOptions(fo)) + defer f.Close() + + // Set & retrieve time quantum. + if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload field and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + +// Ensure a field can set & read a bsiGroup value. +func TestField_SetValue(t *testing.T) { + t.Run("OK", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 30, + }) + if err != nil { + t.Fatal(err) + } + + // Set value on field. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if value != 21 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + + // Setting value should return no change. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if changed { + t.Fatal("expected no change") + } + }) + + t.Run("Overwrite", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 0, + Max: 30, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if changed, err := f.SetValue(100, 21); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Set different value. + if changed, err := f.SetValue(100, 23); err != nil { + t.Fatal(err) + } else if !changed { + t.Fatal("expected change") + } + + // Read value. + if value, exists, err := f.Value(100); err != nil { + t.Fatal(err) + } else if value != 23 { + t.Fatalf("unexpected value: %d", value) + } else if !exists { + t.Fatal("expected value to exist") + } + }) + + t.Run("ErrBSIGroupNotFound", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeSet, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 21); err != pilosa.ErrBSIGroupNotFound { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 20, + Max: 30, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 15); err != pilosa.ErrBSIGroupValueTooLow { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { + idx := test.MustOpenIndex() + defer idx.Close() + + f, err := idx.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 20, + Max: 30, + }) + if err != nil { + t.Fatal(err) + } + + // Set value. + if _, err := f.SetValue(100, 31); err != pilosa.ErrBSIGroupValueTooHigh { + t.Fatalf("unexpected error: %s", err) + } + }) +} + +func TestField_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + field, err := pilosa.NewField(path, "i", ".meta") + if field != nil { + t.Fatalf("unexpected field name %s", err) + } +} + +// Ensure that field name validation is consistent. +func TestField_NameValidation(t *testing.T) { + validFieldNames := []string{ + "foo", + "hyphen-ated", + "under_score", + "abc123", + "trailing_", + } + invalidFieldNames := []string{ + "", + "123abc", + "x.y", + "_foo", + "-bar", + "abc def", + "camelCase", + "UPPERCASE", + "a12345678901234567890123456789012345678901234567890123456789012345", + } + + path, err := ioutil.TempDir("", "pilosa-field-") + if err != nil { + panic(err) + } + for _, name := range validFieldNames { + _, err := pilosa.NewField(path, "i", name) + if err != nil { + t.Fatalf("unexpected field name: %s %s", name, err) + } + } + for _, name := range invalidFieldNames { + _, err := pilosa.NewField(path, "i", name) + if err == nil { + t.Fatalf("expected error on field name: %s", name) + } + } +} + +// Ensure field can open and retrieve a view. +func TestField_DeleteView(t *testing.T) { + f := test.MustOpenField() + defer f.Close() + + viewName := pilosa.ViewStandard + "_v" + + // Create view. + view, err := f.CreateViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == nil { + t.Fatal("expected view") + } + + err = f.DeleteView(viewName) + if err != nil { + t.Fatal(err) + } + + if f.View(viewName) != nil { + t.Fatal("view still exists in field") + } + + // Recreate view with same name, verify that the old view was not reused. + view2, err := f.CreateViewIfNotExists(viewName) + if err != nil { + t.Fatal(err) + } else if view == view2 { + t.Fatal("failed to create new view") + } +} diff --git a/fragment.go b/fragment.go index 8044f579b..799c8b695 100644 --- a/fragment.go +++ b/fragment.go @@ -25,7 +25,6 @@ import ( "hash" "io" "io/ioutil" - "net/http" "os" "sort" "sync" @@ -48,31 +47,29 @@ const ( // SliceWidth is the number of column IDs in a slice. SliceWidth = 1048576 - // SnapshotExt is the file extension used for an in-process snapshot. - SnapshotExt = ".snapshotting" + // snapshotExt is the file extension used for an in-process snapshot. + snapshotExt = ".snapshotting" - // CopyExt is the file extension used for the temp file used while copying. - CopyExt = ".copying" + // copyExt is the file extension used for the temp file used while copying. + copyExt = ".copying" - // CacheExt is the file extension for persisted cache ids. - CacheExt = ".cache" + // cacheExt is the file extension for persisted cache ids. + cacheExt = ".cache" // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 + + // defaultFragmentMaxOpN is the default value for Fragment.MaxOpN. + defaultFragmentMaxOpN = 2000 ) -const ( - // DefaultFragmentMaxOpN is the default value for Fragment.MaxOpN. - DefaultFragmentMaxOpN = 2000 -) - -// Fragment represents the intersection of a frame and slice in an index. +// Fragment represents the intersection of a field and slice in an index. type Fragment struct { mu sync.RWMutex // Composite identifiers index string - frame string + field string view string slice uint64 @@ -84,7 +81,7 @@ type Fragment struct { opN int // number of ops since snapshot // Cache for row counts. - CacheType string // passed in by frame + CacheType string // passed in by field cache Cache CacheSize uint32 @@ -106,51 +103,32 @@ type Fragment struct { Logger Logger // Row attribute storage. - // This is set by the parent frame unless overridden for testing. + // This is set by the parent field unless overridden for testing. RowAttrStore AttrStore stats StatsClient } // NewFragment returns a new instance of Fragment. -func NewFragment(path, index, frame, view string, slice uint64) *Fragment { +func NewFragment(path, index, field, view string, slice uint64) *Fragment { return &Fragment{ path: path, index: index, - frame: frame, + field: field, view: view, slice: slice, - CacheType: DefaultCacheType, - CacheSize: DefaultCacheSize, + CacheType: defaultCacheType, + CacheSize: defaultCacheSize, Logger: NopLogger, - MaxOpN: DefaultFragmentMaxOpN, + MaxOpN: defaultFragmentMaxOpN, stats: NopStatsClient, } } -// Path returns the path the fragment was initialized with. -func (f *Fragment) Path() string { return f.path } - -// CachePath returns the path to the fragment's cache data. -func (f *Fragment) CachePath() string { return f.path + CacheExt } - -// Index returns the index that the fragment was initialized with. -func (f *Fragment) Index() string { return f.index } - -// Frame returns the frame the fragment was initialized with. -func (f *Fragment) Frame() string { return f.frame } - -// View returns the view the fragment was initialized with. -func (f *Fragment) View() string { return f.view } - -// Slice returns the slice the fragment was initialized with. -func (f *Fragment) Slice() uint64 { return f.slice } - -// Cache returns the fragment's cache. -// This is not safe for concurrent use. -func (f *Fragment) Cache() Cache { return f.cache } +// cachePath returns the path to the fragment's cache data. +func (f *Fragment) cachePath() string { return f.path + cacheExt } // Open opens the underlying storage. func (f *Fragment) Open() error { @@ -247,7 +225,7 @@ func (f *Fragment) openStorage() error { // openCache initializes the cache from row ids persisted to disk. func (f *Fragment) openCache() error { - // Determine cache type from frame name. + // Determine cache type from field name. switch f.CacheType { case CacheTypeRanked: f.cache = NewRankCache(f.CacheSize) @@ -261,7 +239,7 @@ func (f *Fragment) openCache() error { } // Read cache data from disk. - path := f.CachePath() + path := f.cachePath() buf, err := ioutil.ReadFile(path) if os.IsNotExist(err) { return nil @@ -342,14 +320,14 @@ func (f *Fragment) closeStorage() error { return nil } -// Row returns a row by ID. -func (f *Fragment) Row(rowID uint64) *Row { +// row returns a row by ID. +func (f *Fragment) row(rowID uint64) *Row { f.mu.Lock() defer f.mu.Unlock() - return f.row(rowID, true, true) + return f.unprotectedRow(rowID, true, true) } -func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { +func (f *Fragment) unprotectedRow(rowID uint64, checkRowCache bool, updateRowCache bool) *Row { if checkRowCache { r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { @@ -380,15 +358,15 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a bit for a given column & row within the fragment. +// setBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(rowID, columnID) + return f.unprotectedSetBit(rowID, columnID) } -func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedSetBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -415,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from row cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.SetBit(columnID) // Update the cache. @@ -432,15 +410,15 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a bit for a given column & row within the fragment. +// clearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(rowID, columnID) + return f.unprotectedClearBit(rowID, columnID) } -func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { +func (f *Fragment) unprotectedClearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) @@ -467,7 +445,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { } // Get the row from cache or fragment.storage. - row := f.row(rowID, true, true) + row := f.unprotectedRow(rowID, true, true) row.ClearBit(columnID) // Update the cache. @@ -486,8 +464,8 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// FieldValue uses a column of bits to read a multi-bit value. -func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (f *Fragment) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() @@ -510,20 +488,20 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi return value, true, nil } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (f *Fragment) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() for i := uint(0); i < bitDepth; i++ { if value&(1< uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := consider.Difference(row) count = x.Count() @@ -647,11 +625,11 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err return min, count, nil } -// FieldMax returns the max of a given field as well as the number of columns involved. +// max returns the max of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *Fragment) max(filter *Row, bitDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.row(uint64(bitDepth)) if filter != nil { consider = consider.Intersect(filter) } @@ -663,7 +641,7 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (bitDepth-1) to 0 - row := f.Row(uint64(ii)) + row := f.row(uint64(ii)) x := row.Intersect(consider) count = x.Count() @@ -678,29 +656,29 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err return max, count, nil } -// FieldRange returns bitmaps with a field value encoding matching the predicate. -func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns bitmaps with a bsiGroup value encoding matching the predicate. +func (f *Fragment) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: - return f.fieldRangeEQ(bitDepth, predicate) + return f.rangeEQ(bitDepth, predicate) case pql.NEQ: - return f.fieldRangeNEQ(bitDepth, predicate) + return f.rangeNEQ(bitDepth, predicate) case pql.LT, pql.LTE: - return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + return f.rangeLT(bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) + return f.rangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } } -func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) rangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 if bit == 1 { @@ -713,12 +691,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) rangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Get the equal bitmap. - eq, err := f.fieldRangeEQ(bitDepth, predicate) + eq, err := f.rangeEQ(bitDepth, predicate) if err != nil { return nil, err } @@ -729,16 +707,16 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) return b, nil } -func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *Fragment) rangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.row(uint64(bitDepth)) // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Remove any columns with higher bits set. @@ -777,13 +755,13 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(bitDepth)) +func (f *Fragment) rangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + b := f.row(uint64(bitDepth)) keep := NewRow() // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. @@ -812,20 +790,20 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -// FieldNotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) { - return f.Row(uint64(bitDepth)), nil +// notNull returns the not-null row (stored at bitDepth). +func (f *Fragment) notNull(bitDepth uint) (*Row, error) { + return f.row(uint64(bitDepth)), nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(bitDepth)) +// rangeBetween returns bitmaps with a bsiGroup value encoding matching any value between predicateMin and predicateMax. +func (f *Fragment) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { + b := f.row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { - row := f.Row(uint64(i)) + row := f.row(uint64(i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 @@ -864,12 +842,12 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { if columnID < minColumnID || columnID >= minColumnID+SliceWidth { return 0, errors.New("column out of bounds") } - return Pos(rowID, columnID), nil + return pos(rowID, columnID), nil } -// ForEachBit executes fn for every bit set in the fragment. +// forEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { +func (f *Fragment) forEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -886,10 +864,10 @@ func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { return err } -// Top returns the top rows from the fragment. +// top returns the top rows from the fragment. // If opt.Src is specified then only rows which intersect src are returned. // If opt.FilterValues exist then the row attribute specified by field is matched. -func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { +func (f *Fragment) top(opt TopOptions) ([]Pair, error) { // Retrieve pairs. If no row ids specified then return from cache. pairs := f.topBitmapPairs(opt.RowIDs) @@ -900,7 +878,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Create a fast lookup of filter values. var filters map[interface{}]struct{} - if opt.FilterField != "" && len(opt.FilterValues) > 0 { + if opt.FilterName != "" && len(opt.FilterValues) > 0 { filters = make(map[interface{}]struct{}) for _, v := range opt.FilterValues { filters[v] = struct{}{} @@ -948,7 +926,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return nil, errors.Wrap(err, "getting attrs") } else if attr == nil { continue - } else if attrValue := attr[opt.FilterField]; attrValue == nil { + } else if attrValue := attr[opt.FilterName]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { continue @@ -960,7 +938,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Row(rowID)) + count = opt.Src.IntersectionCount(f.row(rowID)) } if count == 0 { continue @@ -1004,7 +982,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate the intersecting column count and skip if it's below our // last row in our current result set. - count := opt.Src.IntersectionCount(f.Row(rowID)) + count := opt.Src.IntersectionCount(f.row(rowID)) if count < threshold { continue } @@ -1048,7 +1026,7 @@ func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { continue } - row := f.Row(rowID) + row := f.row(rowID) if row.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, BitmapPair{ @@ -1074,7 +1052,7 @@ type TopOptions struct { MinThreshold uint64 // Filter field name & values. - FilterField string + FilterName string FilterValues []interface{} TanimotoThreshold uint64 } @@ -1089,13 +1067,6 @@ func (f *Fragment) Checksum() []byte { return h.Sum(nil) } -// BlockN returns the number of blocks in the fragment. -func (f *Fragment) BlockN() int { - f.mu.Lock() - defer f.mu.Unlock() - return int(f.storage.Max() / (HashBlockSize * SliceWidth)) -} - // InvalidateChecksums clears all cached block checksums. func (f *Fragment) InvalidateChecksums() { f.mu.Lock() @@ -1184,8 +1155,8 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as row & column ID pairs. -func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { +// blockData returns bits in a block as row & column ID pairs. +func (f *Fragment) blockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() @@ -1196,17 +1167,17 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { return } -// MergeBlock compares the block's bits and computes a diff with another set of block bits. +// mergeBlock compares the block's bits and computes a diff with another set of block bits. // The state of a bit is determined by consensus from all blocks being considered. // // For example, if 3 blocks are compared and two have a set bit and one has a // cleared bit then the bit is considered cleared. The function returns the // diff per incoming block so that all can be in sync. -func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { +func (f *Fragment) mergeBlock(id int, data []pairSet) (sets, clears []pairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].RowIDs) != len(data[i].ColumnIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) + if len(data[i].rowIDs) != len(data[i].columnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].rowIDs), len(data[i].columnIDs)) } } @@ -1214,8 +1185,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e defer f.mu.Unlock() // Track sets and clears for all blocks (including local). - sets = make([]PairSet, len(data)+1) - clears = make([]PairSet, len(data)+1) + sets = make([]pairSet, len(data)+1) + clears = make([]pairSet, len(data)+1) // Limit upper row/column pair. maxRowID := uint64(id+1) * HashBlockSize @@ -1231,7 +1202,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) + var itr Iterator = NewSliceIterator(data[i].rowIDs, data[i].columnIDs) itr = NewLimitIterator(itr, maxRowID, maxColumnID) itrs = append(itrs, NewBufIterator(itr)) } @@ -1296,25 +1267,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) - sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + sets[i].rowIDs = append(sets[i].rowIDs, min.rowID) + sets[i].columnIDs = append(sets[i].columnIDs, min.columnID) } else { - clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) - clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) + clears[i].rowIDs = append(sets[i].rowIDs, min.rowID) + clears[i].columnIDs = append(sets[i].columnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ColumnIDs { - if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { + for i := range sets[0].columnIDs { + if _, err := f.unprotectedSetBit(sets[0].rowIDs[i], (f.slice*SliceWidth)+sets[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } // Clear local bits. - for i := range clears[0].ColumnIDs { - if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { + for i := range clears[0].columnIDs { + if _, err := f.unprotectedClearBit(clears[0].rowIDs[i], (f.slice*SliceWidth)+clears[0].columnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") } } @@ -1322,9 +1293,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of bits and then snapshots the storage. +// bulkImport bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { +func (f *Fragment) bulkImport(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of row ids and column ids. @@ -1373,7 +1344,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { // Import should ALWAYS have row() load a new row from fragment.storage // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) + f.cache.BulkAdd(rowID, f.unprotectedRow(rowID, false, false).Count()) } f.cache.Invalidate() @@ -1392,8 +1363,8 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { return nil } -// ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { +// importValue bulk imports a set of range-encoded values. +func (f *Fragment) importValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1408,7 +1379,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error for i := range columnIDs { columnID, value := columnIDs[i], values[i] - _, err := f.importSetFieldValue(columnID, bitDepth, value) + _, err := f.importSetValue(columnID, bitDepth, value) if err != nil { return errors.Wrap(err, "setting") } @@ -1452,13 +1423,13 @@ func track(start time.Time, message string, stats StatsClient, logger Logger) { } func (f *Fragment) snapshot() error { - f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) - completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice) + f.Logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.field, f.view, f.slice) + completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.field, f.view, f.slice) start := time.Now() defer track(start, completeMessage, f.stats, f.Logger) // Create a temporary file to snapshot to. - snapshotPath := f.path + SnapshotExt + snapshotPath := f.path + snapshotExt file, err := os.Create(snapshotPath) if err != nil { return fmt.Errorf("create snapshot file: %s", err) @@ -1529,7 +1500,7 @@ func (f *Fragment) flushCache() error { } // Write to disk. - if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1603,7 +1574,7 @@ func (f *Fragment) writeCacheToArchive(tw *tar.Writer) error { defer f.mu.Unlock() // Read cache into buffer. - buf, err := ioutil.ReadFile(f.CachePath()) + buf, err := ioutil.ReadFile(f.cachePath()) if os.IsNotExist(err) { return nil } else if err != nil { @@ -1662,7 +1633,7 @@ func (f *Fragment) ReadFrom(r io.Reader) (n int64, err error) { func (f *Fragment) readStorageFromArchive(r io.Reader) error { // Create a temporary file to copy into. - path := f.path + CopyExt + path := f.path + copyExt file, err := os.Create(path) if err != nil { return errors.Wrap(err, "creating directory") @@ -1697,7 +1668,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error { buf, err := ioutil.ReadAll(r) if err != nil { return errors.Wrap(err, "reading") - } else if err := ioutil.WriteFile(f.CachePath(), buf, 0666); err != nil { + } else if err := ioutil.WriteFile(f.cachePath(), buf, 0666); err != nil { return errors.Wrap(err, "writing") } @@ -1745,9 +1716,8 @@ func (h *blockHasher) WriteValue(v uint64) { type FragmentSyncer struct { Fragment *Fragment - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster Closing <-chan struct{} } @@ -1762,11 +1732,11 @@ func (s *FragmentSyncer) isClosing() bool { } } -// SyncFragment compares checksums for the local and remote fragments and +// syncFragment compares checksums for the local and remote fragments and // then merges any blocks which have differences. -func (s *FragmentSyncer) SyncFragment() error { +func (s *FragmentSyncer) syncFragment() error { // Determine replica set. - nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.sliceNodes(s.Fragment.index, s.Fragment.slice) if len(nodes) == 1 { return nil } @@ -1782,8 +1752,7 @@ func (s *FragmentSyncer) SyncFragment() error { } // Retrieve remote blocks. - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.Slice()) + blocks, err := s.Cluster.InternalClient.FragmentBlocks(context.Background(), nil, s.Fragment.index, s.Fragment.field, s.Fragment.slice) if err != nil && err != ErrFragmentNotFound { return errors.Wrap(err, "getting blocks") } @@ -1846,9 +1815,9 @@ func (s *FragmentSyncer) syncBlock(id int) error { f := s.Fragment // Read pairs from each remote block. - var pairSets []PairSet - var clients []InternalClient - for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { + var uris []*URI + var pairSets []pairSet + for _, node := range s.Cluster.sliceNodes(f.index, f.slice) { if s.Node.ID == node.ID { continue } @@ -1858,18 +1827,18 @@ func (s *FragmentSyncer) syncBlock(id int) error { return nil } - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - clients = append(clients, client) + uri := &node.URI + uris = append(uris, uri) // Only sync the standard block. - rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), f.Slice(), id) + rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(context.Background(), &node.URI, f.index, f.field, f.slice, id) if err != nil { return errors.Wrap(err, "getting block") } - pairSets = append(pairSets, PairSet{ - ColumnIDs: columnIDs, - RowIDs: rowIDs, + pairSets = append(pairSets, pairSet{ + columnIDs: columnIDs, + rowIDs: rowIDs, }) } @@ -1879,23 +1848,23 @@ func (s *FragmentSyncer) syncBlock(id int) error { } // Merge blocks together. - sets, clears, err := f.MergeBlock(id, pairSets) + sets, clears, err := f.mergeBlock(id, pairSets) if err != nil { return errors.Wrap(err, "merging") } // Write updates to remote blocks. - for i := 0; i < len(clients); i++ { + for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] count := 0 // Ignore if there are no differences. - if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { + if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { continue } // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.ColumnIDs) + len(clear.ColumnIDs) + total := len(set.columnIDs) + len(clear.columnIDs) maxWrites := s.Cluster.MaxWritesPerRequest if maxWrites <= 0 { maxWrites = 5000 @@ -1903,12 +1872,12 @@ func (s *FragmentSyncer) syncBlock(id int) error { buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. - for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + for j := 0; j < len(set.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(field=%q, row=%d, col=%d)\n", f.field, set.rowIDs[j], (f.slice*SliceWidth)+set.columnIDs[j]) count++ } - for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + for j := 0; j < len(clear.columnIDs); j++ { + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(field=%q, row=%d, col=%d)\n", f.field, clear.rowIDs[j], (f.slice*SliceWidth)+clear.columnIDs[j]) count++ } @@ -1924,7 +1893,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { Query: buffers[k].String(), Remote: true, } - _, err := clients[i].Query(context.Background(), f.Index(), queryRequest) + _, err := s.Cluster.InternalClient.QueryNode(context.Background(), uris[i], f.index, queryRequest) if err != nil { return errors.Wrap(err, "executing") } @@ -1942,10 +1911,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length row and column id lists. -type PairSet struct { - RowIDs []uint64 - ColumnIDs []uint64 +// pairSet is a list of equal length row and column id lists. +type pairSet struct { + rowIDs []uint64 + columnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1962,7 +1931,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the row position of a row/column pair. -func Pos(rowID, columnID uint64) uint64 { +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { return (rowID * SliceWidth) + (columnID % SliceWidth) } diff --git a/fragment_test.go b/fragment_internal_test.go similarity index 57% rename from fragment_test.go rename to fragment_internal_test.go index 5c5f03b16..6c733ba4c 100644 --- a/fragment_test.go +++ b/fragment_internal_test.go @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "bytes" "flag" + "io/ioutil" "math" "reflect" "testing" "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" ) // Test flags @@ -35,82 +34,79 @@ var ( FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") ) -// SliceWidth is a helper reference to use when testing. -const SliceWidth = pilosa.SliceWidth - // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1); err != nil { + if _, err := f.setBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6); err != nil { + } else if _, err := f.setBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0); err != nil { + } else if _, err := f.setBit(121, 0); err != nil { t.Fatal(err) } // Verify counts on rows. - if n := f.Row(120).Count(); n != 2 { + if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(120).Count(); n != 2 { + } else if n := f.row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Row(121).Count(); n != 1 { + } else if n := f.row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify count on row. - if n := f.Row(1000).Count(); n != 1 { + if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } -// Ensure a fragment can set & read a field value. -func TestFragment_SetFieldValue(t *testing.T) { +// Ensure a fragment can set & read a value. +func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 3829 { t.Fatalf("unexpected value: %d", value) @@ -119,7 +115,7 @@ func TestFragment_SetFieldValue(t *testing.T) { } // Setting value should return no change. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if changed { t.Fatal("expected no change") @@ -127,25 +123,25 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 16, 3829); err != nil { + if changed, err := f.setValue(100, 16, 3829); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Overwriting value should overwrite all bits. - if changed, err := f.SetFieldValue(100, 16, 2028); err != nil { + if changed, err := f.setValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Read value. - if value, exists, err := f.FieldValue(100, 16); err != nil { + if value, exists, err := f.value(100, 16); err != nil { t.Fatal(err) } else if value != 2028 { t.Fatalf("unexpected value: %d", value) @@ -155,18 +151,18 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("NotExists", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set value. - if changed, err := f.SetFieldValue(100, 10, 20); err != nil { + if changed, err := f.setValue(100, 10, 20); err != nil { t.Fatal(err) } else if !changed { t.Fatal("expected change") } // Non-existent value. - if value, exists, err := f.FieldValue(100, 11); err != nil { + if value, exists, err := f.value(100, 11); err != nil { t.Fatal(err) } else if value != 0 { t.Fatalf("unexpected value: %d", value) @@ -185,7 +181,7 @@ func TestFragment_SetFieldValue(t *testing.T) { values[i] = values[i] % (1 << bitDepth) } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. @@ -195,14 +191,14 @@ func TestFragment_SetFieldValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil { + if _, err := f.setValue(columnID, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.FieldValue(columnID, bitDepth) + v, exists, err := f.value(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { @@ -219,26 +215,26 @@ func TestFragment_SetFieldValue(t *testing.T) { }) } -// Ensure a fragment can sum field values. -func TestFragment_FieldSum(t *testing.T) { +// Ensure a fragment can sum values. +func TestFragment_Sum(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { + if sum, n, err := f.sum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -248,7 +244,7 @@ func TestFragment_FieldSum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.sum(NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -258,45 +254,45 @@ func TestFragment_FieldSum(t *testing.T) { }) } -// Ensure a fragment can find the min and max of field values. -func TestFragment_FieldMinMax(t *testing.T) { +// Ensure a fragment can find the min and max of values. +func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(7000, bitDepth, 0); err != nil { t.Fatal(err) } t.Run("Min", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 0, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + if min, cnt, err := f.min(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -308,19 +304,19 @@ func TestFragment_FieldMinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { tests := []struct { - filter *pilosa.Row + filter *Row exp uint64 cnt uint64 }{ {filter: nil, exp: 2818, cnt: 2}, - {filter: pilosa.NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, - {filter: pilosa.NewRow(2000, 4000), exp: 300, cnt: 2}, - {filter: pilosa.NewRow(1), exp: 0, cnt: 0}, - {filter: pilosa.NewRow(1000), exp: 382, cnt: 1}, - {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, + {filter: NewRow(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: NewRow(2000, 4000), exp: 300, cnt: 2}, + {filter: NewRow(1), exp: 0, cnt: 0}, + {filter: NewRow(1000), exp: 382, cnt: 1}, + {filter: NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + if max, cnt, err := f.max(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -331,27 +327,27 @@ func TestFragment_FieldMinMax(t *testing.T) { }) } -// Ensure a fragment query for matching fields. -func TestFragment_FieldRange(t *testing.T) { +// Ensure a fragment query for matching values. +func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -359,22 +355,22 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("NEQ", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.rangeOp(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -382,47 +378,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields less than (ending with set column). - if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { + // Query for values less than (ending with set column). + if b, err := f.rangeOp(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than (ending with unset column). - if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { + // Query for values less than (ending with unset column). + if b, err := f.rangeOp(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with set column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { + // Query for values less than or equal to (ending with set column). + if b, err := f.rangeOp(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with unset column). - if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { + // Query for values less than or equal to (ending with unset column). + if b, err := f.rangeOp(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -430,47 +426,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { + // Query for values greater than (ending with unset bit). + if b, err := f.rangeOp(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { + // Query for values greater than (ending with set bit). + if b, err := f.rangeOp(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { + // Query for values greater than or equal to (ending with unset bit). + if b, err := f.rangeOp(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { + // Query for values greater than or equal to (ending with set bit). + if b, err := f.rangeOp(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -478,47 +474,47 @@ func TestFragment_FieldRange(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.setValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.setValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.setValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.setValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.setValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.setValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { + // Query for values greater than (ending with unset column). + if b, err := f.rangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { + // Query for values greater than (ending with set column). + if b, err := f.rangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset column). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { + // Query for values greater than or equal to (ending with unset column). + if b, err := f.rangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set column). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { + // Query for values greater than or equal to (ending with set column). + if b, err := f.rangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -528,50 +524,50 @@ func TestFragment_FieldRange(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.clearBit(1000, 1); err != nil { t.Fatal(err) } // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 1 { + } else if n := f.row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20); err != nil { + if _, err := f.setBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38); err != nil { + } else if _, err := f.setBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37); err != nil { + } else if _, err := f.setBit(2, 37); err != nil { t.Fatal(err) } // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(rowID, columnID uint64) error { + if err := f.forEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { @@ -586,75 +582,75 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil { + if pairs, err := f.top(TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[1] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + f.mustSetBits(100, 1, 3, 200) + f.mustSetBits(101, 1) + f.mustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. - f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) - f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) + f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": int64(10)}) + f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": int64(20)}) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{ + if pairs, err := f.top(TopOptions{ N: 2, - FilterField: "x", + FilterName: "x", FilterValues: []interface{}{int64(10), int64(15), int64(20)}, }); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[0] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 1}) { + } else if pairs[1] != (Pair{ID: 101, Count: 1}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on various rows. - f.MustSetBits(100, 1, 10, 11, 12) // one intersection - f.MustSetBits(101, 1, 2, 3, 4) // three intersections - f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections - f.MustSetBits(103, 1000, 1001, 1002) // no intersection + f.mustSetBits(100, 1, 10, 11, 12) // one intersection + f.mustSetBits(101, 1, 2, 3, 4) // three intersections + f.mustSetBits(102, 1, 2, 4, 5, 6) // two intersections + f.mustSetBits(103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 3}, {ID: 102, Count: 2}, {ID: 100, Count: 1}, @@ -669,11 +665,11 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Create an intersecting input row. - src := pilosa.NewRow( + src := NewRow( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) @@ -681,15 +677,15 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { - f.MustSetBits(i, j) + f.mustSetBits(i, j) } } f.RecalculateCache() // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 999, Count: 19}, {ID: 998, Count: 18}, {ID: 997, Count: 17}, @@ -707,18 +703,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs, []Pair{ {ID: 101, Count: 4}, {ID: 100, Count: 3}, }) { @@ -728,18 +724,18 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeNone) defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { + if pairs, err := f.top(TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{}) { + } else if !reflect.DeepEqual(pairs, []Pair{}) { t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs)) } } @@ -750,17 +746,17 @@ func TestFragment_TopN_CacheSize(t *testing.T) { cacheSize := uint32(3) // Create Index. - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked, CacheSize: cacheSize}) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -773,38 +769,34 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Close the storage so we can re-open it without encountering a flock. frag.Close() - f := &test.Fragment{ - Fragment: frag, - RowAttrStore: test.MustOpenAttrStore(), - } - f.Fragment.RowAttrStore = f.RowAttrStore + f := frag if err := f.Open(); err != nil { panic(err) } defer f.Close() // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) - f.MustSetBits(103, 8, 9, 10, 11, 12, 13) - f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) - f.MustSetBits(105, 10, 11) + f.mustSetBits(100, 1, 2, 3) + f.mustSetBits(101, 4, 5, 6, 7) + f.mustSetBits(102, 8, 9, 10, 11, 12) + f.mustSetBits(103, 8, 9, 10, 11, 12, 13) + f.mustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.mustSetBits(105, 10, 11) f.RecalculateCache() - p := []pilosa.Pair{ + p := []Pair{ {ID: 104, Count: 7}, {ID: 103, Count: 6}, {ID: 102, Count: 5}, } // Retrieve top rows. - if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { + if pairs, err := f.top(TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { t.Fatalf("TopN count cannot exceed cache size: %d", cacheSize) - } else if pairs[0] != (pilosa.Pair{ID: 104, Count: 7}) { + } else if pairs[0] != (Pair{ID: 104, Count: 7}) { t.Fatalf("unexpected pair(0): %v", pairs) } else if !reflect.DeepEqual(pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(pairs)) @@ -813,14 +805,14 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200); err != nil { + if _, err := f.setBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { + } else if _, err := f.setBit(HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -832,14 +824,14 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Retrieve initial checksum. - var prev []pilosa.FragmentBlock + var prev []FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0); err != nil { + if _, err := f.setBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -849,7 +841,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different row. - if _, err := f.SetBit(20, 0); err != nil { + if _, err := f.setBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -859,7 +851,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different column. - if _, err := f.SetBit(20, 100); err != nil { + if _, err := f.setBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -870,11 +862,11 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1); err != nil { + if _, err := f.setBit(100, 1); err != nil { t.Fatal(err) } @@ -888,31 +880,31 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeLRU) defer f.Close() // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the fragment. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) } // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.LRUCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*LRUCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -920,17 +912,17 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { - index := test.MustOpenIndex() + index := mustOpenIndex() defer index.Close() - // Create frame. - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + // Create field. + field, err := index.CreateFieldIfNotExists("f", FieldOptions{CacheType: CacheTypeRanked}) if err != nil { t.Fatal(err) } // Create view. - view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard) + view, err := field.CreateViewIfNotExists(ViewStandard) if err != nil { t.Fatal(err) } @@ -943,29 +935,29 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.setBit(i, 0); err != nil { t.Fatal(err) } } // Verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } // Reopen the index. - if err := index.Reopen(); err != nil { + if err := index.reopen(); err != nil { t.Fatal(err) } // Re-fetch fragment. - f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0) + f = index.Field("f").View(ViewStandard).Fragment(0) // Re-verify correct cache type and size. - if cache, ok := f.Cache().(*pilosa.RankCache); !ok { - t.Fatalf("unexpected cache: %T", f.Cache()) + if cache, ok := f.cache.(*RankCache); !ok { + t.Fatalf("unexpected cache: %T", f.cache) } else if cache.Len() != 1000 { t.Fatalf("unexpected cache len: %d", cache.Len()) } @@ -973,20 +965,20 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f0 := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1); err != nil { + if _, err := f0.setBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2); err != nil { + } else if _, err := f0.setBit(1000, 2); err != nil { t.Fatal(err) - } else if _, err := f0.ClearBit(1000, 1); err != nil { + } else if _, err := f0.clearBit(1000, 1); err != nil { t.Fatal(err) } // Verify cache is populated. - if n := f0.Cache().Len(); n != 1 { + if n := f0.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } @@ -998,7 +990,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f1 := mustOpenFragment("i", "f", ViewStandard, 0, "") if rn, err := f1.ReadFrom(&buf); err != nil { t.Fatal(err) } else if wn != rn { @@ -1006,21 +998,21 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify cache is in other fragment. - if n := f1.Cache().Len(); n != 1 { + if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size: %d", n) } // Verify data in other fragment. - if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns: %+v", a) } // Close and reopen the fragment & verify the data. - if err := f1.Reopen(); err != nil { + if err := f1.reopen(); err != nil { t.Fatal(err) - } else if n := f1.Cache().Len(); n != 1 { + } else if n := f1.cache.Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1031,7 +1023,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { } // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := NewFragment(*FragmentPath, "i", "f", ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1047,18 +1039,18 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() f.MaxOpN = math.MaxInt32 // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i)); err != nil { + if _, err := f.setBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i)); err != nil { + if _, err := f.setBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -1071,67 +1063,67 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { + if n := f.row(1).IntersectionCount(f.row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } } func TestFragment_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) } } func TestFragment_Zero_Tanimoto(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) + f := mustOpenFragment("i", "f", ViewStandard, 0, CacheTypeRanked) defer f.Close() - src := pilosa.NewRow(1, 2, 3) + src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + f.mustSetBits(100, 1, 3, 2, 200) + f.mustSetBits(101, 1, 3) + f.mustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() - if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { + if pairs, err := f.top(TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { t.Fatal(err) } else if len(pairs) != 3 { t.Fatalf("unexpected count: %d", len(pairs)) - } else if pairs[0] != (pilosa.Pair{ID: 100, Count: 3}) { + } else if pairs[0] != (Pair{ID: 100, Count: 3}) { t.Fatalf("unexpected pair(0): %v", pairs[0]) - } else if pairs[1] != (pilosa.Pair{ID: 101, Count: 2}) { + } else if pairs[1] != (Pair{ID: 101, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[1]) - } else if pairs[2] != (pilosa.Pair{ID: 102, Count: 2}) { + } else if pairs[2] != (Pair{ID: 102, Count: 2}) { t.Fatalf("unexpected pair(1): %v", pairs[2]) } } func TestFragment_Snapshot_Run(t *testing.T) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Set bits on the fragment. for i := uint64(1); i < 3; i++ { - if _, err := f.SetBit(1000, i); err != nil { + if _, err := f.setBit(1000, i); err != nil { t.Fatal(err) } } @@ -1139,14 +1131,14 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. - if err := f.Reopen(); err != nil { + if err := f.reopen(); err != nil { t.Fatal(err) - } else if n := f.Row(1000).Count(); n != 2 { + } else if n := f.row(1000).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -1158,7 +1150,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { b.ReportAllocs() // Open the fragment specified by the path. - f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0) + f := NewFragment(*FragmentPath, "i", "f", ViewStandard, 0) if err := f.Open(); err != nil { b.Fatal(err) } @@ -1177,7 +1169,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { } func BenchmarkFragment_FullSnapshot(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() // Generate some intersecting data. maxX := 1048576 / 2 @@ -1195,7 +1187,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { val += 2 i++ } - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } if row > max { @@ -1214,7 +1206,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { } func BenchmarkFragment_Import(b *testing.B) { - f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + f := mustOpenFragment("i", "f", ViewStandard, 0, "") defer f.Close() maxX := 1048576 * 5 * 2 sz := maxX @@ -1236,8 +1228,53 @@ func BenchmarkFragment_Import(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := f.Import(rows, cols); err != nil { + if err := f.bulkImport(rows, cols); err != nil { b.Fatalf("Error Building Sample: %s", err) } } } + +///////////////////////////////////////////////////////////////////// + +// mustOpenFragment returns a new instance of Fragment with a temporary path. +func mustOpenFragment(index, field, view string, slice uint64, cacheType string) *Fragment { + file, err := ioutil.TempFile("", "pilosa-fragment-") + if err != nil { + panic(err) + } + file.Close() + + if cacheType == "" { + cacheType = defaultCacheType + } + + f := NewFragment(file.Name(), index, field, view, slice) + f.CacheType = cacheType + f.RowAttrStore = newMemAttrStore() + + if err := f.Open(); err != nil { + panic(err) + } + return f +} + +// Reopen closes the fragment and reopens it as a new instance. +func (f *Fragment) reopen() error { + if err := f.Close(); err != nil { + return err + } + if err := f.Open(); err != nil { + return err + } + return nil +} + +// mustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (f *Fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.setBit(rowID, columnID); err != nil { + panic(err) + } + } +} diff --git a/frame.go b/frame.go deleted file mode 100644 index ab005edcc..000000000 --- a/frame.go +++ /dev/null @@ -1,1156 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "sync" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" - "github.com/pkg/errors" -) - -// Default frame settings. -const ( - DefaultCacheType = CacheTypeRanked - - // Default ranked frame cache - DefaultCacheSize = 50000 -) - -// Frame represents a container for views. -type Frame struct { - mu sync.RWMutex - path string - index string - name string - - views map[string]*View - - // Row attribute storage and cache - rowAttrStore AttrStore - - broadcaster Broadcaster - Stats StatsClient - - // Frame options. - cacheType string - cacheSize uint32 - timeQuantum TimeQuantum - fields []*Field - - Logger Logger -} - -// NewFrame returns a new instance of frame. -func NewFrame(path, index, name string) (*Frame, error) { - err := ValidateName(name) - if err != nil { - return nil, err - } - - return &Frame{ - path: path, - index: index, - name: name, - - views: make(map[string]*View), - - rowAttrStore: NopAttrStore, - - broadcaster: NopBroadcaster, - Stats: NopStatsClient, - - cacheType: DefaultCacheType, - cacheSize: DefaultCacheSize, - //timeQuantum - //fields - - Logger: NopLogger, - }, nil -} - -// Name returns the name the frame was initialized with. -func (f *Frame) Name() string { return f.name } - -// Index returns the index name the frame was initialized with. -func (f *Frame) Index() string { return f.index } - -// Path returns the path the frame was initialized with. -func (f *Frame) Path() string { return f.path } - -// RowAttrStore returns the attribute storage. -func (f *Frame) RowAttrStore() AttrStore { return f.rowAttrStore } - -// MaxSlice returns the max slice in the frame. -func (f *Frame) MaxSlice() uint64 { - f.mu.RLock() - defer f.mu.RUnlock() - - var max uint64 - for _, view := range f.views { - if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { - max = viewMaxSlice - } - } - return max -} - -// CacheType returns the caching mode for the frame. -func (f *Frame) CacheType() string { - return f.cacheType -} - -// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. -// defaults to DefaultCacheSize 50000 -func (f *Frame) SetCacheSize(v uint32) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Ignore if no change occurred. - if v == 0 || f.cacheSize == v { - return nil - } - - // Persist meta data to disk on change. - f.cacheSize = v - if err := f.saveMeta(); err != nil { - return errors.Wrap(err, "saving") - } - - return nil -} - -// CacheSize returns the ranked frame cache size. -func (f *Frame) CacheSize() uint32 { - f.mu.Lock() - v := f.cacheSize - f.mu.Unlock() - return v -} - -// Options returns all options for this frame. -func (f *Frame) Options() FrameOptions { - f.mu.RLock() - defer f.mu.RUnlock() - return f.options() -} - -func (f *Frame) options() FrameOptions { - return FrameOptions{ - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: f.timeQuantum, - Fields: f.fields, - } -} - -// Open opens and initializes the frame. -func (f *Frame) Open() error { - if err := func() error { - // Ensure the frame's path exists. - if err := os.MkdirAll(f.path, 0777); err != nil { - return errors.Wrap(err, "creating frame dir") - } - - if err := f.loadMeta(); err != nil { - return errors.Wrap(err, "loading meta") - } - - if err := f.openViews(); err != nil { - return errors.Wrap(err, "opening views") - } - - if err := f.rowAttrStore.Open(); err != nil { - return errors.Wrap(err, "opening attrstore") - } - - return nil - }(); err != nil { - f.Close() - return err - } - - return nil -} - -// openViews opens and initializes the views inside the frame. -func (f *Frame) openViews() error { - file, err := os.Open(filepath.Join(f.path, "views")) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "opening view directory") - } - defer file.Close() - - fis, err := file.Readdir(0) - if err != nil { - return errors.Wrap(err, "reading directory") - } - - for _, fi := range fis { - if !fi.IsDir() { - continue - } - - name := filepath.Base(fi.Name()) - view := f.newView(f.ViewPath(name), name) - if err := view.Open(); err != nil { - return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) - } - view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view - } - - return nil -} - -// loadMeta reads meta data for the frame, if any. -func (f *Frame) loadMeta() error { - var pb internal.FrameMeta - - // Read data from meta file. - buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) - if os.IsNotExist(err) { - f.cacheType = DefaultCacheType - f.cacheSize = DefaultCacheSize - f.timeQuantum = "" - //f.fields - return nil - } else if err != nil { - return errors.Wrap(err, "reading meta") - } else { - if err := proto.Unmarshal(buf, &pb); err != nil { - return errors.Wrap(err, "unmarshaling") - } - } - - // Copy metadata fields. - f.cacheType = pb.CacheType - if f.cacheType == "" { - f.cacheType = DefaultCacheType - } - f.cacheSize = pb.CacheSize - f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.fields = decodeFields(pb.Fields) - - return nil -} - -// saveMeta writes meta data for the frame. -func (f *Frame) saveMeta() error { - // Marshal metadata. - fo := f.options() - buf, err := proto.Marshal(fo.Encode()) - if err != nil { - return errors.Wrap(err, "marshaling") - } - - // Write to meta file. - if err := ioutil.WriteFile(filepath.Join(f.path, ".meta"), buf, 0666); err != nil { - return errors.Wrap(err, "writing meta") - } - - return nil -} - -// Close closes the frame and its views. -func (f *Frame) Close() error { - f.mu.Lock() - defer f.mu.Unlock() - - // Close the attribute store. - if f.rowAttrStore != nil { - _ = f.rowAttrStore.Close() - } - - // Close all views. - for _, view := range f.views { - if err := view.Close(); err != nil { - return err - } - } - f.views = make(map[string]*View) - - return nil -} - -// Field returns a field by name. -func (f *Frame) Field(name string) *Field { - f.mu.RLock() - defer f.mu.RUnlock() - for _, field := range f.fields { - if field.Name == name { - return field - } - } - return nil -} - -// Fields returns the fields on the frame. -func (f *Frame) Fields() []*Field { - f.mu.RLock() - defer f.mu.RUnlock() - return f.fields -} - -// HasField returns true if a field exists on the frame. -func (f *Frame) HasField(name string) bool { - for _, fld := range f.fields { - if fld.Name == name { - return true - } - } - return false -} - -// CreateField creates a new field on the frame. -func (f *Frame) CreateField(field *Field) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Append field. - if err := f.addField(field); err != nil { - return err - } - f.saveMeta() - return nil -} - -// addField adds a single field to fields. -func (f *Frame) addField(field *Field) error { - if err := ValidateField(field); err != nil { - return errors.Wrap(err, "validating field") - } else if f.HasField(field.Name) { - return ErrFieldExists - } - - // Add field to list. - f.fields = append(f.fields, field) - - // Sort fields by name. - sort.Slice(f.fields, func(i, j int) bool { - return f.fields[i].Name < f.fields[j].Name - }) - - return nil -} - -// GetFields returns a list of all the fields in the frame. -func (f *Frame) GetFields() ([]*Field, error) { - f.mu.RLock() - defer f.mu.RUnlock() - - err := f.loadMeta() - if err != nil { - return nil, errors.Wrap(err, "loading meta") - } - - return f.fields, nil -} - -// DeleteField deletes an existing field on the schema. -func (f *Frame) DeleteField(name string) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Remove field. - if err := f.deleteField(name); err != nil { - return err - } - - // Remove views. - viewName := ViewFieldPrefix + name - if view := f.views[viewName]; view != nil { - delete(f.views, viewName) - - if err := view.Close(); err != nil { - return errors.Wrap(err, "closing view") - } else if err := os.RemoveAll(view.Path()); err != nil { - return errors.Wrap(err, "deleting directory") - } - } - - return nil -} - -// deleteField removes a single field from fields. -func (f *Frame) deleteField(name string) error { - for i, field := range f.fields { - if field.Name == name { - copy(f.fields[i:], f.fields[i+1:]) - f.fields, f.fields[len(f.fields)-1] = f.fields[:len(f.fields)-1], nil - return nil - } - } - return ErrFieldNotFound -} - -// TimeQuantum returns the time quantum for the frame. -func (f *Frame) TimeQuantum() TimeQuantum { - f.mu.Lock() - defer f.mu.Unlock() - return f.timeQuantum -} - -// SetTimeQuantum sets the time quantum for the frame. -func (f *Frame) SetTimeQuantum(q TimeQuantum) error { - f.mu.Lock() - defer f.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on frame. - f.timeQuantum = q - - // Persist meta data to disk. - if err := f.saveMeta(); err != nil { - return errors.Wrap(err, "saving meta") - } - - return nil -} - -// ViewPath returns the path to a view in the frame. -func (f *Frame) ViewPath(name string) string { - return filepath.Join(f.path, "views", name) -} - -// View returns a view in the frame by name. -func (f *Frame) View(name string) *View { - f.mu.RLock() - defer f.mu.RUnlock() - return f.view(name) -} - -func (f *Frame) view(name string) *View { return f.views[name] } - -// Views returns a list of all views in the frame. -func (f *Frame) Views() []*View { - f.mu.RLock() - defer f.mu.RUnlock() - - other := make([]*View, 0, len(f.views)) - for _, view := range f.views { - other = append(other, view) - } - return other -} - -// viewNames returns a list of all views (as a string) in the frame. -func (f *Frame) viewNames() []string { - f.mu.Lock() - defer f.mu.Unlock() - - other := make([]string, 0, len(f.views)) - for viewName, _ := range f.views { - other = append(other, viewName) - } - return other -} - -// RecalculateCaches recalculates caches on every view in the frame. -func (f *Frame) RecalculateCaches() { - for _, view := range f.Views() { - view.RecalculateCaches() - } -} - -// CreateViewIfNotExists returns the named view, creating it if necessary. -// Additionally, a CreateViewMessage is sent to the cluster. -func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { - - view, created, err := f.createViewIfNotExistsBase(name) - if err != nil { - return nil, err - } - - if created { - // Broadcast view creation to the cluster. - err = f.broadcaster.SendSync( - &internal.CreateViewMessage{ - Index: f.index, - Frame: f.name, - View: name, - }) - if err != nil { - return nil, errors.Wrap(err, "sending CreateView message") - } - } - - return view, nil -} - -// createViewIfNotExistsBase returns the named view, creating it if necessary. -// The returned bool indicates whether the view was created or not. -func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { - f.mu.Lock() - defer f.mu.Unlock() - - if view := f.views[name]; view != nil { - return view, false, nil - } - - view := f.newView(f.ViewPath(name), name) - - if err := view.Open(); err != nil { - return nil, false, errors.Wrap(err, "opening view") - } - view.RowAttrStore = f.rowAttrStore - f.views[view.Name()] = view - - return view, true, nil -} - -func (f *Frame) newView(path, name string) *View { - view := NewView(path, f.index, f.name, name, f.cacheSize) - view.cacheType = f.cacheType - view.Logger = f.Logger - view.RowAttrStore = f.rowAttrStore - view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name)) - view.broadcaster = f.broadcaster - return view -} - -// DeleteView removes the view from the frame. -func (f *Frame) DeleteView(name string) error { - view := f.views[name] - if view == nil { - return ErrInvalidView - } - - // Close data files before deletion. - if err := view.Close(); err != nil { - return errors.Wrap(err, "closing view") - } - - // Delete view directory. - if err := os.RemoveAll(view.Path()); err != nil { - return errors.Wrap(err, "deleting directory") - } - - delete(f.views, name) - - return nil -} - -// SetBit sets a bit on a view within the frame. -func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { - // Validate view name. - if !IsValidView(name) { - return false, ErrInvalidView - } - - // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(name) - if err != nil { - return changed, errors.Wrap(err, "creating view") - } - - // Set non-time bit. - if v, err := view.SetBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") - } else if v { - changed = v - } - - // Exit early if no timestamp is specified. - if t == nil { - return changed, nil - } - - // If a timestamp is specified then set bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) - } - - if c, err := view.SetBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) - } else if c { - changed = true - } - } - - return changed, nil -} - -// ClearBit clears a bit within the frame. -func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { - // Validate view name. - if !IsValidView(name) { - return false, ErrInvalidView - } - - // Retrieve view. Exit if it doesn't exist. - view, err := f.CreateViewIfNotExists(name) - if err != nil { - return changed, errors.Wrap(err, "creating view") - } - - // Clear non-time bit. - if v, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrap(err, "setting on view") - } else if v { - changed = v - } - - // Exit early if no timestamp is specified. - if t == nil { - return changed, nil - } - - // If a timestamp is specified then clear bits across all views for the quantum. - for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { - view, err := f.CreateViewIfNotExists(subname) - if err != nil { - return changed, errors.Wrapf(err, "creating view %s", subname) - } - - if c, err := view.ClearBit(rowID, colID); err != nil { - return changed, errors.Wrapf(err, "setting on view %s", subname) - } else if c { - changed = true - } - } - - return changed, nil -} - -// FieldValue reads a field value for a column. -func (f *Frame) FieldValue(columnID uint64, name string) (value int64, exists bool, err error) { - field := f.Field(name) - if field == nil { - return 0, false, ErrFieldNotFound - } - - // Fetch target view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, false, nil - } - - v, exists, err := view.FieldValue(columnID, field.BitDepth()) - if err != nil { - return 0, false, err - } else if !exists { - return 0, false, nil - } - return int64(v) + field.Min, true, nil -} - -// SetFieldValue sets a field value for a column. -func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (changed bool, err error) { - // Fetch field and validate value. - field := f.Field(name) - if field == nil { - return false, ErrFieldNotFound - } else if value < field.Min { - return false, ErrFieldValueTooLow - } else if value > field.Max { - return false, ErrFieldValueTooHigh - } - - // Fetch target view. - view, err := f.CreateViewIfNotExists(ViewFieldPrefix + name) - if err != nil { - return false, errors.Wrap(err, "creating view") - } - - // Determine base value to store. - baseValue := uint64(value - field.Min) - - return view.SetFieldValue(columnID, field.BitDepth(), baseValue) -} - -// FieldSum returns the sum and count for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldSum(filter *Row, name string) (sum, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil -} - -// FieldMin returns the min for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldMin(filter *Row, name string) (min, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vmin) + field.Min, int64(vcount), nil -} - -// FieldMax returns the max for a field. -// An optional filtering row can be provided. -func (f *Frame) FieldMax(filter *Row, name string) (max, count int64, err error) { - field := f.Field(name) - if field == nil { - return 0, 0, ErrFieldNotFound - } - - view := f.View(ViewFieldPrefix + name) - if view == nil { - return 0, 0, nil - } - - vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) - if err != nil { - return 0, 0, err - } - return int64(vmax) + field.Min, int64(vcount), nil -} - -func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Row, error) { - // Retrieve and validate field. - field := f.Field(name) - if field == nil { - return nil, ErrFieldNotFound - } else if predicate < field.Min || predicate > field.Max { - return nil, nil - } - - // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return nil, nil - } - - baseValue, outOfRange := field.BaseValue(op, predicate) - if outOfRange { - return NewRow(), nil - } - - return view.FieldRange(op, field.BitDepth(), baseValue) -} - -func (f *Frame) FieldRangeBetween(name string, predicateMin, predicateMax int64) (*Row, error) { - // Retrieve and validate field. - field := f.Field(name) - if field == nil { - return nil, ErrFieldNotFound - } else if predicateMin > predicateMax { - return nil, ErrInvalidBetweenValue - } - - // Retrieve field's view. - view := f.View(ViewFieldPrefix + name) - if view == nil { - return nil, nil - } - - baseValueMin, baseValueMax, outOfRange := field.BaseValueBetween(predicateMin, predicateMax) - if outOfRange { - return NewRow(), nil - } - - return view.FieldRangeBetween(field.BitDepth(), baseValueMin, baseValueMax) -} - -// Import bulk imports data. -func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { - // Determine quantum if timestamps are set. - q := f.TimeQuantum() - if hasTime(timestamps) && q == "" { - return errors.New("time quantum not set in either index or frame") - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importData) - for i := range rowIDs { - rowID, columnID := rowIDs[i], columnIDs[i] - var timestamp *time.Time - if len(timestamps) > i { - timestamp = timestamps[i] - } - - var standard []string - if timestamp == nil { - standard = []string{ViewStandard} - } else { - standard = ViewsByTime(ViewStandard, *timestamp, q) - // In order to match the logic of `SetBit()`, we want bits - // with timestamps to write to both time and standard views. - standard = append(standard, ViewStandard) - } - - // Attach bit to each standard view. - for _, name := range standard { - key := importKey{View: name, Slice: columnID / SliceWidth} - data := dataByFragment[key] - data.RowIDs = append(data.RowIDs, rowID) - data.ColumnIDs = append(data.ColumnIDs, columnID) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - view, err := f.CreateViewIfNotExists(key.View) - if err != nil { - return errors.Wrap(err, "creating view") - } - - frag, err := view.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return errors.Wrap(err, "creating view") - } - - if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { - return err - } - } - - return nil -} - -// ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - viewName := ViewFieldPrefix + fieldName - // Get the field so we know bitDepth. - field := f.Field(fieldName) - if field == nil { - return fmt.Errorf("Field does not exist: %s", fieldName) - } - - // Split import data by fragment. - dataByFragment := make(map[importKey]importValueData) - for i := range columnIDs { - columnID, value := columnIDs[i], values[i] - if int64(value) > field.Max { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooHigh, columnID, value) - } else if int64(value) < field.Min { - return fmt.Errorf("%v, columnID=%v, value=%v", ErrFieldValueTooLow, columnID, value) - } - - // Attach value to each field view. - for _, name := range []string{viewName} { - key := importKey{View: name, Slice: columnID / SliceWidth} - data := dataByFragment[key] - data.ColumnIDs = append(data.ColumnIDs, columnID) - data.Values = append(data.Values, value) - dataByFragment[key] = data - } - } - - // Import into each fragment. - for key, data := range dataByFragment { - - // The view must already exist (i.e. we can't create it) - // because we need to know bitDepth (based on min/max value). - view, err := f.CreateViewIfNotExists(key.View) - if err != nil { - return errors.Wrap(err, "creating view") - } - - frag, err := view.CreateFragmentIfNotExists(key.Slice) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - baseValues := make([]uint64, len(data.Values)) - for i, value := range data.Values { - baseValues[i] = uint64(value - field.Min) - } - - if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil { - return err - } - } - - return nil -} - -// encodeFrames converts a into its internal representation. -func encodeFrames(a []*Frame) []*internal.Frame { - other := make([]*internal.Frame, len(a)) - for i := range a { - other[i] = encodeFrame(a[i]) - } - return other -} - -// encodeFrame converts f into its internal representation. -func encodeFrame(f *Frame) *internal.Frame { - fo := f.options() - return &internal.Frame{ - Name: f.name, - Meta: fo.Encode(), - Views: f.viewNames(), - } -} - -type frameSlice []*Frame - -func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameSlice) Len() int { return len(p) } -func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } - -// FrameInfo represents schema information for a frame. -type FrameInfo struct { - Name string `json:"name"` - Options FrameOptions `json:"options"` - Views []*ViewInfo `json:"views,omitempty"` -} - -type frameInfoSlice []*FrameInfo - -func (p frameInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p frameInfoSlice) Len() int { return len(p) } -func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } - -// FrameOptions represents options to set when initializing a frame. -type FrameOptions struct { - CacheType string `json:"cacheType,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*Field `json:"fields,omitempty"` -} - -// Encode converts o into its internal representation. -func (o *FrameOptions) Encode() *internal.FrameMeta { - return encodeFrameOptions(o) -} - -func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { - if o == nil { - return nil - } - return &internal.FrameMeta{ - CacheType: o.CacheType, - CacheSize: o.CacheSize, - TimeQuantum: string(o.TimeQuantum), - Fields: encodeFields(o.Fields), - } -} - -func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { - if options == nil { - return nil - } - return &FrameOptions{ - CacheType: options.CacheType, - CacheSize: options.CacheSize, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Fields: decodeFields(options.Fields), - } -} - -// List of field data types. -const ( - FieldTypeInt = "int" -) - -func IsValidFieldType(v string) bool { - switch v { - case FieldTypeInt: - return true - default: - return false - } -} - -// Field represents a range field on a frame. -type Field struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -// BitDepth returns the number of bits required to store a value between min & max. -func (f *Field) BitDepth() uint { - for i := uint(0); i < 63; i++ { - if f.Max-f.Min < (1 << i) { - return i - } - } - return 63 -} - -// BaseValue adjusts the value to align with the range for Field for a certain -// operation type. -// Note: There is an edge case for GT and LT where this returns a baseValue -// that does not fully encompass the range. -// ex: Field.Min = 0, Field.Max = 1023 -// BaseValue(LT, 2000) returns 1023, which will perform "LT 1023" and effectively -// exclude any columns with value = 1023. -// Note that in this case (because the range uses the full BitDepth 0 to 1023), -// we can't simply return 1024. -// In order to make this work, we effectively need to change the operator to LTE. -// Executor.executeFieldRangeSlice() takes this into account and returns -// `frag.FieldNotNull(field.BitDepth())` in such instances. -func (f *Field) BaseValue(op pql.Token, value int64) (baseValue uint64, outOfRange bool) { - if op == pql.GT || op == pql.GTE { - if value > f.Max { - return baseValue, true - } else if value > f.Min { - baseValue = uint64(value - f.Min) - } - } else if op == pql.LT || op == pql.LTE { - if value < f.Min { - return baseValue, true - } else if value > f.Max { - baseValue = uint64(f.Max - f.Min) - } else { - baseValue = uint64(value - f.Min) - } - } else if op == pql.EQ || op == pql.NEQ { - if value < f.Min || value > f.Max { - return baseValue, true - } - baseValue = uint64(value - f.Min) - } - return baseValue, false -} - -// BaseValueBetween adjusts the min/max value to align with the range for Field. -func (f *Field) BaseValueBetween(min, max int64) (baseValueMin, baseValueMax uint64, outOfRange bool) { - if max < f.Min || min > f.Max { - return baseValueMin, baseValueMax, true - } - // Adjust min/max to range. - if min > f.Min { - baseValueMin = uint64(min - f.Min) - } - // Make sure the high value of the BETWEEN does not exceed BitDepth. - if max > f.Max { - baseValueMax = uint64(f.Max - f.Min) - } else if max > f.Min { - baseValueMax = uint64(max - f.Min) - } - return baseValueMin, baseValueMax, false -} - -func ValidateField(f *Field) error { - if f.Name == "" { - return ErrFieldNameRequired - } else if !IsValidFieldType(f.Type) { - return ErrInvalidFieldType - } else if f.Min > f.Max { - return ErrInvalidFieldRange - } - return nil -} - -func encodeFields(a []*Field) []*internal.Field { - if len(a) == 0 { - return nil - } - other := make([]*internal.Field, len(a)) - for i := range a { - other[i] = encodeField(a[i]) - } - return other -} - -func decodeFields(a []*internal.Field) []*Field { - if len(a) == 0 { - return nil - } - other := make([]*Field, len(a)) - for i := range a { - other[i] = decodeField(a[i]) - } - return other -} - -func encodeField(f *Field) *internal.Field { - if f == nil { - return nil - } - return &internal.Field{ - Name: f.Name, - Type: f.Type, - Min: int64(f.Min), - Max: int64(f.Max), - } -} - -func decodeField(f *internal.Field) *Field { - if f == nil { - return nil - } - return &Field{ - Name: f.Name, - Type: f.Type, - Min: f.Min, - Max: f.Max, - } -} - -// importBitSet represents slices of row and column ids. -// This is used to sort data during import. -type importBitSet struct { - rowIDs, columnIDs []uint64 -} - -func (p importBitSet) Swap(i, j int) { - p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i] - p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i] -} -func (p importBitSet) Len() int { return len(p.rowIDs) } -func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } - -// Cache types. -const ( - CacheTypeLRU = "lru" - CacheTypeRanked = "ranked" - CacheTypeNone = "none" -) - -// IsValidCacheType returns true if v is a valid cache type. -func IsValidCacheType(v string) bool { - switch v { - case CacheTypeLRU, CacheTypeRanked, CacheTypeNone: - return true - default: - return false - } -} diff --git a/frame_test.go b/frame_test.go deleted file mode 100644 index 78014bbce..000000000 --- a/frame_test.go +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "reflect" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/test" -) - -// Ensure frame can open and retrieve a view. -func TestFrame_CreateViewIfNotExists(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - // Create view. - view, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - // Retrieve existing view. - view2, err := f.CreateViewIfNotExists("v") - if err != nil { - t.Fatal(err) - } else if view != view2 { - t.Fatal("view mismatch") - } - - if view != f.View("v") { - t.Fatal("view mismatch") - } -} - -// Ensure frame can set its time quantum. -func TestFrame_SetTimeQuantum(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - // Set & retrieve time quantum. - if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload frame and verify that it is persisted. - if err := f.Reopen(); err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - -// Ensure a frame can set & read a field value. -func TestFrame_SetFieldValue(t *testing.T) { - t.Run("OK", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value on first field. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Set value on same column but different field. - if changed, err := f.SetFieldValue(100, "field1", 25); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { - t.Fatal(err) - } else if value != 21 { - t.Fatalf("unexpected value: %d", value) - } else if !exists { - t.Fatal("expected value to exist") - } - - // Setting value should return no change. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if changed { - t.Fatal("expected no change") - } - }) - - t.Run("Overwrite", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if changed, err := f.SetFieldValue(100, "field0", 21); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Set different value. - if changed, err := f.SetFieldValue(100, "field0", 23); err != nil { - t.Fatal(err) - } else if !changed { - t.Fatal("expected change") - } - - // Read value. - if value, exists, err := f.FieldValue(100, "field0"); err != nil { - t.Fatal(err) - } else if value != 23 { - t.Fatalf("unexpected value: %d", value) - } else if !exists { - t.Fatal("expected value to exist") - } - }) - - t.Run("ErrFieldNotFound", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "no_such_field", 21); err != pilosa.ErrFieldNotFound { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrFieldValueTooLow", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "field0", 15); err != pilosa.ErrFieldValueTooLow { - t.Fatalf("unexpected error: %s", err) - } - }) - - t.Run("ErrFieldValueTooHigh", func(t *testing.T) { - idx := test.MustOpenIndex() - defer idx.Close() - - f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, - }, - }) - if err != nil { - t.Fatal(err) - } - - // Set value. - if _, err := f.SetFieldValue(100, "field0", 31); err != pilosa.ErrFieldValueTooHigh { - t.Fatalf("unexpected error: %s", err) - } - }) -} - -func TestFrame_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - frame, err := pilosa.NewFrame(path, "i", ".meta") - if frame != nil { - t.Fatalf("unexpected frame name %s", err) - } -} - -// Ensure that frame name validation is consistent. -func TestFrame_NameValidation(t *testing.T) { - validFrameNames := []string{ - "foo", - "hyphen-ated", - "under_score", - "abc123", - "trailing_", - } - invalidFrameNames := []string{ - "", - "123abc", - "x.y", - "_foo", - "-bar", - "abc def", - "camelCase", - "UPPERCASE", - "a12345678901234567890123456789012345678901234567890123456789012345", - } - - path, err := ioutil.TempDir("", "pilosa-frame-") - if err != nil { - panic(err) - } - for _, name := range validFrameNames { - _, err := pilosa.NewFrame(path, "i", name) - if err != nil { - t.Fatalf("unexpected frame name: %s %s", name, err) - } - } - for _, name := range invalidFrameNames { - _, err := pilosa.NewFrame(path, "i", name) - if err == nil { - t.Fatalf("expected error on frame name: %s", name) - } - } -} - -// Ensure frame can open and retrieve a view. -func TestFrame_DeleteView(t *testing.T) { - f := test.MustOpenFrame() - defer f.Close() - - viewName := pilosa.ViewStandard + "_v" - - // Create view. - view, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == nil { - t.Fatal("expected view") - } - - err = f.DeleteView(viewName) - if err != nil { - t.Fatal(err) - } - - if f.View(viewName) != nil { - t.Fatal("view still exists in frame") - } - - // Recreate view with same name, verify that the old view was not reused. - view2, err := f.CreateViewIfNotExists(viewName) - if err != nil { - t.Fatal(err) - } else if view == view2 { - t.Fatal("failed to create new view") - } -} - -// Ensure a field can adjust to its baseValue. -func TestField_BaseValue(t *testing.T) { - f0 := &pilosa.Field{ - Name: "f0", - Type: pilosa.FieldTypeInt, - Min: -100, - Max: 900, - } - f1 := &pilosa.Field{ - Name: "f1", - Type: pilosa.FieldTypeInt, - Min: 0, - Max: 1000, - } - - f2 := &pilosa.Field{ - Name: "f2", - Type: pilosa.FieldTypeInt, - Min: 100, - Max: 1100, - } - - t.Run("Normal Condition", func(t *testing.T) { - - for _, tt := range []struct { - f *pilosa.Field - op pql.Token - val int64 - expBaseValue uint64 - expOutOfRange bool - }{ - // LT - {f0, pql.LT, 5, 105, false}, - {f0, pql.LT, -8, 92, false}, - {f0, pql.LT, -108, 0, true}, - {f0, pql.LT, 1005, 1000, false}, - {f0, pql.LT, 0, 100, false}, - - {f1, pql.LT, 5, 5, false}, - {f1, pql.LT, -8, 0, true}, - {f1, pql.LT, 1005, 1000, false}, - {f1, pql.LT, 0, 0, false}, - - {f2, pql.LT, 5, 0, true}, - {f2, pql.LT, -8, 0, true}, - {f2, pql.LT, 105, 5, false}, - {f2, pql.LT, 1105, 1000, false}, - - // GT - {f0, pql.GT, -105, 0, false}, - {f0, pql.GT, 5, 105, false}, - {f0, pql.GT, 905, 0, true}, - {f0, pql.GT, 0, 100, false}, - - {f1, pql.GT, 5, 5, false}, - {f1, pql.GT, -8, 0, false}, - {f1, pql.GT, 1005, 0, true}, - {f1, pql.GT, 0, 0, false}, - - {f2, pql.GT, 5, 0, false}, - {f2, pql.GT, -8, 0, false}, - {f2, pql.GT, 105, 5, false}, - {f2, pql.GT, 1105, 0, true}, - - // EQ - {f0, pql.EQ, -105, 0, true}, - {f0, pql.EQ, 5, 105, false}, - {f0, pql.EQ, 905, 0, true}, - {f0, pql.EQ, 0, 100, false}, - - {f1, pql.EQ, 5, 5, false}, - {f1, pql.EQ, -8, 0, true}, - {f1, pql.EQ, 1005, 0, true}, - {f1, pql.EQ, 0, 0, false}, - - {f2, pql.EQ, 5, 0, true}, - {f2, pql.EQ, -8, 0, true}, - {f2, pql.EQ, 105, 5, false}, - {f2, pql.EQ, 1105, 0, true}, - } { - bv, oor := tt.f.BaseValue(tt.op, tt.val) - if oor != tt.expOutOfRange { - t.Fatalf("baseValue calculation on %s op %s, expected outOfRange %v, got %v", tt.f.Name, tt.op, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(bv, tt.expBaseValue) { - t.Fatalf("baseValue calculation on %s, expected value %v, got %v", tt.f.Name, tt.expBaseValue, bv) - } - } - }) - - t.Run("Betwween Condition", func(t *testing.T) { - for _, tt := range []struct { - f *pilosa.Field - predMin int64 - predMax int64 - expBaseValueMin uint64 - expBaseValueMax uint64 - expOutOfRange bool - }{ - - {f0, -205, -105, 0, 0, true}, - {f0, -105, 80, 0, 180, false}, - {f0, 5, 20, 105, 120, false}, - {f0, 20, 1005, 120, 1000, false}, - {f0, 1005, 2000, 0, 0, true}, - - {f1, -105, -5, 0, 0, true}, - {f1, -5, 20, 0, 20, false}, - {f1, 5, 20, 5, 20, false}, - {f1, 20, 1005, 20, 1000, false}, - {f1, 1005, 2000, 0, 0, true}, - - {f2, 5, 95, 0, 0, true}, - {f2, 95, 120, 0, 20, false}, - {f2, 105, 120, 5, 20, false}, - {f2, 120, 1105, 20, 1000, false}, - {f2, 1105, 2000, 0, 0, true}, - } { - min, max, oor := tt.f.BaseValueBetween(tt.predMin, tt.predMax) - if oor != tt.expOutOfRange { - t.Fatalf("baseValueBetween calculation on %s, expected outOfRange %v, got %v", tt.f.Name, tt.expOutOfRange, oor) - } else if !reflect.DeepEqual(min, tt.expBaseValueMin) || !reflect.DeepEqual(max, tt.expBaseValueMax) { - t.Fatalf("baseValueBetween calculation on %s, expected min/max %v/%v, got %v/%v", tt.f.Name, tt.expBaseValueMin, tt.expBaseValueMax, min, max) - } - } - }) -} diff --git a/gossip/gossip.go b/gossip/gossip.go index 2da6e3e4c..9154e9fc7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -18,13 +18,12 @@ import ( "fmt" "io/ioutil" "log" + "net" "strconv" "strings" "sync" "time" - "golang.org/x/sync/errgroup" - "github.com/gogo/protobuf/proto" "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" @@ -35,7 +34,6 @@ import ( // Ensure GossipMemberSet implements interfaces. var _ pilosa.BroadcastReceiver = &GossipMemberSet{} -var _ pilosa.Gossiper = &GossipMemberSet{} var _ memberlist.Delegate = &GossipMemberSet{} // GossipMemberSet represents a gossip implementation of MemberSet using memberlist. @@ -213,7 +211,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe conf.BindAddr = host conf.BindPort = port conf.AdvertisePort = port - conf.AdvertiseAddr = pilosa.HostToIP(host) + conf.AdvertiseAddr = hostToIP(host) // conf.TCPTimeout = time.Duration(cfg.StreamTimeout) conf.SuspicionMult = cfg.SuspicionMult @@ -239,49 +237,6 @@ func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventRe return g, nil } -// SendSync implementation of the Broadcaster interface. -func (g *GossipMemberSet) SendSync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - mlist := g.memberlist - - // Direct sends the message directly to every node. - // An error from any node raises an error on the entire operation. - // - // Gossip uses the gossip protocol to eventually deliver the message - // to every node. - var eg errgroup.Group - for _, n := range mlist.Members() { - // Don't send the message to the local node. - if n == mlist.LocalNode() { - continue - } - node := n - eg.Go(func() error { - return mlist.SendToTCP(node, msg) - }) - } - return eg.Wait() -} - -// SendAsync implementation of the Gossiper interface. -func (g *GossipMemberSet) SendAsync(pb proto.Message) error { - msg, err := pilosa.MarshalMessage(pb) - if err != nil { - return fmt.Errorf("marshal message: %s", err) - } - - b := &broadcast{ - msg: msg, - notify: nil, - } - g.broadcasts.QueueBroadcast(b) - return nil -} - // NodeMeta implementation of the memberlist.Delegate interface. func (g *GossipMemberSet) NodeMeta(limit int) []byte { buf, err := proto.Marshal(pilosa.EncodeNode(g.node)) @@ -580,3 +535,21 @@ type Config struct { Nodes int `toml:"nodes"` ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` } + +// hostToIP converts host to an IP4 address based on net.LookupIP(). +func hostToIP(host string) string { + // if host is not an IP addr, check net.LookupIP() + if net.ParseIP(host) == nil { + hosts, err := net.LookupIP(host) + if err != nil { + return host + } + for _, h := range hosts { + // this restricts pilosa to IP4 + if h.To4() != nil { + return h.String() + } + } + } + return host +} diff --git a/handler.go b/handler.go index 9ddabaeeb..7c2b76a3f 100644 --- a/handler.go +++ b/handler.go @@ -1,1058 +1,8 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package pilosa import ( "encoding/json" - "expvar" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. - _ "net/http/pprof" - "reflect" - "runtime/debug" - "strconv" - "strings" - "time" - - "github.com/gogo/protobuf/proto" - "github.com/gorilla/handlers" - "github.com/gorilla/mux" - "github.com/pilosa/pilosa/internal" - - "github.com/pkg/errors" -) - -// Handler represents an HTTP handler. -type Handler struct { - Handler http.Handler - - FileSystem FileSystem - - Logger Logger - - // Keeps the query argument validators for each handler - validators map[string]*queryValidationSpec - - API *API - - AllowedOrigins []string -} - -// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. -// This is used for stats tagging. -var externalPrefixFlag = map[string]bool{ - "schema": true, - "query": true, - "import": true, - "export": true, - "index": true, - "frame": true, - "nodes": true, - "version": true, -} - -type errorResponse struct { - Error string `json:"error"` -} - -// HandlerOption is a functional option type for pilosa.Handler -type HandlerOption func(s *Handler) error - -func OptHandlerAllowedOrigins(origins []string) HandlerOption { - return func(h *Handler) error { - h.Handler = handlers.CORS( - handlers.AllowedOrigins(origins), - handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) - return nil - } -} - -// NewHandler returns a new instance of Handler with a default logger. -func NewHandler(opts ...HandlerOption) (*Handler, error) { - handler := &Handler{ - FileSystem: NopFileSystem, - Logger: NopLogger, - } - handler.Handler = NewRouter(handler) - handler.populateValidators() - - for _, opt := range opts { - err := opt(handler) - if err != nil { - return nil, errors.Wrap(err, "applying option") - } - } - - return handler, nil -} - -func (h *Handler) populateValidators() { - h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired() - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") - h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "slice") - h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "slice") -} - -func (h *Handler) queryArgValidator(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - key := mux.CurrentRoute(r).GetName() - if validator, ok := h.validators[key]; ok { - if err := validator.validate(r.URL.Query()); err != nil { - // TODO: Return the response depending on the Accept header - response := errorResponse{Error: err.Error()} - body, err := json.Marshal(response) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - http.Error(w, string(body), http.StatusBadRequest) - return - } - } - next.ServeHTTP(w, r) - }) -} - -// NewRouter creates a new mux http router. -func NewRouter(handler *Handler) *mux.Router { - router := mux.NewRouter() - router.HandleFunc("/", handler.handleWebUI).Methods("GET") - router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") - router.HandleFunc("/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") - router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") - router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") - router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") - router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") - //router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented. - router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") - router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") - router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") - router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") - - // TODO: Apply MethodNotAllowed statuses to all endpoints. - // Ideally this would be automatic, as described in this (wontfix) ticket: - // https://github.com/gorilla/mux/issues/6 - // For now we just do it for the most commonly used handler, /query - router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - - router.Use(handler.queryArgValidator) - return router -} - -func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) -} - -// ServeHTTP handles an HTTP request. -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - w.WriteHeader(http.StatusInternalServerError) - stack := debug.Stack() - msg := "PANIC: %s\n%s" - h.Logger.Printf(msg, err, stack) - fmt.Fprintf(w, msg, err, stack) - } - }() - - t := time.Now() - h.Handler.ServeHTTP(w, r) - dif := time.Since(t) - - // Calculate per request StatsD metrics when the handler is fully configured. - statsTags := make([]string, 0, 3) - - longQueryTime := h.API.LongQueryTime() - if longQueryTime > 0 && dif > longQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - if stats != nil { - stats.Histogram("http."+endpointName, float64(dif), 0.1) - } -} - -func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { - // If user is using curl, don't chuck HTML at them - if strings.HasPrefix(r.UserAgent(), "curl") { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the WebUI by visiting this URL in your browser.", http.StatusNotFound) - return - } - filesystem, err := h.FileSystem.New() - if err != nil { - _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") - return - } - http.FileServer(filesystem).ServeHTTP(w, r) -} - -// handleGetSchema handles GET /schema requests. -func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { - schema := h.API.Schema(r.Context()) - if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: schema, - }); err != nil { - h.Logger.Printf("write schema response error: %s", err) - } -} - -// handleGetStatus handles GET /status requests. -func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status := getStatusResponse{ - State: h.API.State(), - Nodes: h.API.Hosts(r.Context()), - LocalID: h.API.LocalID(), - } - if err := json.NewEncoder(w).Encode(status); err != nil { - h.Logger.Printf("write status response error: %s", err) - } -} - -func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { - info := h.API.Info() - if err := json.NewEncoder(w).Encode(info); err != nil { - h.Logger.Printf("write info response error: %s", err) - } -} - -type getSchemaResponse struct { - Indexes []*IndexInfo `json:"indexes"` -} - -type getStatusResponse struct { - State string `json:"state"` - Nodes []*Node `json:"nodes"` - LocalID string `json:"localID"` -} - -// handlePostQuery handles /query requests. -func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - // Parse incoming request. - req, err := h.readQueryRequest(r) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - // TODO: Remove - req.Index = mux.Vars(r)["index"] - - resp, err := h.API.Query(r.Context(), req) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Set appropriate status code, if there is an error. - if resp.Err != nil { - switch resp.Err { - case ErrTooManyWrites: - w.WriteHeader(http.StatusRequestEntityTooLarge) - default: - w.WriteHeader(http.StatusInternalServerError) - } - } - - // Write response back to client. - if err := h.writeQueryResponse(w, r, &resp); err != nil { - h.Logger.Printf("write query response error: %s", err) - } -} - -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.API.MaxSlices(r.Context()), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) - } -} - -type getSlicesMaxResponse struct { - Standard map[string]uint64 `json:"standard"` -} - -// handleGetIndexes handles GET /index request. -func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { - h.handleGetSchema(w, r) -} - -// handleGetIndex handles GET /index/ requests. -func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - index, err := h.API.Index(r.Context(), indexName) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - if err := json.NewEncoder(w).Encode(getIndexResponse{ - map[string]string{"name": index.Name()}, - }); err != nil { - h.Logger.Printf("write response error: %s", err) - } -} - -type getIndexResponse struct { - Index map[string]string `json:"index"` -} - -type postIndexRequest struct { - Options IndexOptions `json:"options"` -} - -//_postIndexRequest is necessary to avoid recursion while decoding. -type _postIndexRequest postIndexRequest - -// Custom Unmarshal JSON to validate request body when creating a new index. -func (p *postIndexRequest) UnmarshalJSON(b []byte) error { - - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshalling unexpected values") - } - - validIndexOptions := getValidOptions(IndexOptions{}) - err := validateOptions(m, validIndexOptions) - if err != nil { - return err - } - // Unmarshal expected values. - var _p _postIndexRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected values") - } - - p.Options = _p.Options - - return nil -} - -// Raise errors for any unknown key -func validateOptions(data map[string]interface{}, validIndexOptions []string) error { - for k, v := range data { - switch k { - case "options": - options, ok := v.(map[string]interface{}) - if !ok { - return errors.New("options is not map[string]interface{}") - } - for kk, vv := range options { - if !foundItem(validIndexOptions, kk) { - return fmt.Errorf("Unknown key: %v:%v", kk, vv) - } - } - default: - return fmt.Errorf("Unknown key: %v:%v", k, v) - } - } - return nil -} - -func foundItem(items []string, item string) bool { - for _, i := range items { - if item == i { - return true - } - } - return false -} - -type postIndexResponse struct{} - -// handleDeleteIndex handles DELETE /index request. -func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - err := h.API.DeleteIndex(r.Context(), indexName) - if err != nil { - h.Logger.Printf("problem deleting index: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteIndexResponse struct{} - -// handlePostIndex handles POST /index request. -func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the index - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) - if errors.Cause(err) == ErrIndexExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -// handlePostIndexAttrDiff handles POST /index/attr/diff requests. -func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req postIndexAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postIndexAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postIndexAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// handlePostFrame handles POST /frame request. -func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req postFrameRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err == io.EOF { - // If no data was provided (EOF), we still create the frame - // with default values. - } else if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrFrameExists: - http.Error(w, err.Error(), http.StatusConflict) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type _postFrameRequest postFrameRequest - -// Custom Unmarshal JSON to validate request body when creating a new frame. If there's new FrameOptions, -// adding it to validFrameOptions to make sure the new option is validated, otherwise the request will be failed -func (p *postFrameRequest) UnmarshalJSON(b []byte) error { - // m is an overflow map used to capture additional, unexpected keys. - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return errors.Wrap(err, "unmarshaling unexpected keys") - } - - validFrameOptions := getValidOptions(FrameOptions{}) - err := validateOptions(m, validFrameOptions) - if err != nil { - return err - } - - // Unmarshal expected values. - var _p _postFrameRequest - if err := json.Unmarshal(b, &_p); err != nil { - return errors.Wrap(err, "unmarshalling expected keys") - } - - p.Options = _p.Options - return nil - -} - -func getValidOptions(option interface{}) []string { - validOptions := []string{} - val := reflect.ValueOf(option) - for i := 0; i < val.Type().NumField(); i++ { - jsonTag := val.Type().Field(i).Tag.Get("json") - s := strings.Split(jsonTag, ",") - validOptions = append(validOptions, s[0]) - } - return validOptions -} - -type postFrameRequest struct { - Options FrameOptions `json:"options"` -} - -type postFrameResponse struct{} - -// handleDeleteFrame handles DELETE /frame request. -func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - err := h.API.DeleteFrame(r.Context(), indexName, frameName) - if err != nil { - if errors.Cause(err) == ErrIndexNotFound { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } - return - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type deleteFrameResponse struct{} - -// handlePostFrameField handles POST /frame/field request. -func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - // Decode request. - var req postFrameFieldRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - field := &Field{ - Name: fieldName, - Type: req.Type, - Min: req.Min, - Max: req.Max, - } - - if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFrameFieldRequest struct { - Type string `json:"type,omitempty"` - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} - -type postFrameFieldResponse struct{} - -// handleDeleteFrameField handles DELETE /frame/field request. -func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - fieldName := mux.Vars(r)["field"] - - if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - fields, err := h.API.Fields(r.Context(), indexName, frameName) - if err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFrameFieldsResponse{Fields: fields}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type getFrameFieldsResponse struct { - Fields []*Field `json:"fields,omitempty"` -} - -type deleteFrameFieldResponse struct{} - -// handlePostFrameAttrDiff handles POST /frame/attr/diff requests. -func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req postFrameAttrDiffRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) - if err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(postFrameAttrDiffResponse{ - Attrs: attrs, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type postFrameAttrDiffRequest struct { - Blocks []AttrBlock `json:"blocks"` -} - -type postFrameAttrDiffResponse struct { - Attrs map[uint64]map[string]interface{} `json:"attrs"` -} - -// readQueryRequest parses an query parameters from r. -func (h *Handler) readQueryRequest(r *http.Request) (*QueryRequest, error) { - switch r.Header.Get("Content-Type") { - case "application/x-protobuf": - return h.readProtobufQueryRequest(r) - default: - return h.readURLQueryRequest(r) - } -} - -// readProtobufQueryRequest parses query parameters in protobuf from r. -func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, error) { - // Slurp the body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - - // Unmarshal into object. - var req internal.QueryRequest - if err := proto.Unmarshal(body, &req); err != nil { - return nil, errors.Wrap(err, "unmarshalling") - } - - return decodeQueryRequest(&req), nil -} - -// readURLQueryRequest parses query parameters from URL parameters from r. -func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { - q := r.URL.Query() - - // Parse query string. - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - query := string(buf) - - // Parse list of slices. - slices, err := parseUint64Slice(q.Get("slices")) - if err != nil { - return nil, errors.New("invalid slice argument") - } - - return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", - ExcludeColumns: q.Get("excludeColumns") == "true", - }, nil -} - -// writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - return h.writeProtobufQueryResponse(w, resp) - } - return h.writeJSONQueryResponse(w, resp) -} - -// writeProtobufQueryResponse writes the response from the executor to w as protobuf. -func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { - return errors.Wrap(err, "marshalling") - } else if _, err := w.Write(buf); err != nil { - return errors.Wrap(err, "writing") - } - return nil -} - -// writeJSONQueryResponse writes the response from the executor to w as JSON. -func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *QueryResponse) error { - return json.NewEncoder(w).Encode(resp) -} - -// handlePostImport handles /import requests. -func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.Import(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handlePostImportValue handles /import-value requests. -func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } else if r.Header.Get("Accept") != "application/x-protobuf" { - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - return - } - - // Read entire body. - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - var req internal.ImportValueRequest - if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ImportValue(r.Context(), req); err != nil { - switch errors.Cause(err) { - case ErrIndexNotFound: - fallthrough - case ErrFrameNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) - if e != nil { - http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) - return - } - - // Write response. - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - } - w.Write(buf) -} - -// handleGetExport handles /export requests. -func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { - switch r.Header.Get("Accept") { - case "text/csv": - h.handleGetExportCSV(w, r) - default: - http.Error(w, "Not acceptable", http.StatusNotAcceptable) - } -} - -func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { - // Parse query parameters. - q := r.URL.Query() - index, frame := q.Get("index"), q.Get("frame") - - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "invalid slice", http.StatusBadRequest) - return - } - - if err = h.API.ExportCSV(r.Context(), index, frame, slice, w); err != nil { - switch errors.Cause(err) { - case ErrFragmentNotFound: - break - case ErrClusterDoesNotOwnSlice: - http.Error(w, err.Error(), http.StatusPreconditionFailed) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } -} - -// handleGetFragmentNodes handles /fragment/nodes requests. -func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - index := q.Get("index") - - // Read slice parameter. - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) - return - } - - // Retrieve fragment owner nodes. - nodes, err := h.API.SliceNodes(r.Context(), index, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Write to response. - if err := json.NewEncoder(w).Encode(nodes); err != nil { - h.Logger.Printf("json write error: %s", err) - } -} - -// handleGetFragmentBlockData handles GET /fragment/block/data requests. -func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - buf, err := h.API.FragmentBlockData(r.Context(), r.Body) - if err != nil { - if _, ok := err.(BadRequestError); ok { - http.Error(w, err.Error(), http.StatusBadRequest) - } else if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Write response. - w.Header().Set("Content-Type", "application/protobuf") - w.Header().Set("Content-Length", strconv.Itoa(len(buf))) - w.Write(buf) -} - -// handleGetFragmentBlocks handles GET /fragment/blocks requests. -func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), slice) - if err != nil { - if errors.Cause(err) == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ - Blocks: blocks, - }); err != nil { - h.Logger.Printf("block response encoding error: %s", err) - } -} - -type getFragmentBlocksResponse struct { - Blocks []FragmentBlock `json:"blocks"` -} - -// handleGetVersion handles /version requests. -func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - err := json.NewEncoder(w).Encode(struct { - Version string `json:"version"` - }{ - Version: h.API.Version(), - }) - if err != nil { - h.Logger.Printf("write version response error: %s", err) - } -} - -// QueryResult types. -const ( - QueryResultTypeNil uint32 = iota - QueryResultTypeRow - QueryResultTypePairs - QueryResultTypeValCount - QueryResultTypeUint64 - QueryResultTypeBool + "net" ) // QueryRequest represent a request to process a query. @@ -1081,19 +31,6 @@ type QueryRequest struct { Remote bool } -func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeRowAttrs: pb.ExcludeRowAttrs, - ExcludeColumns: pb.ExcludeColumns, - } - - return req -} - // QueryResponse represent a response from a processed query. type QueryResponse struct { // Result for each top-level query call. @@ -1123,234 +60,19 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { return json.Marshal(output) } -func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { - pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), - } - - for i := range resp.Results { - pb.Results[i] = &internal.QueryResult{} - - switch result := resp.Results[i].(type) { - case *Row: - pb.Results[i].Type = QueryResultTypeRow - pb.Results[i].Row = encodeRow(result) - case []Pair: - pb.Results[i].Type = QueryResultTypePairs - pb.Results[i].Pairs = encodePairs(result) - case ValCount: - pb.Results[i].Type = QueryResultTypeValCount - pb.Results[i].ValCount = encodeValCount(result) - case uint64: - pb.Results[i].Type = QueryResultTypeUint64 - pb.Results[i].N = result - case bool: - pb.Results[i].Type = QueryResultTypeBool - pb.Results[i].Changed = result - case nil: - pb.Results[i].Type = QueryResultTypeNil - } - } - - if resp.Err != nil { - pb.Err = resp.Err.Error() - } - - return pb +type Handler interface { + Serve(ln net.Listener, closing <-chan struct{}) + GetAPI() *API } -// parseUint64Slice returns a slice of uint64s from a comma-delimited string. -func parseUint64Slice(s string) ([]uint64, error) { - var a []uint64 - for _, str := range strings.Split(s, ",") { - // Ignore blanks. - if str == "" { - continue - } +type NopHandler struct{} - // Parse number. - num, err := strconv.ParseUint(str, 10, 64) - if err != nil { - return nil, errors.Wrap(err, "parsing int") - } - a = append(a, num) - } - return a, nil -} +func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {} -// errorString returns the string representation of err. -func errorString(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req setCoordinatorRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) - return - } - - oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) - } - return - } - // Encode response. - if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ - Old: oldNode, - New: newNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type setCoordinatorRequest struct { - ID string `json:"id"` -} - -type setCoordinatorResponse struct { - Old *Node `json:"old"` - New *Node `json:"new"` -} - -// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. -func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { - // Decode request. - var req removeNodeRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - removeNode, err := h.API.RemoveNode(req.ID) - if err != nil { - if errors.Cause(err) == ErrNodeIDNotExists { - http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) - } else { - http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(removeNodeResponse{ - Remove: removeNode, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type removeNodeRequest struct { - ID string `json:"id"` -} - -type removeNodeResponse struct { - Remove *Node `json:"remove"` -} - -// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. -func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { - err := h.API.ResizeAbort() - var msg string - if err != nil { - switch errors.Cause(err) { - case ErrNodeNotCoordinator: - http.Error(w, err.Error(), http.StatusBadRequest) - return - case ErrResizeNotRunning: - msg = err.Error() - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - // Encode response. - if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ - Info: msg, - }); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type clusterResizeAbortResponse struct { - Info string `json:"info"` -} - -func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.API.RecalculateCaches(r.Context()) - if err != nil { - http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} - -func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { - // Verify that request is only communicating over protobufs. - if r.Header.Get("Content-Type") != "application/x-protobuf" { - http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) - return - } - - err := h.API.ClusterMessage(r.Context(), r.Body) - if err != nil { - // TODO this was the previous behavior, but perhaps not everything is a bad request - http.Error(w, err.Error(), http.StatusBadRequest) - } - - if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type defaultClusterMessageResponse struct{} - -type queryValidationSpec struct { - required []string - args map[string]struct{} -} - -func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { - args := map[string]struct{}{} - for _, arg := range requiredArgs { - args[arg] = struct{}{} - } - - return &queryValidationSpec{ - required: requiredArgs, - args: args, - } -} - -func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { - for _, arg := range args { - s.args[arg] = struct{}{} - } - return s -} - -func (s queryValidationSpec) validate(query url.Values) error { - for _, req := range s.required { - if query.Get(req) == "" { - return errors.Errorf("%s is required", req) - } - } - for k := range query { - if _, ok := s.args[k]; !ok { - return errors.Errorf("%s is not a valid argument", k) - } - } +func (n *NopHandler) GetAPI() *API { return nil } + +func NewNopHandler() Handler { + return &NopHandler{} +} diff --git a/holder.go b/holder.go index 6d13159b9..316ca1244 100644 --- a/holder.go +++ b/holder.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "os" "path" "path/filepath" @@ -34,8 +33,8 @@ import ( ) const ( - // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. - DefaultCacheFlushInterval = 1 * time.Minute + // defaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. + defaultCacheFlushInterval = 1 * time.Minute // FileLimit is the maximum open file limit (ulimit -n) to automatically set. FileLimit = 262144 // (512^2) @@ -84,7 +83,7 @@ func NewHolder() *Holder { NewAttrStore: NewNopAttrStore, - CacheFlushInterval: DefaultCacheFlushInterval, + CacheFlushInterval: defaultCacheFlushInterval, Logger: NopLogger, } @@ -112,7 +111,8 @@ func (h *Holder) Open() error { } for _, fi := range fis { - if !fi.IsDir() { + // Skip files or hidden directories. + if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { continue } @@ -209,20 +209,20 @@ func (h *Holder) MaxSlices() map[string]uint64 { return a } -// Schema returns schema information for all indexes, frames, and views. +// Schema returns schema information for all indexes, fields, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name()} - for _, frame := range index.Frames() { - fi := &FrameInfo{Name: frame.Name(), Options: frame.Options()} - for _, view := range frame.Views() { - fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()}) + for _, field := range index.Fields() { + fi := &FieldInfo{Name: field.Name(), Options: field.Options()} + for _, view := range field.Views() { + fi.Views = append(fi.Views, &ViewInfo{Name: view.name}) } sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) + di.Fields = append(di.Fields, fi) } - sort.Sort(frameInfoSlice(di.Frames)) + sort.Sort(fieldInfoSlice(di.Fields)) a = append(a, di) } sort.Sort(indexInfoSlice(a)) @@ -238,16 +238,16 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { if err != nil { return errors.Wrap(err, "creating index") } - // Create frames that don't exist. - for _, f := range index.Frames { - opt := decodeFrameOptions(f.Meta) - frame, err := idx.CreateFrameIfNotExists(f.Name, *opt) + // Create fields that don't exist. + for _, f := range index.Fields { + opt := decodeFieldOptions(f.Meta) + field, err := idx.CreateFieldIfNotExists(f.Name, *opt) if err != nil { - return errors.Wrap(err, "creating frame") + return errors.Wrap(err, "creating field") } // Create views that don't exist. for _, v := range f.Views { - _, err := frame.CreateViewIfNotExists(v) + _, err := field.CreateViewIfNotExists(v) if err != nil { return errors.Wrap(err, "creating view") } @@ -339,12 +339,15 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { return nil, errors.Wrap(err, "creating") } + index.keys = opt.Keys + if err := index.Open(); err != nil { return nil, errors.Wrap(err, "opening") + } else if err := index.saveMeta(); err != nil { + return nil, errors.Wrap(err, "meta") } // Update options. - h.indexes[index.Name()] = index return index, nil @@ -390,27 +393,27 @@ func (h *Holder) DeleteIndex(name string) error { return nil } -// Frame returns the frame for an index and name. -func (h *Holder) Frame(index, name string) *Frame { +// Field returns the field for an index and name. +func (h *Holder) Field(index, name string) *Field { idx := h.Index(index) if idx == nil { return nil } - return idx.Frame(name) + return idx.Field(name) } -// View returns the view for an index, frame, and name. -func (h *Holder) View(index, frame, name string) *View { - f := h.Frame(index, frame) +// View returns the view for an index, field, and name. +func (h *Holder) View(index, field, name string) *View { + f := h.Field(index, field) if f == nil { return nil } return f.View(name) } -// Fragment returns the fragment for an index, frame & slice. -func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment { - v := h.View(index, frame, view) +// Fragment returns the fragment for an index, field & slice. +func (h *Holder) Fragment(index, field, view string, slice uint64) *Fragment { + v := h.View(index, field, view) if v == nil { return nil } @@ -435,9 +438,9 @@ func (h *Holder) monitorCacheFlush() { func (h *Holder) flushCaches() { for _, index := range h.Indexes() { - for _, frame := range index.Frames() { - for _, view := range frame.Views() { - for _, fragment := range view.Fragments() { + for _, field := range index.Fields() { + for _, view := range field.Views() { + for _, fragment := range view.allFragments() { select { case <-h.closing: return @@ -445,7 +448,7 @@ func (h *Holder) flushCaches() { } if err := fragment.FlushCache(); err != nil { - h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath()) + h.Logger.Printf("error flushing cache: err=%s, path=%s", err, fragment.cachePath()) } } } @@ -563,9 +566,8 @@ func (h *Holder) logStartup() error { type HolderSyncer struct { Holder *Holder - Node *Node - Cluster *Cluster - RemoteClient *http.Client + Node *Node + Cluster *Cluster // Stats Stats StatsClient @@ -600,15 +602,15 @@ func (s *HolderSyncer) SyncHolder() error { } tf := time.Now() - for _, fi := range di.Frames { + for _, fi := range di.Fields { // Verify syncer has not closed. if s.IsClosing() { return nil } - // Sync frame row attributes. - if err := s.syncFrame(di.Name, fi.Name); err != nil { - return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err) + // Sync field row attributes. + if err := s.syncField(di.Name, fi.Name); err != nil { + return fmt.Errorf("field sync error: index=%s, field=%s, err=%s", di.Name, fi.Name, err) } for _, vi := range fi.Views { @@ -619,7 +621,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { + if !s.Cluster.ownsSlice(s.Node.ID, di.Name, slice) { continue } @@ -630,11 +632,11 @@ func (s *HolderSyncer) SyncHolder() error { // Sync fragment if own it. if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil { - return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) + return fmt.Errorf("fragment sync error: index=%s, field=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err) } } } - s.Stats.Histogram("syncFrame", float64(time.Since(tf)), 1.0) + s.Stats.Histogram("syncField", float64(time.Since(tf)), 1.0) tf = time.Now() // reset tf } s.Stats.Histogram("syncIndex", float64(time.Since(ti)), 1.0) @@ -662,11 +664,9 @@ func (s *HolderSyncer) syncIndex(index string) error { // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ColumnAttrDiff(context.Background(), index, blks) + m, err := s.Cluster.InternalClient.ColumnAttrDiff(context.Background(), &node.URI, index, blks) if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { @@ -689,38 +689,36 @@ func (s *HolderSyncer) syncIndex(index string) error { return nil } -// syncFrame synchronizes frame attributes with the rest of the cluster. -func (s *HolderSyncer) syncFrame(index, name string) error { - // Retrieve frame reference. - f := s.Holder.Frame(index, name) +// syncField synchronizes field attributes with the rest of the cluster. +func (s *HolderSyncer) syncField(index, name string) error { + // Retrieve field reference. + f := s.Holder.Field(index, name) if f == nil { return nil } indexTag := fmt.Sprintf("index:%s", index) - frameTag := fmt.Sprintf("frame:%s", name) + fieldTag := fmt.Sprintf("field:%s", name) // Read block checksums. blks, err := f.RowAttrStore().Blocks() if err != nil { return errors.Wrap(err, "getting blocks") } - s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, frameTag}) + s.Stats.CountWithCustomTags("RowAttrStoreBlocks", int64(len(blks)), 1.0, []string{indexTag, fieldTag}) // Sync with every other host. for _, node := range Nodes(s.Cluster.Nodes).FilterID(s.Node.ID) { - client := NewInternalHTTPClientFromURI(&node.URI, s.RemoteClient) - // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.RowAttrDiff(context.Background(), index, name, blks) - if err == ErrFrameNotFound { - continue // frame not created remotely yet, skip + m, err := s.Cluster.InternalClient.RowAttrDiff(context.Background(), &node.URI, index, name, blks) + if err == ErrFieldNotFound { + continue // field not created remotely yet, skip } else if err != nil { return errors.Wrap(err, "getting differing blocks") } else if len(m) == 0 { continue } - s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, frameTag, node.ID}) + s.Stats.CountWithCustomTags("RowAttrDiff", int64(len(m)), 1.0, []string{indexTag, fieldTag, node.ID}) // Update local copy. if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { @@ -738,11 +736,11 @@ func (s *HolderSyncer) syncFrame(index, name string) error { } // syncFragment synchronizes a fragment with the rest of the cluster. -func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error { - // Retrieve local frame. - f := s.Holder.Frame(index, frame) +func (s *HolderSyncer) syncFragment(index, field, view string, slice uint64) error { + // Retrieve local field. + f := s.Holder.Field(index, field) if f == nil { - return ErrFrameNotFound + return ErrFieldNotFound } // Ensure view exists locally. @@ -759,13 +757,12 @@ func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) err // Sync fragments together. fs := FragmentSyncer{ - Fragment: frag, - Node: s.Node, - Cluster: s.Cluster, - Closing: s.Closing, - RemoteClient: s.RemoteClient, + Fragment: frag, + Node: s.Node, + Cluster: s.Cluster, + Closing: s.Closing, } - if err := fs.SyncFragment(); err != nil { + if err := fs.syncFragment(); err != nil { return errors.Wrap(err, "syncing fragment") } @@ -803,19 +800,19 @@ func (c *HolderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedSlices := c.Cluster.ContainsSlices(index.Name(), index.MaxSlice(), c.Node) + containedSlices := c.Cluster.containsSlices(index.Name(), index.MaxSlice(), c.Node) // Get the fragments registered in memory. - for _, frame := range index.Frames() { - for _, view := range frame.Views() { - for _, fragment := range view.Fragments() { - fragSlice := fragment.Slice() + for _, field := range index.Fields() { + for _, view := range field.Views() { + for _, fragment := range view.allFragments() { + fragSlice := fragment.slice // Ignore fragments that should be present. if uint64InSlice(fragSlice, containedSlices) { continue } // Delete fragment. - if err := view.DeleteFragment(fragSlice); err != nil { + if err := view.deleteFragment(fragSlice); err != nil { return errors.Wrap(err, "deleting fragment") } } diff --git a/holder_test.go b/holder_test.go index c7877e3ba..72a10b815 100644 --- a/holder_test.go +++ b/holder_test.go @@ -24,8 +24,8 @@ import ( "testing" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -91,7 +91,7 @@ func TestHolder_Open(t *testing.T) { } }) - t.Run("ErrFramePermission", func(t *testing.T) { + t.Run("ErrFieldPermission", func(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } @@ -100,7 +100,7 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -113,13 +113,13 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameMetaCorrupt", func(t *testing.T) { + t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -127,17 +127,17 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=loading meta: unmarshaling: unexpected EOF") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=loading meta: unmarshaling: unexpected EOF") { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) { + t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if _, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -145,7 +145,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening frames: open frame: name=bar, err=opening attrstore: opening storage: invalid database") { + if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "open index: name=foo, err=opening fields: open field: name=bar, err=opening attrstore: opening storage: invalid database") { t.Fatalf("unexpected error: %s", err) } }) @@ -159,9 +159,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -183,9 +183,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if _, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if _, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -208,11 +208,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -231,11 +229,9 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { - t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := h.Holder.Close(); err != nil { t.Fatal(err) @@ -257,11 +253,11 @@ func TestHolder_Open(t *testing.T) { if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if frame, err := idx.CreateFrame("bar", pilosa.FrameOptions{}); err != nil { + } else if field, err := idx.CreateField("bar", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if view, err := frame.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { + } else if view, err := field.CreateViewIfNotExists(pilosa.ViewStandard); err != nil { t.Fatal(err) - } else if _, err := view.SetBit(0, 0); err != nil { + } else if _, err := field.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } else if err := view.Fragment(0).FlushCache(); err != nil { t.Fatal(err) @@ -331,14 +327,8 @@ func TestHolder_DeleteIndex(t *testing.T) { defer hldr.Close() // Write bits to separate indexes. - f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) - if _, err := f0.SetBit(100, 200); err != nil { - t.Fatal(err) - } - f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0) - if _, err := f1.SetBit(100, 200); err != nil { - t.Fatal(err) - } + hldr.SetBit("i0", "f", 100, 200) + hldr.SetBit("i1", "f", 100, 200) // Ensure i0 exists. if _, err := os.Stat(hldr.IndexPath("i0")); err != nil { @@ -360,8 +350,19 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { + s := test.NewServer() + defer s.Close() + + uri, err := pilosa.NewURIFromAddress(s.URL) + if err != nil { + t.Fatal(err) + } + cluster := test.NewCluster(2) - client := server.GetHTTPClient(nil) + client := http.GetHTTPClient(nil) + httpClient := http.NewInternalClientFromURI(uri, client) + cluster.InternalClient = httpClient + // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -369,11 +370,9 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Create a remote holder wrapped by an HTTP hldr1 := test.MustOpenHolder() defer hldr1.Close() - s := test.NewServer() - defer s.Close() s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(client) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr1.Holder e.Node = cluster.Nodes[1] e.Cluster = cluster @@ -383,58 +382,35 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - uri, err := pilosa.NewURIFromAddress(s.URL) - if err != nil { - t.Fatal(err) - } - - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) cluster.Nodes[1].URI = *uri - // Create frames on nodes. + // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0, hldr1} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") } // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f0", 9, SliceWidth+5) - hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0) + // Set a bit to create the fragment. + hldr0.SetBit("y", "z", 0, 0) // Set data on the remote holder. - f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } + hldr1.SetBit("i", "f", 0, 4000) + hldr1.SetBit("i", "f", 3, 10) + hldr1.SetBit("i", "f", 120, 10) - f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { - t.Fatal(err) - } + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+4) + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+5) + hldr1.SetBit("y", "z", 10, (3*SliceWidth)+7) // Set highest slice. hldr0.Index("i").SetRemoteMaxSlice(1) @@ -442,11 +418,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Set up syncer. syncer := pilosa.HolderSyncer{ - Holder: hldr0.Holder, - Node: cluster.Nodes[0], - Cluster: cluster, - RemoteClient: server.GetHTTPClient(nil), - Stats: pilosa.NopStatsClient, + Holder: hldr0.Holder, + Node: cluster.Nodes[0], + Cluster: cluster, + Stats: pilosa.NopStatsClient, } if err := syncer.SyncHolder(); err != nil { @@ -455,29 +430,23 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -485,7 +454,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Ensure holder can clean up orphaned fragments. func TestHolderCleaner_CleanHolder(t *testing.T) { - cluster := test.NewCluster(2) + cluster := pilosa.NewTestCluster(2) // Create a local holder. hldr0 := test.MustOpenHolder() @@ -494,44 +463,28 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Mock 2-node, fully replicated cluster. cluster.ReplicaN = 2 - cluster.Nodes[0].URI = test.NewURIFromHostPort("localhost", 0) + cluster.Nodes[0].URI = pilosa.NewTestURIFromHostPort("localhost", 0) - // Create frames on nodes. + // Create fields on nodes. for _, hldr := range []*test.Holder{hldr0} { - hldr.MustCreateFrameIfNotExists("i", "f") - hldr.MustCreateFrameIfNotExists("i", "f0") - hldr.MustCreateFrameIfNotExists("y", "z") + hldr.MustCreateFieldIfNotExists("i", "f") + hldr.MustCreateFieldIfNotExists("i", "f0") + hldr.MustCreateFieldIfNotExists("y", "z") } // Set data on the local holder. - f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - if _, err := f.SetBit(0, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(0, 4000); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(2, 20); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(3, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(120, 10); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(200, 4); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f", 0, 10) + hldr0.SetBit("i", "f", 0, 4000) + hldr0.SetBit("i", "f", 2, 20) + hldr0.SetBit("i", "f", 3, 10) + hldr0.SetBit("i", "f", 120, 10) + hldr0.SetBit("i", "f", 200, 4) - f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1) - if _, err := f.SetBit(9, SliceWidth+5); err != nil { - t.Fatal(err) - } + hldr0.SetBit("i", "f0", 9, SliceWidth+5) - f = hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 2) - if _, err := f.SetBit(10, (2*SliceWidth)+4); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+5); err != nil { - t.Fatal(err) - } else if _, err := f.SetBit(10, (2*SliceWidth)+7); err != nil { - t.Fatal(err) - } + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+4) + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+5) + hldr0.SetBit("y", "z", 10, (2*SliceWidth)+7) // Set highest slice. hldr0.Index("i").SetRemoteMaxSlice(1) @@ -553,29 +506,23 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Columns() - if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) - } - if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := hldr.Row("i", "f0", 9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -596,26 +543,24 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { - f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected columns(%d/0): %+v", i, a) - } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := hldr.Row("i", "f", 2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected columns(%d/2): %+v", i, a) - } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/3): %+v", i, a) - } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := hldr.Row("i", "f", 120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected columns(%d/120): %+v", i, a) - } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected columns(%d/200): %+v", i, a) } - f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) + f := hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) if f != nil { t.Fatalf("expected fragment to be deleted: (%d/i/f0): %+v", i, f) } - f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + if a := hldr.Row("y", "z", 10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } diff --git a/http/client.go b/http/client.go new file mode 100644 index 000000000..d8f0d290d --- /dev/null +++ b/http/client.go @@ -0,0 +1,1039 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "sort" + "strconv" + + "crypto/tls" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" +) + +// ClientOptions represents the configuration for a InternalHTTPClient +type ClientOptions struct { + TLS *tls.Config +} + +// InternalClient represents a client to the Pilosa cluster. +type InternalClient struct { + defaultURI *pilosa.URI + + // The client to use for HTTP communication. + HTTPClient *http.Client +} + +// NewInternalClient returns a new instance of InternalClient to connect to host. +func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) { + if host == "" { + return nil, pilosa.ErrHostRequired + } + + uri, err := pilosa.NewURIFromAddress(host) + if err != nil { + return nil, errors.Wrap(err, "getting URI") + } + + client := NewInternalClientFromURI(uri, remoteClient) + return client, nil +} + +func NewInternalClientFromURI(defaultURI *pilosa.URI, remoteClient *http.Client) *InternalClient { + return &InternalClient{ + defaultURI: defaultURI, + HTTPClient: remoteClient, + } +} + +// Host returns the host the client was initialized with. +func (c *InternalClient) Host() *pilosa.URI { return c.defaultURI } + +// MaxSliceByIndex returns the number of slices on a server by index. +func (c *InternalClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + return c.maxSliceByIndex(ctx) +} + +// maxSliceByIndex returns the number of slices on a server by index. +func (c *InternalClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/slices/max") + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSlicesMaxResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return rsp.Standard, nil +} + +// Schema returns all index and field schema information. +func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + // Execute request against the host. + u := c.defaultURI.Path("/schema") + + // Build request. + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var rsp getSchemaResponse + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + return rsp.Indexes, nil +} + +// CreateIndex creates a new index on the server. +func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilosa.IndexOptions) error { + // Encode query request. + buf, err := json.Marshal(&postIndexRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "encoding request") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s", index)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrIndexExists + default: + return errors.New(string(body)) + } +} + +// FragmentNodes returns a list of nodes that own a slice. +func (c *InternalClient) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*pilosa.Node, error) { + // Execute request against the host. + u := uriPathToURL(c.defaultURI, "/fragment/nodes") + u.RawQuery = (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + var a []*pilosa.Node + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) + } else if err := json.NewDecoder(resp.Body).Decode(&a); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + + return a, nil +} + +// Query executes query against the index. +func (c *InternalClient) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + return c.QueryNode(ctx, c.defaultURI, index, queryRequest) +} + +// QueryNode executes query against the index, sending the request to the node specified. +func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) { + if index == "" { + return nil, pilosa.ErrIndexRequired + } else if queryRequest.Query == "" { + return nil, pilosa.ErrQueryRequired + } + + // Encode request object. + buf, err := proto.Marshal(queryRequest) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Create HTTP request. + u := uri.Path(fmt.Sprintf("/index/%s/query", index)) + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return nil, errors.New(string(body)) + } + + qresp := &internal.QueryResponse{} + if err := proto.Unmarshal(body, qresp); err != nil { + return nil, fmt.Errorf("unmarshal response: %s", err) + } else if s := qresp.Err; s != "" { + return nil, errors.New(s) + } + + return qresp, nil +} + +// Import bulk imports bits for a single slice to a host. +func (c *InternalClient) Import(ctx context.Context, index, field string, slice uint64, bits []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayload(index, field, slice, bits) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// ImportK bulk imports bits specified by string keys to a host. +func (c *InternalClient) ImportK(ctx context.Context, index, field string, columns []pilosa.Bit) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportPayloadK(index, field, columns) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + node := &pilosa.Node{ + URI: *c.defaultURI, + } + + // Import to node. + if err := c.importNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + + return nil +} + +func (c *InternalClient) EnsureIndex(ctx context.Context, name string, options pilosa.IndexOptions) error { + err := c.CreateIndex(ctx, name, options) + if err == nil || err == pilosa.ErrIndexExists { + return nil + } + return err +} + +func (c *InternalClient) EnsureField(ctx context.Context, indexName string, fieldName string, options pilosa.FieldOptions) error { + err := c.CreateField(ctx, indexName, fieldName, options) + if err == nil || err == pilosa.ErrFieldExists { + return nil + } + return err +} + +// marshalImportPayload marshalls the import parameters into a protobuf byte slice. +func marshalImportPayload(index, field string, slice uint64, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + Slice: slice, + RowIDs: rowIDs, + ColumnIDs: columnIDs, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. +func marshalImportPayloadK(index, field string, bits []pilosa.Bit) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + rowKeys := Bits(bits).RowKeys() + columnKeys := Bits(bits).ColumnKeys() + timestamps := Bits(bits).Timestamps() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportRequest{ + Index: index, + Field: field, + RowKeys: rowKeys, + ColumnKeys: columnKeys, + Timestamps: timestamps, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importNode sends a pre-marshaled import request to a node. +func (c *InternalClient) importNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ImportValue bulk imports field values for a single slice to a host. +func (c *InternalClient) ImportValue(ctx context.Context, index, field string, slice uint64, vals []pilosa.FieldValue) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + buf, err := marshalImportValuePayload(index, field, slice, vals) + if err != nil { + return fmt.Errorf("Error Creating Payload: %s", err) + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Import to each node. + for _, node := range nodes { + if err := c.importValueNode(ctx, node, buf); err != nil { + return fmt.Errorf("import node: host=%s, err=%s", node.URI, err) + } + } + + return nil +} + +// marshalImportValuePayload marshalls the import parameters into a protobuf byte slice. +func marshalImportValuePayload(index, field string, slice uint64, vals []pilosa.FieldValue) ([]byte, error) { + // Separate row and column IDs to reduce allocations. + columnIDs := FieldValues(vals).ColumnIDs() + values := FieldValues(vals).Values() + + // Marshal data to protobuf. + buf, err := proto.Marshal(&internal.ImportValueRequest{ + Index: index, + Field: field, + Slice: slice, + ColumnIDs: columnIDs, + Values: values, + }) + if err != nil { + return nil, fmt.Errorf("marshal import request: %s", err) + } + return buf, nil +} + +// importValueNode sends a pre-marshaled import request to a node. +func (c *InternalClient) importValueNode(ctx context.Context, node *pilosa.Node, buf []byte) error { + // Create URL & HTTP request. + u := nodePathToURL(node, "/import-value") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body and unmarshal response. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } else if resp.StatusCode != http.StatusOK { + return errors.New(string(body)) + } + + var isresp internal.ImportResponse + if err := proto.Unmarshal(body, &isresp); err != nil { + return fmt.Errorf("unmarshal import response: %s", err) + } else if s := isresp.Err; s != "" { + return errors.New(s) + } + + return nil +} + +// ExportCSV bulk exports data for a single slice from a host to CSV format. +func (c *InternalClient) ExportCSV(ctx context.Context, index, field string, slice uint64, w io.Writer) error { + if index == "" { + return pilosa.ErrIndexRequired + } else if field == "" { + return pilosa.ErrFieldRequired + } + + // Retrieve a list of nodes that own the slice. + nodes, err := c.FragmentNodes(ctx, index, slice) + if err != nil { + return fmt.Errorf("slice nodes: %s", err) + } + + // Attempt nodes in random order. + var e error + for _, i := range rand.Perm(len(nodes)) { + node := nodes[i] + + if err := c.exportNodeCSV(ctx, node, index, field, slice, w); err != nil { + e = fmt.Errorf("export node: host=%s, err=%s", node.URI, err) + continue + } else { + return nil + } + } + + return e +} + +// exportNode copies a CSV export from a node to w. +func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, index, field string, slice uint64, w io.Writer) error { + // Create URL. + u := nodePathToURL(node, "/export") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Generate HTTP request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Accept", "text/csv") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Validate status code. + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid status: %d", resp.StatusCode) + } + + // Copy body to writer. + if _, err := io.Copy(w, resp.Body); err != nil { + return errors.Wrap(err, "copying") + } + + return nil +} + +func (c *InternalClient) RetrieveSliceFromURI(ctx context.Context, index, field string, slice uint64, uri pilosa.URI) (io.ReadCloser, error) { + node := &pilosa.Node{ + URI: uri, + } + return c.backupSliceNode(ctx, index, field, slice, node) +} + +func (c *InternalClient) backupSliceNode(ctx context.Context, index, field string, slice uint64, node *pilosa.Node) (io.ReadCloser, error) { + u := nodePathToURL(node, "/fragment/data") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + + // Return error if status is not OK. + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil, pilosa.ErrFragmentNotFound + } else if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("unexpected backup status code: host=%s, code=%d", node.URI, resp.StatusCode) + } + + return resp.Body, nil +} + +// CreateField creates a new field on the server. +func (c *InternalClient) CreateField(ctx context.Context, index, field string, opt pilosa.FieldOptions) error { + if index == "" { + return pilosa.ErrIndexRequired + } + + // Encode query request. + buf, err := json.Marshal(&postFieldRequest{ + Options: opt, + }) + if err != nil { + return errors.Wrap(err, "marshaling") + } + + // Create URL & HTTP request. + u := uriPathToURL(c.defaultURI, fmt.Sprintf("/index/%s/field/%s", index, field)) + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "reading") + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return pilosa.ErrFieldExists + default: + return errors.New(string(body)) + } +} + +// FragmentBlocks returns a list of block checksums for a fragment on a host. +// Only returns blocks which contain data. +func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64) ([]pilosa.FragmentBlock, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, "/fragment/blocks") + u.RawQuery = url.Values{ + "index": {index}, + "field": {field}, + "slice": {strconv.FormatUint(slice, 10)}, + }.Encode() + + // Build request. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFragmentNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp getFragmentBlocksResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Blocks, nil +} + +// BlockData returns row/column id pairs for a block. +func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, field string, slice uint64, block int) ([]uint64, []uint64, error) { + buf, err := proto.Marshal(&internal.BlockDataRequest{ + Index: index, + Field: field, + Slice: slice, + Block: uint64(block), + }) + if err != nil { + return nil, nil, errors.Wrap(err, "marshaling") + } + + u := uriPathToURL(c.defaultURI, "/fragment/block/data") + req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/protobuf") + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Accept", "application/protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // fallthrough + case http.StatusNotFound: + return nil, nil, nil + default: + return nil, nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp internal.BlockDataResponse + if body, err := ioutil.ReadAll(resp.Body); err != nil { + return nil, nil, errors.Wrap(err, "reading") + } else if err := proto.Unmarshal(body, &rsp); err != nil { + return nil, nil, errors.Wrap(err, "unmarshalling") + } + return rsp.RowIDs, rsp.ColumnIDs, nil +} + +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, index string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/attr/diff", index)) + + // Encode request. + buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postIndexAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index, field string, blks []pilosa.AttrBlock) (map[uint64]map[string]interface{}, error) { + if uri == nil { + uri = c.defaultURI + } + u := uriPathToURL(uri, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) + + // Encode request. + buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) + if err != nil { + return nil, errors.Wrap(err, "marshaling") + } + + // Build request. + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + case http.StatusNotFound: + return nil, pilosa.ErrFieldNotFound + default: + return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) + } + + // Decode response object. + var rsp postFieldAttrDiffResponse + if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { + return nil, errors.Wrap(err, "decoding") + } + return rsp.Attrs, nil +} + +// SendMessage posts a message synchronously. +func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb proto.Message) error { + msg, err := pilosa.MarshalMessage(pb) + if err != nil { + return fmt.Errorf("marshaling message: %v", err) + } + + u := uriPathToURL(uri, "/cluster/message") + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) + if err != nil { + return errors.Wrap(err, "making new request") + } + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + // Execute request. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("executing http request: %v", err) + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %v", err) + } + + // Return error if status is not OK. + switch resp.StatusCode { + case http.StatusOK: // ok + default: + return fmt.Errorf("unexpected response status code: %d: %s", resp.StatusCode, body) + } + + return nil +} + +// Bits is a slice of Bit. +type Bits []pilosa.Bit + +func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Bits) Len() int { return len(p) } + +func (p Bits) Less(i, j int) bool { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { + return p[i].Timestamp < p[j].Timestamp + } + return p[i].ColumnID < p[j].ColumnID + } + return p[i].RowID < p[j].RowID +} + +// RowIDs returns a slice of all the row IDs. +func (p Bits) RowIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].RowID + } + return other +} + +// ColumnIDs returns a slice of all the column IDs. +func (p Bits) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// RowKeys returns a slice of all the row keys. +func (p Bits) RowKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].RowKey + } + return other +} + +// ColumnKeys returns a slice of all the column keys. +func (p Bits) ColumnKeys() []string { + other := make([]string, len(p)) + for i := range p { + other[i] = p[i].ColumnKey + } + return other +} + +// Timestamps returns a slice of all the timestamps. +func (p Bits) Timestamps() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Timestamp + } + return other +} + +// GroupBySlice returns a map of bits by slice. +func (p Bits) GroupBySlice() map[uint64][]pilosa.Bit { + m := make(map[uint64][]pilosa.Bit) + for _, bit := range p { + slice := bit.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], bit) + } + + for slice, bits := range m { + sort.Sort(Bits(bits)) + m[slice] = bits + } + + return m +} + +// FieldValues represents a slice of field values. +type FieldValues []pilosa.FieldValue + +func (p FieldValues) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p FieldValues) Len() int { return len(p) } + +func (p FieldValues) Less(i, j int) bool { + return p[i].ColumnID < p[j].ColumnID +} + +// ColumnIDs returns a slice of all the column IDs. +func (p FieldValues) ColumnIDs() []uint64 { + other := make([]uint64, len(p)) + for i := range p { + other[i] = p[i].ColumnID + } + return other +} + +// Values returns a slice of all the values. +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) + for i := range p { + other[i] = p[i].Value + } + return other +} + +// GroupBySlice returns a map of field values by slice. +func (p FieldValues) GroupBySlice() map[uint64][]pilosa.FieldValue { + m := make(map[uint64][]pilosa.FieldValue) + for _, val := range p { + slice := val.ColumnID / pilosa.SliceWidth + m[slice] = append(m[slice], val) + } + + for slice, vals := range m { + sort.Sort(FieldValues(vals)) + m[slice] = vals + } + + return m +} + +// BitsByPos is a slice of bits sorted row then column. +type BitsByPos []pilosa.Bit + +func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitsByPos) Len() int { return len(p) } +func (p BitsByPos) Less(i, j int) bool { + p0, p1 := pos(p[i].RowID, p[i].ColumnID), pos(p[j].RowID, p[j].ColumnID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 +} + +// pos returns the row position of a row/column pair. +func pos(rowID, columnID uint64) uint64 { + return (rowID * pilosa.SliceWidth) + (columnID % pilosa.SliceWidth) +} + +func uriPathToURL(uri *pilosa.URI, path string) url.URL { + return url.URL{ + Scheme: uri.Scheme(), + Host: uri.HostPort(), + Path: path, + } +} + +func nodePathToURL(node *pilosa.Node, path string) url.URL { + return url.URL{ + Scheme: node.URI.Scheme(), + Host: node.URI.HostPort(), + Path: path, + } +} diff --git a/client_test.go b/http/client_test.go similarity index 63% rename from client_test.go rename to http/client_test.go index 6e5f2dc41..185a3c378 100644 --- a/client_test.go +++ b/http/client_test.go @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "context" "fmt" - "net/http" + gohttp "net/http" "reflect" "testing" "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -43,10 +43,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { return server, hldr } -var defaultClient *http.Client +var defaultClient *gohttp.Client func init() { - defaultClient = server.GetHTTPClient(nil) + defaultClient = http.GetHTTPClient(nil) } @@ -61,21 +61,24 @@ func TestClient_MultiNode(t *testing.T) { } s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[0].Holder e.Node = cluster.Nodes[0] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[1].Holder e.Node = cluster.Nodes[1] e.Cluster = cluster return e.Execute(ctx, index, query, slices, opt) } s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { - e := pilosa.NewExecutor(defaultClient) + httpClient := http.NewInternalClientFromURI(&cluster.Nodes[0].URI, defaultClient) + e := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(httpClient)) e.Holder = hldr[2].Holder e.Node = cluster.Nodes[2] e.Cluster = cluster @@ -84,10 +87,17 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} + + // This was generated with: `owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI())` + owns := [][]uint64{ + {1, 3, 4, 8, 10, 13, 17, 19}, + {2, 5, 7, 11, 12, 14, 18}, + {0, 6, 9, 15, 16, 20}, + } + for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false - for _, ownNum := range owns { + for _, ownNum := range owns[i] { if ownNum == num { ownsNum = true break @@ -98,9 +108,9 @@ func TestClient_MultiNode(t *testing.T) { } } - baseBit0 := SliceWidth * sliceNums[0] - baseBit1 := SliceWidth * sliceNums[1] - baseBit2 := SliceWidth * sliceNums[2] + baseBit0 := pilosa.SliceWidth * sliceNums[0] + baseBit1 := pilosa.SliceWidth * sliceNums[1] + baseBit2 := pilosa.SliceWidth * sliceNums[2] maxSlice := uint64(0) for _, x := range sliceNums { @@ -109,26 +119,26 @@ func TestClient_MultiNode(t *testing.T) { } } - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustSetBits("i", "f", 100, baseBit0+10) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustSetBits("i", "f", 4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustSetBits("i", "f", 2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustSetBits("i", "f", 3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustSetBits("i", "f", 22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[1].MustSetBits("i", "f", 99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustSetBits("i", "f", 100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustSetBits("i", "f", 98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustSetBits("i", "f", 1, baseBit1+4) + hldr[1].MustSetBits("i", "f", 22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustSetBits("i", "f", 20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustSetBits("i", "f", 21, baseBit2+10) + hldr[2].MustSetBits("i", "f", 100, baseBit2+10) + hldr[2].MustSetBits("i", "f", 99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustSetBits("i", "f", 98, baseBit2+10, baseBit2+11) + hldr[2].MustSetBits("i", "f", 22, baseBit2+10, baseBit2+11, baseBit2+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay @@ -138,14 +148,14 @@ func TestClient_MultiNode(t *testing.T) { hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache() // Connect to each node to compare results. - client := make([]*test.Client, 3) - client[0] = test.MustNewClient(s[0].Host(), defaultClient) - client[1] = test.MustNewClient(s[1].Host(), defaultClient) - client[2] = test.MustNewClient(s[2].Host(), defaultClient) + client := make([]*Client, 3) + client[0] = MustNewClient(s[0].Host(), defaultClient) + client[1] = MustNewClient(s[1].Host(), defaultClient) + client[2] = MustNewClient(s[2].Host(), defaultClient) topN := 4 queryRequest := &internal.QueryRequest{ - Query: fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN), + Query: fmt.Sprintf(`TopN(field="%s", n=%d)`, "f", topN), Remote: false, } result, err := client[0].Query(context.Background(), "i", queryRequest) @@ -211,8 +221,8 @@ func TestClient_Import(t *testing.T) { defer hldr.Close() // Load bitmap into cache to ensure cache gets updated. - f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f.Row(0) + hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created. + hldr.Row("i", "f", 0) s := test.NewServer() defer s.Close() @@ -221,7 +231,7 @@ func TestClient_Import(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) + c := MustNewClient(s.Host(), defaultClient) if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ {RowID: 0, ColumnID: 1}, {RowID: 0, ColumnID: 5}, @@ -231,10 +241,10 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + if a := hldr.Row("i", "f", 200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected columns: %+v", a) } } @@ -244,8 +254,9 @@ func TestClient_ImportValue(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - fld := pilosa.Field{ - Name: "fld", + fldName := "f" + + fo := pilosa.FieldOptions{ Type: pilosa.FieldTypeInt, Min: -100, Max: 100, @@ -253,7 +264,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) + field, err := index.CreateFieldIfNotExists(fldName, fo) if err != nil { t.Fatal(err) } @@ -265,8 +276,8 @@ func TestClient_ImportValue(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) - if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ + c := MustNewClient(s.Host(), defaultClient) + if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{ {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, {ColumnID: 3, Value: 40}, @@ -275,7 +286,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Sum. - sum, cnt, err := frame.FieldSum(nil, fld.Name) + sum, cnt, err := field.Sum(nil, fldName) if err != nil { t.Fatal(err) } @@ -284,7 +295,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min. - min, cnt, err := frame.FieldMin(nil, fld.Name) + min, cnt, err := field.Min(nil, fldName) if err != nil { t.Fatal(err) } @@ -293,11 +304,11 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Min with Filter. - filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + filter, err := field.Range(fldName, pql.GT, 40) if err != nil { t.Fatal(err) } - min, cnt, err = frame.FieldMin(filter, fld.Name) + min, cnt, err = field.Min(filter, fldName) if err != nil { t.Fatal(err) } @@ -306,7 +317,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify Max. - max, cnt, err := frame.FieldMax(nil, fld.Name) + max, cnt, err := field.Max(nil, fldName) if err != nil { t.Fatal(err) } @@ -321,11 +332,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer hldr.Close() // Set two bits on blocks 0 & 3. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) + hldr.SetBit("i", "f", 0, 1) + hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) + hldr.SetBit("i", "f", 0, 1) s := test.NewServer() defer s.Close() @@ -334,8 +345,8 @@ func TestClient_FragmentBlocks(t *testing.T) { s.Handler.API.Holder = hldr.Holder // Retrieve blocks. - c := test.MustNewClient(s.Host(), defaultClient) - blocks, err := c.FragmentBlocks(context.Background(), "i", "f", 0) + c := MustNewClient(s.Host(), defaultClient) + blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0) if err != nil { t.Fatal(err) } else if len(blocks) != 2 { @@ -351,3 +362,17 @@ func TestClient_FragmentBlocks(t *testing.T) { t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks)) } } + +// Client represents a test wrapper for pilosa.Client. +type Client struct { + *http.InternalClient +} + +// MustNewClient returns a new instance of Client. Panic on error. +func MustNewClient(host string, h *gohttp.Client) *Client { + c, err := http.NewInternalClient(host, h) + if err != nil { + panic(err) + } + return &Client{InternalClient: c} +} diff --git a/http/handler.go b/http/handler.go new file mode 100644 index 000000000..509378d6e --- /dev/null +++ b/http/handler.go @@ -0,0 +1,1289 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "crypto/tls" + "encoding/json" + "expvar" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" + "reflect" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/internal" + + "github.com/pkg/errors" +) + +// Handler represents an HTTP handler. +type Handler struct { + Handler http.Handler + + Logger pilosa.Logger + + // Keeps the query argument validators for each handler + validators map[string]*queryValidationSpec + + API *pilosa.API + + AllowedOrigins []string +} + +// externalPrefixFlag denotes endpoints that are intended to be exposed to clients. +// This is used for stats tagging. +var externalPrefixFlag = map[string]bool{ + "schema": true, + "query": true, + "import": true, + "export": true, + "index": true, + "field": true, + "nodes": true, + "version": true, +} + +type errorResponse struct { + Error string `json:"error"` +} + +// HandlerOption is a functional option type for pilosa.Handler +type HandlerOption func(s *Handler) error + +func OptHandlerAllowedOrigins(origins []string) HandlerOption { + return func(h *Handler) error { + h.Handler = handlers.CORS( + handlers.AllowedOrigins(origins), + handlers.AllowedHeaders([]string{"Content-Type"}), + )(h.Handler) + return nil + } +} + +func OptHandlerAPI(api *pilosa.API) HandlerOption { + return func(h *Handler) error { + h.API = api + return nil + } +} + +func OptHandlerLogger(logger pilosa.Logger) HandlerOption { + return func(h *Handler) error { + h.Logger = logger + return nil + } +} + +// NewHandler returns a new instance of Handler with a default logger. +func NewHandler(opts ...HandlerOption) (*Handler, error) { + handler := &Handler{ + Logger: pilosa.NopLogger, + } + handler.Handler = NewRouter(handler) + handler.populateValidators() + + for _, opt := range opts { + err := opt(handler) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return handler, nil +} + +func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) { + server := &http.Server{Handler: h} + go func() { + <-closing + server.Close() + }() + err := server.Serve(ln) + if err != nil && err.Error() != "http: Server closed" { + h.Logger.Printf("HTTP handler terminated with error: %s\n", err) + } +} + +func (h *Handler) populateValidators() { + h.validators = map[string]*queryValidationSpec{} + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") + h.validators["GetSliceMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") + h.validators["GetExport"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "field", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "slice") +} + +func (h *Handler) queryArgValidator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := mux.CurrentRoute(r).GetName() + if validator, ok := h.validators[key]; ok { + if err := validator.validate(r.URL.Query()); err != nil { + // TODO: Return the response depending on the Accept header + response := errorResponse{Error: err.Error()} + body, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + http.Error(w, string(body), http.StatusBadRequest) + return + } + } + next.ServeHTTP(w, r) + }) +} + +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() + router.HandleFunc("/", handler.handleHome).Methods("GET") + router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") + + router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") + router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") + router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") + router.HandleFunc("/import", handler.handlePostImport).Methods("POST") + router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") + router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") + router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") + router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") + router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") + router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. + router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") + router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") + + // TODO: Apply MethodNotAllowed statuses to all endpoints. + // Ideally this would be automatic, as described in this (wontfix) ticket: + // https://github.com/gorilla/mux/issues/6 + // For now we just do it for the most commonly used handler, /query + router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") + + router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET") + + router.Use(handler.queryArgValidator) + return router +} + +func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) +} + +// ServeHTTP handles an HTTP request. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + stack := debug.Stack() + msg := "PANIC: %s\n%s" + h.Logger.Printf(msg, err, stack) + fmt.Fprintf(w, msg, err, stack) + } + }() + + t := time.Now() + h.Handler.ServeHTTP(w, r) + dif := time.Since(t) + + // Calculate per request StatsD metrics when the handler is fully configured. + statsTags := make([]string, 0, 3) + + longQueryTime := h.API.LongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") + } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + stats := h.API.StatsWithTags(statsTags) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } +} + +func (h *Handler) handleHome(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) +} + +// handleGetSchema handles GET /schema requests. +func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) + if err := json.NewEncoder(w).Encode(getSchemaResponse{ + Indexes: schema, + }); err != nil { + h.Logger.Printf("write schema response error: %s", err) + } +} + +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), + LocalID: h.API.LocalID(), + } + if err := json.NewEncoder(w).Encode(status); err != nil { + h.Logger.Printf("write status response error: %s", err) + } +} + +func (h *Handler) handleGetInfo(w http.ResponseWriter, r *http.Request) { + info := h.API.Info() + if err := json.NewEncoder(w).Encode(info); err != nil { + h.Logger.Printf("write info response error: %s", err) + } +} + +type getSchemaResponse struct { + Indexes []*pilosa.IndexInfo `json:"indexes"` +} + +type getStatusResponse struct { + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` +} + +// handlePostQuery handles /query requests. +func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { + // Parse incoming request. + req, err := h.readQueryRequest(r) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + // TODO: Remove + req.Index = mux.Vars(r)["index"] + + resp, err := h.API.Query(r.Context(), req) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + return + } + + // Set appropriate status code, if there is an error. + if resp.Err != nil { + switch resp.Err { + case pilosa.ErrTooManyWrites: + w.WriteHeader(http.StatusRequestEntityTooLarge) + default: + w.WriteHeader(http.StatusInternalServerError) + } + } + + // Write response back to client. + if err := h.writeQueryResponse(w, r, &resp); err != nil { + h.Logger.Printf("write query response error: %s", err) + } +} + +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) + } +} + +type getSlicesMaxResponse struct { + Standard map[string]uint64 `json:"standard"` +} + +// handleGetIndexes handles GET /index request. +func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { + h.handleGetSchema(w, r) +} + +// handleGetIndex handles GET /index/ requests. +func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + for _, idx := range h.API.Schema(r.Context()) { + if idx.Name == indexName { + if err := json.NewEncoder(w).Encode(idx); err != nil { + h.Logger.Printf("write response error: %s", err) + } + return + } + } + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) +} + +type postIndexRequest struct { + Options pilosa.IndexOptions `json:"options"` +} + +//_postIndexRequest is necessary to avoid recursion while decoding. +type _postIndexRequest postIndexRequest + +// Custom Unmarshal JSON to validate request body when creating a new index. +func (p *postIndexRequest) UnmarshalJSON(b []byte) error { + + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshalling unexpected values") + } + + validIndexOptions := getValidOptions(pilosa.IndexOptions{}) + err := validateOptions(m, validIndexOptions) + if err != nil { + return err + } + // Unmarshal expected values. + var _p _postIndexRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected values") + } + + p.Options = _p.Options + + return nil +} + +// Raise errors for any unknown key +func validateOptions(data map[string]interface{}, validIndexOptions []string) error { + for k, v := range data { + switch k { + case "options": + options, ok := v.(map[string]interface{}) + if !ok { + return errors.New("options is not map[string]interface{}") + } + for kk, vv := range options { + if !foundItem(validIndexOptions, kk) { + return fmt.Errorf("Unknown key: %v:%v", kk, vv) + } + } + default: + return fmt.Errorf("Unknown key: %v:%v", k, v) + } + } + return nil +} + +func foundItem(items []string, item string) bool { + for _, i := range items { + if item == i { + return true + } + } + return false +} + +type postIndexResponse struct{} + +// handleDeleteIndex handles DELETE /index request. +func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteIndexResponse struct{} + +// handlePostIndex handles POST /index request. +func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the index + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) + if errors.Cause(err) == pilosa.ErrIndexExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +// handlePostIndexAttrDiff handles POST /index/attr/diff requests. +func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + + // Decode request. + var req postIndexAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postIndexAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postIndexAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// handlePostField handles POST /field request. +func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err == io.EOF { + // If no data was provided (EOF), we still create the field + // with default values. + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _, err = h.API.CreateField(r.Context(), indexName, fieldName, req.Options) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrFieldExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type _postFieldRequest postFieldRequest + +// Custom Unmarshal JSON to validate request body when creating a new field. If there's new FieldOptions, +// adding it to validFieldOptions to make sure the new option is validated, otherwise the request will be failed +func (p *postFieldRequest) UnmarshalJSON(b []byte) error { + // m is an overflow map used to capture additional, unexpected keys. + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return errors.Wrap(err, "unmarshaling unexpected keys") + } + + validFieldOptions := getValidOptions(pilosa.FieldOptions{}) + err := validateOptions(m, validFieldOptions) + if err != nil { + return err + } + + // Unmarshal expected values. + var _p _postFieldRequest + if err := json.Unmarshal(b, &_p); err != nil { + return errors.Wrap(err, "unmarshalling expected keys") + } + + p.Options = _p.Options + return nil + +} + +func getValidOptions(option interface{}) []string { + validOptions := []string{} + val := reflect.ValueOf(option) + for i := 0; i < val.Type().NumField(); i++ { + jsonTag := val.Type().Field(i).Tag.Get("json") + s := strings.Split(jsonTag, ",") + validOptions = append(validOptions, s[0]) + } + return validOptions +} + +type postFieldRequest struct { + Options pilosa.FieldOptions `json:"options"` +} + +type postFieldResponse struct{} + +// handleDeleteField handles DELETE /field request. +func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + err := h.API.DeleteField(r.Context(), indexName, fieldName) + if err != nil { + if errors.Cause(err) == pilosa.ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFieldResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type deleteFieldResponse struct{} + +// handlePostFieldAttrDiff handles POST /field/attr/diff requests. +func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + + // Decode request. + var req postFieldAttrDiffRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + attrs, err := h.API.FieldAttrDiff(r.Context(), indexName, fieldName, req.Blocks) + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postFieldAttrDiffResponse{ + Attrs: attrs, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type postFieldAttrDiffRequest struct { + Blocks []pilosa.AttrBlock `json:"blocks"` +} + +type postFieldAttrDiffResponse struct { + Attrs map[uint64]map[string]interface{} `json:"attrs"` +} + +// readQueryRequest parses an query parameters from r. +func (h *Handler) readQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + switch r.Header.Get("Content-Type") { + case "application/x-protobuf": + return h.readProtobufQueryRequest(r) + default: + return h.readURLQueryRequest(r) + } +} + +// readProtobufQueryRequest parses query parameters in protobuf from r. +func (h *Handler) readProtobufQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + // Slurp the body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + + // Unmarshal into object. + var req internal.QueryRequest + if err := proto.Unmarshal(body, &req); err != nil { + return nil, errors.Wrap(err, "unmarshalling") + } + + return decodeQueryRequest(&req), nil +} + +// readURLQueryRequest parses query parameters from URL parameters from r. +func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, error) { + q := r.URL.Query() + + // Parse query string. + buf, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, errors.Wrap(err, "reading") + } + query := string(buf) + + // Parse list of slices. + slices, err := parseUint64Slice(q.Get("slices")) + if err != nil { + return nil, errors.New("invalid slice argument") + } + + return &pilosa.QueryRequest{ + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", + }, nil +} + +// writeQueryResponse writes the response from the executor to w. +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { + return h.writeProtobufQueryResponse(w, resp) + } + return h.writeJSONQueryResponse(w, resp) +} + +// writeProtobufQueryResponse writes the response from the executor to w as protobuf. +func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + if buf, err := proto.Marshal(encodeQueryResponse(resp)); err != nil { + return errors.Wrap(err, "marshalling") + } else if _, err := w.Write(buf); err != nil { + return errors.Wrap(err, "writing") + } + return nil +} + +// writeJSONQueryResponse writes the response from the executor to w as JSON. +func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, resp *pilosa.QueryResponse) error { + return json.NewEncoder(w).Encode(resp) +} + +// handlePostImport handles /import requests. +func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := h.API.Import(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handlePostImportValue handles /import-value requests. +func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + // Read entire body. + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshal into request object. + var req internal.ImportValueRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch errors.Cause(err) { + case pilosa.ErrIndexNotFound: + fallthrough + case pilosa.ErrFieldNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Marshal response object. + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) + if e != nil { + http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) + return + } + + // Write response. + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + w.Write(buf) +} + +// handleGetExport handles /export requests. +func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) { + switch r.Header.Get("Accept") { + case "text/csv": + h.handleGetExportCSV(w, r) + default: + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + } +} + +func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { + // Parse query parameters. + q := r.URL.Query() + index, field := q.Get("index"), q.Get("field") + + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "invalid slice", http.StatusBadRequest) + return + } + + if err = h.API.ExportCSV(r.Context(), index, field, slice, w); err != nil { + switch errors.Cause(err) { + case pilosa.ErrFragmentNotFound: + break + case pilosa.ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } +} + +// handleGetFragmentNodes handles /fragment/nodes requests. +func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + index := q.Get("index") + + // Read slice parameter. + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice should be an unsigned integer", http.StatusBadRequest) + return + } + + // Retrieve fragment owner nodes. + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Write to response. + if err := json.NewEncoder(w).Encode(nodes); err != nil { + h.Logger.Printf("json write error: %s", err) + } +} + +// handleGetFragmentBlockData handles GET /fragment/block/data requests. +func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) + if err != nil { + if _, ok := err.(pilosa.BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Write response. + w.Header().Set("Content-Type", "application/protobuf") + w.Header().Set("Content-Length", strconv.Itoa(len(buf))) + w.Write(buf) +} + +// handleGetFragmentBlocks handles GET /fragment/blocks requests. +func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { + // Read slice parameter. + q := r.URL.Query() + slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) + if err != nil { + http.Error(w, "slice required", http.StatusBadRequest) + return + } + + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("field"), slice) + if err != nil { + if errors.Cause(err) == pilosa.ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ + Blocks: blocks, + }); err != nil { + h.Logger.Printf("block response encoding error: %s", err) + } +} + +type getFragmentBlocksResponse struct { + Blocks []pilosa.FragmentBlock `json:"blocks"` +} + +// handleGetVersion handles /version requests. +func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { + err := json.NewEncoder(w).Encode(struct { + Version string `json:"version"` + }{ + Version: h.API.Version(), + }) + if err != nil { + h.Logger.Printf("write version response error: %s", err) + } +} + +// QueryResult types. +const ( + QueryResultTypeNil uint32 = iota + QueryResultTypeRow + QueryResultTypePairs + QueryResultTypeValCount + QueryResultTypeUint64 + QueryResultTypeBool +) + +func decodeQueryRequest(pb *internal.QueryRequest) *pilosa.QueryRequest { + req := &pilosa.QueryRequest{ + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeRowAttrs: pb.ExcludeRowAttrs, + ExcludeColumns: pb.ExcludeColumns, + } + + return req +} + +func encodeQueryResponse(resp *pilosa.QueryResponse) *internal.QueryResponse { + pb := &internal.QueryResponse{ + Results: make([]*internal.QueryResult, len(resp.Results)), + ColumnAttrSets: pilosa.EncodeColumnAttrSets(resp.ColumnAttrSets), + } + + for i := range resp.Results { + pb.Results[i] = &internal.QueryResult{} + + switch result := resp.Results[i].(type) { + case *pilosa.Row: + pb.Results[i].Type = QueryResultTypeRow + pb.Results[i].Row = pilosa.EncodeRow(result) + case []pilosa.Pair: + pb.Results[i].Type = QueryResultTypePairs + pb.Results[i].Pairs = pilosa.EncodePairs(result) + case pilosa.ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = pilosa.EncodeValCount(result) + case uint64: + pb.Results[i].Type = QueryResultTypeUint64 + pb.Results[i].N = result + case bool: + pb.Results[i].Type = QueryResultTypeBool + pb.Results[i].Changed = result + case nil: + pb.Results[i].Type = QueryResultTypeNil + } + } + + if resp.Err != nil { + pb.Err = resp.Err.Error() + } + + return pb +} + +// parseUint64Slice returns a slice of uint64s from a comma-delimited string. +func parseUint64Slice(s string) ([]uint64, error) { + var a []uint64 + for _, str := range strings.Split(s, ",") { + // Ignore blanks. + if str == "" { + continue + } + + // Parse number. + num, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "parsing int") + } + a = append(a, num) + } + return a, nil +} + +// errorString returns the string representation of err. +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req setCoordinatorRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) + return + } + + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) + } + return + } + // Encode response. + if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ + Old: oldNode, + New: newNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type setCoordinatorRequest struct { + ID string `json:"id"` +} + +type setCoordinatorResponse struct { + Old *pilosa.Node `json:"old"` + New *pilosa.Node `json:"new"` +} + +// handlePostClusterResizeRemoveNode handles POST /cluster/resize/remove-node request. +func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req removeNodeRequest + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + removeNode, err := h.API.RemoveNode(req.ID) + if err != nil { + if errors.Cause(err) == pilosa.ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(removeNodeResponse{ + Remove: removeNode, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type removeNodeRequest struct { + ID string `json:"id"` +} + +type removeNodeResponse struct { + Remove *pilosa.Node `json:"remove"` +} + +// handlePostClusterResizeAbort handles POST /cluster/resize/abort request. +func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() + var msg string + if err != nil { + switch errors.Cause(err) { + case pilosa.ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case pilosa.ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + // Encode response. + if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ + Info: msg, + }); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +type clusterResizeAbortResponse struct { + Info string `json:"info"` +} + +func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } + + err := h.API.ClusterMessage(r.Context(), r.Body) + if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request + http.Error(w, err.Error(), http.StatusBadRequest) + } + + if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } +} + +func (h *Handler) GetAPI() *pilosa.API { + return h.API +} + +type defaultClusterMessageResponse struct{} + +// TranslateStoreBufferSize is the buffer size used for streaming data. +const TranslateStoreBufferSize = 65536 + +func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + offset, _ := strconv.ParseInt(q.Get("offset"), 10, 64) + + rc, err := h.API.TranslateStore.Reader(r.Context(), offset) + if err == pilosa.ErrNotImplemented { + http.Error(w, err.Error(), http.StatusNotImplemented) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer rc.Close() + + // Ensure reader is closed when the client disconnects. + go func() { <-r.Context().Done(); rc.Close() }() + + // Flush header so client can continue. + w.WriteHeader(http.StatusOK) + if w, ok := w.(http.Flusher); ok { + w.Flush() + } + + // Copy from reader to client until store or client disconnect. + buf := make([]byte, TranslateStoreBufferSize) + for { + // Read from store. + n, err := rc.Read(buf) + if err == io.EOF { + return + } else if err != nil { + h.Logger.Printf("http: translate store read error: %s", err) + return + } else if n == 0 { + continue + } + + // Write to response & flush. + if _, err := w.Write(buf[:n]); err != nil { + h.Logger.Printf("http: translate store response write error: %s", err) + return + } else if w, ok := w.(http.Flusher); ok { + w.Flush() + } + } +} + +type queryValidationSpec struct { + required []string + args map[string]struct{} +} + +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { + args := map[string]struct{}{} + for _, arg := range requiredArgs { + args[arg] = struct{}{} + } + + return &queryValidationSpec{ + required: requiredArgs, + args: args, + } +} + +func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { + for _, arg := range args { + s.args[arg] = struct{}{} + } + return s +} + +func (s queryValidationSpec) validate(query url.Values) error { + for _, req := range s.required { + if query.Get(req) == "" { + return errors.Errorf("%s is required", req) + } + } + for k := range query { + if _, ok := s.args[k]; !ok { + return errors.Errorf("%s is not a valid argument", k) + } + } + return nil +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} +} diff --git a/handler_internal_test.go b/http/handler_internal_test.go similarity index 84% rename from handler_internal_test.go rename to http/handler_internal_test.go index ceaeb707c..fa7c23060 100644 --- a/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package http import ( "encoding/json" "reflect" "testing" + + "github.com/pilosa/pilosa" ) // Test custom UnmarshalJSON for postIndexRequest object @@ -27,7 +29,7 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { expected postIndexRequest err string }{ - {json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}}, + {json: `{"options": {}}`, expected: postIndexRequest{Options: pilosa.IndexOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, @@ -55,23 +57,23 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { } } -// Test custom UnmarshalJSON for postFrameRequest object -func TestPostFrameRequestUnmarshalJSON(t *testing.T) { +// Test custom UnmarshalJSON for postFieldRequest object +func TestPostFieldRequestUnmarshalJSON(t *testing.T) { tests := []struct { json string - expected postFrameRequest + expected postFieldRequest err string }{ - {json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}}, + {json: `{"options": {}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{}}}, {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, - {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFieldRequest{Options: pilosa.FieldOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { - actual := &postFrameRequest{} + actual := &postFieldRequest{} err := json.Unmarshal([]byte(test.json), actual) if err != nil { if test.err == "" || test.err != err.Error() { diff --git a/handler_test.go b/http/handler_test.go similarity index 71% rename from handler_test.go rename to http/handler_test.go index a7ba8fa85..93f9906b2 100644 --- a/handler_test.go +++ b/http/handler_test.go @@ -12,17 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package http_test import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "reflect" "strings" @@ -30,9 +29,9 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/test" ) @@ -51,7 +50,7 @@ func TestHandlerPanics(t *testing.T) { if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) { t.Fatalf("expected panic in log, but got: %s", bufbytes) } - if w.Code != http.StatusInternalServerError { + if w.Code != gohttp.StatusInternalServerError { t.Fatalf("expected internal server error, but got: %v", w.Code) } bodyBytes := w.Body.Bytes() @@ -71,7 +70,7 @@ func TestHandler_NotFound(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) - if w.Code != http.StatusNotFound { + if w.Code != gohttp.StatusNotFound { t.Fatalf("invalid status: %d", w.Code) } } @@ -84,17 +83,17 @@ func TestHandler_Schema(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -103,10 +102,10 @@ func TestHandler_Schema(t *testing.T) { h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -121,17 +120,17 @@ func TestHandler_Status(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) } - if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -144,7 +143,7 @@ func TestHandler_Status(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -158,9 +157,9 @@ func TestHandler_Info(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", SliceWidth) { + } else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) { t.Fatalf("unexpected body: %s", body) } } @@ -175,7 +174,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { bod, err := ioutil.ReadAll(w.Body) t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { @@ -190,20 +189,20 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1) + hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2) + hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2) + hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8) h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -231,7 +230,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -272,7 +271,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, req) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -288,7 +287,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -297,7 +296,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -319,7 +318,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[100]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -342,14 +341,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeUint64 { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) @@ -372,7 +371,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -405,7 +404,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -430,16 +429,16 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -488,7 +487,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { r.Header.Set("Content-Type", "application/x-protobuf") r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } @@ -496,9 +495,9 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) @@ -537,8 +536,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))) - if w.Code != http.StatusOK { + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))) + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -561,17 +560,17 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypePairs { + } else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if a := resp.Results[0].GetPairs(); len(a) != 2 { t.Fatalf("unexpected pair length: %d", len(a)) @@ -592,7 +591,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -612,10 +611,10 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { } w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) + r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -637,7 +636,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) - if w.Code != http.StatusMethodNotAllowed { + if w.Code != gohttp.StatusMethodNotAllowed { t.Fatalf("invalid status: %d", w.Code) } } @@ -652,7 +651,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"parsing: expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" { t.Fatalf("unexpected body: %s", body) @@ -674,14 +673,14 @@ func TestHandler_Index_Delete(t *testing.T) { } // Send request to delete index. - resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) + resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader(""))) if err != nil { t.Fatal(err) } defer resp.Body.Close() // Verify body response. - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { t.Fatalf("unexpected status: %d", resp.StatusCode) } else if buf, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) @@ -695,12 +694,12 @@ func TestHandler_Index_Delete(t *testing.T) { } } -// Ensure handler can delete a frame. -func TestHandler_DeleteFrame(t *testing.T) { +// Ensure handler can delete a field. +func TestHandler_DeleteField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { + if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -708,13 +707,13 @@ func TestHandler_DeleteFrame(t *testing.T) { h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) - if w.Code != http.StatusOK { + h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader(""))) + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{}`+"\n" { t.Fatalf("unexpected body: %s", body) - } else if f := hldr.Index("i0").Frame("f1"); f != nil { - t.Fatal("expected nil frame") + } else if f := hldr.Index("i0").Field("f1"); f != nil { + t.Fatal("expected nil field") } } @@ -751,7 +750,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( + resp, err := gohttp.Post( s.URL+"/index/i/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), @@ -767,8 +766,8 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { } } -// Ensure the handler can return data in differing blocks for a frame. -func TestHandler_Frame_AttrStore_Diff(t *testing.T) { +// Ensure the handler can return data in differing blocks for a field. +func TestHandler_Field_AttrStore_Diff(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -778,7 +777,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { // Set attributes on the index. idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{}) + f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } @@ -801,8 +800,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { blks[1].Checksum = []byte("MISMATCHED_CHECKSUM") // Send block checksums to determine diff. - resp, err := http.Post( - s.URL+"/index/i/frame/meta/attr/diff", + resp, err := gohttp.Post( + s.URL+"/index/i/field/meta/attr/diff", "application/json", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) @@ -817,245 +816,6 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { } } -// Ensure the handler can create a new field on an existing frame. -func TestHandler_Frame_AddField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); !reflect.DeepEqual(field, &pilosa.Field{Name: "x", Type: "int", Min: 100, Max: 200}) { - t.Fatalf("unexpected field: %#v", field) - } - }) - - t.Run("ErrInvalidFieldType", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"bad_type","min":100,"max":200}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field type`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":200,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: validating field: invalid field range`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) - - t.Run("ErrFieldAlreadyExists", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, - }); err != nil { - t.Fatal(err) - } - - resp, err := http.Post( - s.URL+"/index/i/frame/f/field/x", - "application/json", - strings.NewReader(`{"type":"int","min":0,"max":100}`), - ) - if err != nil { - t.Fatal(err) - } else if body := MustReadAll(resp.Body); string(body) != `creating field: field already exists`+"\n" { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -// Ensure the handler can delete existing fields. -func TestHandler_Frame_DeleteField(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/x", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - if field := f.Field("x"); field != nil { - t.Fatalf("expected nil field, got: %#v", field) - } - }) - - t.Run("ErrFieldNotFound", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("DELETE", s.URL+"/index/i/frame/f/field/y", nil) - if err != nil { - t.Fatal(err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `deleting field: field not found` { - t.Fatalf("unexpected body: %q", body) - } else if err := resp.Body.Close(); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusInternalServerError { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - }) -} - -func TestHandler_Frame_GetFields(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - t.Run("OK", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { - t.Fatal(err) - } - resp, err := http.Get(s.URL + "/index/i/frame/f/fields") - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } - - var fields FrameFields - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - if err = json.Unmarshal([]byte(body), &fields); err != nil { - t.Fatal(err) - } - field := fields.Fields[0] - if field.Name != "x" { - t.Fatalf("expected field's name: x, actuall name: %v", field.Name) - } else if field.Min != 1 { - t.Fatalf("expected field's min: x, actuall min: %v", field.Min) - } else if field.Max != 100 { - t.Fatalf("expected field's max: x, actuall max: %v", field.Max) - } - - }) - - t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) - if err != nil { - t.Fatalf("creating frame: %v", err) - } - - resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") - if err != nil { - t.Fatal(err) - } - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status code: %d", resp.StatusCode) - } else if body, err := ioutil.ReadAll(resp.Body); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { - t.Fatalf("shouldn't get frame fields not allowed error: %q", body) - } - }) - -} - -type FrameFields struct { - Fields []pilosa.Field -} - // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { hldr := test.MustOpenHolder() @@ -1072,7 +832,7 @@ func TestHandler_Version(t *testing.T) { if strings.HasPrefix(version, "v") { version = version[1:] } - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if w.Body.String() != `{"version":"`+version+`"}`+"\n" { t.Fatalf("unexpected body: %q", w.Body.String()) @@ -1092,7 +852,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -1102,7 +862,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -1110,7 +870,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { w = httptest.NewRecorder() r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusBadRequest { + if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -1126,7 +886,7 @@ func TestHandler_Expvars(t *testing.T) { w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) - if w.Code != http.StatusOK { + if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } } @@ -1149,40 +909,12 @@ func TestHandler_RecalculateCaches(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) - if w.Code != http.StatusNoContent { + if w.Code != gohttp.StatusNoContent { t.Fatalf("unexpected status code: %d", w.Code) } } -func TestHandler_WebUI(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - h := test.MustNewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - h.FileSystem = &statik.FileSystem{} - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - if !strings.Contains(w.Body.String(), "Pilosa WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } - - // If curl is the client, the response should be different - w = httptest.NewRecorder() - req := test.MustNewHTTPRequest("GET", "/", nil) - req.Header.Add("User-Agent", "curl/7.54.0") - h.ServeHTTP(w, req) - if !strings.Contains(w.Body.String(), "try the WebUI") { - t.Fatalf("WebUI is not being served correctly.") - } -} - func TestHandler_CORS(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -1208,7 +940,7 @@ func TestHandler_CORS(t *testing.T) { } // CORS config should allow preflight response - handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"})) + handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"})) w = httptest.NewRecorder() handler.ServeHTTP(w, req) result = w.Result() diff --git a/http/translator.go b/http/translator.go new file mode 100644 index 000000000..3ca9b840d --- /dev/null +++ b/http/translator.go @@ -0,0 +1,87 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + + "github.com/pilosa/pilosa" +) + +// Ensure implementation implements inteface. +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +// TranslateStore represents an implementation of TranslateStore that +// communicates over HTTP. This is used with the TranslateHandler. +type TranslateStore struct { + URL string +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore(rawurl string) *TranslateStore { + return &TranslateStore{URL: rawurl} +} + +// TranslateColumnsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateColumnToString is not currently implemented. +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// TranslateRowsToUint64 is not currently implemented. +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return nil, pilosa.ErrNotImplemented +} + +// TranslateRowToString is not currently implemented. +func (s *TranslateStore) TranslateRowToString(index, frame string, values uint64) (string, error) { + return "", pilosa.ErrNotImplemented +} + +// Reader returns a reader that can stream data from a remote store. +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + // Generate remote URL. + u, err := url.Parse(s.URL) + if err != nil { + return nil, err + } + u.Path = "/translate/data" + u.RawQuery = (url.Values{ + "offset": {strconv.FormatInt(off, 10)}, + }).Encode() + + // Connect a stream to the remote server. + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + + // Connect a stream to the remote server. + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http: cannot connect to translate store endpoint: %s", err) + } + + // Handle error codes or return body as stream. + switch resp.StatusCode { + case http.StatusOK: + return resp.Body, nil + case http.StatusNotImplemented: + resp.Body.Close() + return nil, pilosa.ErrNotImplemented + default: + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("http: invalid translate store endpoint status: code=%d url=%s body=%q", resp.StatusCode, u.String(), bytes.TrimSpace(body)) + } +} diff --git a/http/translator_test.go b/http/translator_test.go new file mode 100644 index 000000000..3378ddc58 --- /dev/null +++ b/http/translator_test.go @@ -0,0 +1,134 @@ +package http_test + +import ( + "context" + "io" + "io/ioutil" + "net/http/httptest" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/mock" + "github.com/pilosa/pilosa/test" +) + +func TestTranslateStore_Reader(t *testing.T) { + // Ensure client can connect and stream the translate store data. + t.Run("OK", func(t *testing.T) { + t.Run("ServerDisconnect", func(t *testing.T) { + var mrc mock.ReadCloser + var readN int + mrc.ReadFunc = func(p []byte) (int, error) { + readN++ + switch readN { + case 1: + copy(p, []byte("foo")) + return 3, nil + case 2: + copy(p, []byte("barbaz")) + return 6, nil + case 3: + return 0, io.EOF + default: + t.Fatal("unexpected read") + return 0, nil + } + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + // Setup handler on test server. + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + if off != 100 { + t.Fatalf("unexpected off: %d", off) + } + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + // Connect to server and stream all available data. + store := http.NewTranslateStore(s.URL) + rc, err := store.Reader(context.Background(), 100) + if err != nil { + t.Fatal(err) + } else if data, err := ioutil.ReadAll(rc); err != nil { + t.Fatal(err) + } else if string(data) != `foobarbaz` { + t.Fatalf("unexpected data: %q", data) + } else if err := rc.Close(); err != nil { + t.Fatal(err) + } + + if !closeInvoked { + t.Fatal("expected server close") + } + }) + + // Ensure server closes store reader if client disconnects. + t.Run("ClientDisconnect", func(t *testing.T) { + // Setup mock so that Read() hangs. + done := make(chan struct{}) + + var mrc mock.ReadCloser + mrc.ReadFunc = func(p []byte) (int, error) { + <-done + return 0, io.EOF + } + var closeInvoked bool + mrc.CloseFunc = func() error { + closeInvoked = true + return nil + } + + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return &mrc, nil + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + defer close(done) + + // Connect to server and begin streaming. + ctx, cancel := context.WithCancel(context.Background()) + store := http.NewTranslateStore(s.URL) + if _, err := store.Reader(ctx, 0); err != nil { + t.Fatal(err) + } + + // Cancel the context and check if server is closed. + cancel() + time.Sleep(100 * time.Millisecond) + if !closeInvoked { + t.Fatal("expected server-side close") + } + }) + }) + + // Ensure client is notified if the server doesn't support streaming replication. + t.Run("ErrNotImplemented", func(t *testing.T) { + var translateStore mock.TranslateStore + translateStore.ReaderFunc = func(ctx context.Context, off int64) (io.ReadCloser, error) { + return nil, pilosa.ErrNotImplemented + } + h := test.MustNewHandler() + h.API.TranslateStore = &translateStore + s := httptest.NewServer(h) + defer s.Close() + + _, err := http.NewTranslateStore(s.URL).Reader(context.Background(), 0) + if err != pilosa.ErrNotImplemented { + t.Fatalf("unexpected error: %s", err) + } + }) +} diff --git a/index.go b/index.go index 534930cd5..68c48c829 100644 --- a/index.go +++ b/index.go @@ -28,14 +28,15 @@ import ( "github.com/pkg/errors" ) -// Index represents a container for frames. +// Index represents a container for fields. type Index struct { mu sync.RWMutex path string name string + keys bool // use string keys - // Frames by name. - frames map[string]*Frame + // Fields by name. + fields map[string]*Field // Max Slice on any node in the cluster, according to this node. remoteMaxSlice uint64 @@ -53,7 +54,7 @@ type Index struct { // NewIndex returns a new instance of Index. func NewIndex(path, name string) (*Index, error) { - err := ValidateName(name) + err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } @@ -61,7 +62,7 @@ func NewIndex(path, name string) (*Index, error) { return &Index{ path: path, name: name, - frames: make(map[string]*Frame), + fields: make(map[string]*Field), remoteMaxSlice: 0, @@ -80,6 +81,9 @@ func (i *Index) Name() string { return i.name } // Path returns the path the index was initialized with. func (i *Index) Path() string { return i.path } +// Keys returns true if the index uses string keys. +func (i *Index) Keys() bool { return i.keys } + // ColumnAttrStore returns the storage for column attributes. func (i *Index) ColumnAttrStore() AttrStore { return i.columnAttrStore } @@ -106,8 +110,8 @@ func (i *Index) Open() error { return errors.Wrap(err, "loading meta file") } - if err := i.openFrames(); err != nil { - return errors.Wrap(err, "opening frames") + if err := i.openFields(); err != nil { + return errors.Wrap(err, "opening fields") } if err := i.columnAttrStore.Open(); err != nil { @@ -117,8 +121,8 @@ func (i *Index) Open() error { return nil } -// openFrames opens and initializes the frames inside the index. -func (i *Index) openFrames() error { +// openFields opens and initializes the fields inside the index. +func (i *Index) openFields() error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -135,14 +139,14 @@ func (i *Index) openFrames() error { continue } - fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + fld, err := i.newField(i.FieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if err != nil { return ErrName } - if err := fr.Open(); err != nil { - return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) + if err := fld.Open(); err != nil { + return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } - i.frames[fr.Name()] = fr + i.fields[fld.Name()] = fld } return nil } @@ -164,18 +168,17 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. + i.keys = pb.Keys return nil } -// NOTE: Until we introduce new attributes to store in the index .meta file, -// we don't need to actually write the file. The code related to index.options -// and the index meta file are left in place for future use. -/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{}) + buf, err := proto.Marshal(&internal.IndexMeta{ + Keys: i.keys, + }) if err != nil { return errors.Wrap(err, "marshalling") } @@ -187,9 +190,8 @@ func (i *Index) saveMeta() error { return nil } -*/ -// Close closes the index and its frames. +// Close closes the index and its fields. func (i *Index) Close() error { i.mu.Lock() defer i.mu.Unlock() @@ -197,13 +199,13 @@ func (i *Index) Close() error { // Close the attribute store. i.columnAttrStore.Close() - // Close all frames. - for _, f := range i.frames { + // Close all fields. + for _, f := range i.fields { if err := f.Close(); err != nil { - return errors.Wrap(err, "closing frame") + return errors.Wrap(err, "closing field") } } - i.frames = make(map[string]*Frame) + i.fields = make(map[string]*Field) return nil } @@ -217,7 +219,7 @@ func (i *Index) MaxSlice() uint64 { defer i.mu.RUnlock() max := i.remoteMaxSlice - for _, f := range i.frames { + for _, f := range i.fields { if slice := f.MaxSlice(); slice > max { max = slice } @@ -234,154 +236,139 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { i.remoteMaxSlice = newmax } -// FramePath returns the path to a frame in the index. -func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } +// FieldPath returns the path to a field in the index. +func (i *Index) FieldPath(name string) string { return filepath.Join(i.path, name) } -// Frame returns a frame in the index by name. -func (i *Index) Frame(name string) *Frame { +// Field returns a field in the index by name. +func (i *Index) Field(name string) *Field { i.mu.RLock() defer i.mu.RUnlock() - return i.frame(name) + return i.field(name) } -func (i *Index) frame(name string) *Frame { return i.frames[name] } +func (i *Index) field(name string) *Field { return i.fields[name] } -// Frames returns a list of all frames in the index. -func (i *Index) Frames() []*Frame { +// Fields returns a list of all fields in the index. +func (i *Index) Fields() []*Field { i.mu.RLock() defer i.mu.RUnlock() - a := make([]*Frame, 0, len(i.frames)) - for _, f := range i.frames { + a := make([]*Field, 0, len(i.fields)) + for _, f := range i.fields { a = append(a, f) } - sort.Sort(frameSlice(a)) + sort.Sort(fieldSlice(a)) return a } -// RecalculateCaches recalculates caches on every frame in the index. +// RecalculateCaches recalculates caches on every field in the index. func (i *Index) RecalculateCaches() { - for _, frame := range i.Frames() { - frame.RecalculateCaches() + for _, field := range i.Fields() { + field.RecalculateCaches() } } -// CreateFrame creates a frame. -func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) { +// CreateField creates a field. +func (i *Index) CreateField(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() - // Ensure frame doesn't already exist. - if i.frames[name] != nil { - return nil, ErrFrameExists + // Ensure field doesn't already exist. + if i.fields[name] != nil { + return nil, ErrFieldExists } - return i.createFrame(name, opt) + return i.createField(name, opt) } -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opt FieldOptions) (*Field, error) { i.mu.Lock() defer i.mu.Unlock() - // Find frame in cache first. - if f := i.frames[name]; f != nil { + // Find field in cache first. + if f := i.fields[name]; f != nil { return f, nil } - return i.createFrame(name, opt) + return i.createField(name, opt) } -func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { +func (i *Index) createField(name string, opt FieldOptions) (*Field, error) { if name == "" { - return nil, errors.New("frame name required") - } else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) { + return nil, errors.New("field name required") + } else if opt.CacheType != "" && !isValidCacheType(opt.CacheType) { return nil, ErrInvalidCacheType } - // Validate fields. - for _, field := range opt.Fields { - if err := ValidateField(field); err != nil { - return nil, err - } + // Validate options. + if err := opt.Validate(); err != nil { + return nil, errors.Wrap(err, "validating options") } - // Initialize frame. - f, err := i.newFrame(i.FramePath(name), name) + // Initialize field. + f, err := i.newField(i.FieldPath(name), name) if err != nil { return nil, errors.Wrap(err, "initializing") } - // Open frame. + // Open field. if err := f.Open(); err != nil { return nil, errors.Wrap(err, "opening") } - // Set the time quantum. - if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { + // Apply field options. + if err := f.applyOptions(opt); err != nil { f.Close() - return nil, errors.Wrap(err, "setting time quantum") + return nil, errors.Wrap(err, "applying options") } - // Set cache type. - if opt.CacheType == "" { - opt.CacheType = DefaultCacheType - } - f.cacheType = opt.CacheType - - if opt.CacheSize != 0 { - f.cacheSize = opt.CacheSize - } - - // Set fields. - f.fields = opt.Fields - if err := f.saveMeta(); err != nil { f.Close() return nil, errors.Wrap(err, "saving meta") } - // Add to index's frame lookup. - i.frames[name] = f + // Add to index's field lookup. + i.fields[name] = f return f, nil } -func (i *Index) newFrame(path, name string) (*Frame, error) { - f, err := NewFrame(path, i.name, name) +func (i *Index) newField(path, name string) (*Field, error) { + f, err := NewField(path, i.name, name) if err != nil { return nil, err } f.Logger = i.Logger - f.Stats = i.Stats.WithTags(fmt.Sprintf("frame:%s", name)) + f.Stats = i.Stats.WithTags(fmt.Sprintf("field:%s", name)) f.broadcaster = i.broadcaster f.rowAttrStore = i.NewAttrStore(filepath.Join(f.path, ".data")) return f, nil } -// DeleteFrame removes a frame from the index. -func (i *Index) DeleteFrame(name string) error { +// DeleteField removes a field from the index. +func (i *Index) DeleteField(name string) error { i.mu.Lock() defer i.mu.Unlock() - // Ignore if frame doesn't exist. - f := i.frame(name) + // Ignore if field doesn't exist. + f := i.field(name) if f == nil { return nil } - // Close frame. + // Close field. if err := f.Close(); err != nil { return errors.Wrap(err, "closing") } - // Delete frame directory. - if err := os.RemoveAll(i.FramePath(name)); err != nil { + // Delete field directory. + if err := os.RemoveAll(i.FieldPath(name)); err != nil { return errors.Wrap(err, "removing directory") } // Remove reference. - delete(i.frames, name) + delete(i.fields, name) return nil } @@ -395,7 +382,7 @@ func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() } // IndexInfo represents schema information for an index. type IndexInfo struct { Name string `json:"name"` - Frames []*FrameInfo `json:"frames"` + Fields []*FieldInfo `json:"fields"` } type indexInfoSlice []*IndexInfo @@ -417,16 +404,20 @@ func EncodeIndexes(a []*Index) []*internal.Index { func encodeIndex(d *Index) *internal.Index { return &internal.Index{ Name: d.name, - Frames: encodeFrames(d.Frames()), + Fields: encodeFields(d.Fields()), } } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct{} +type IndexOptions struct { + Keys bool `json:"keys"` +} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{} + return &internal.IndexMeta{ + Keys: i.Keys, + } } // hasTime returns true if a contains a non-nil time. diff --git a/filesystem.go b/index_internal_test.go similarity index 51% rename from filesystem.go rename to index_internal_test.go index 5664f0987..1e6d592ab 100644 --- a/filesystem.go +++ b/index_internal_test.go @@ -15,28 +15,32 @@ package pilosa import ( - "fmt" - "net/http" + "io/ioutil" ) -// Ensure nopFileSystem implements interface. -var _ FileSystem = &nopFileSystem{} - -// FileSystem represents an interface for a WebUI file system. -type FileSystem interface { - New() (http.FileSystem, error) +// mustOpenIndex returns a new, opened index at a temporary path. Panic on error. +func mustOpenIndex() *Index { + path, err := ioutil.TempDir("", "pilosa-index-") + if err != nil { + panic(err) + } + index, err := NewIndex(path, "i") + if err != nil { + panic(err) + } + if err := index.Open(); err != nil { + panic(err) + } + return index } -func init() { - NopFileSystem = &nopFileSystem{} -} - -// NopFileSystem represents a FileSystem that returns an error if called. -var NopFileSystem FileSystem - -type nopFileSystem struct{} - -// New is a no-op implementation of FileSystem New method. -func (n *nopFileSystem) New() (http.FileSystem, error) { - return nil, fmt.Errorf("file system not implemented") +// reopen closes the index and reopens it. +func (i *Index) reopen() error { + if err := i.Close(); err != nil { + return err + } + if err := i.Open(); err != nil { + return err + } + return nil } diff --git a/index_test.go b/index_test.go index 0a42804e2..5beb67f00 100644 --- a/index_test.go +++ b/index_test.go @@ -23,181 +23,184 @@ import ( "github.com/pilosa/pilosa/test" ) -// Ensure index can open and retrieve a frame. -func TestIndex_CreateFrameIfNotExists(t *testing.T) { +// SliceWidth is a helper reference to use when testing. +const SliceWidth = pilosa.SliceWidth + +// Ensure index can open and retrieve a field. +func TestIndex_CreateFieldIfNotExists(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + // Create field. + f, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) } else if f == nil { - t.Fatal("expected frame") + t.Fatal("expected field") } - // Retrieve existing frame. - other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) + // Retrieve existing field. + other, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}) if err != nil { t.Fatal(err) - } else if f.Frame != other.Frame { - t.Fatal("frame mismatch") + } else if f.Field != other.Field { + t.Fatal("field mismatch") } - if f.Frame != index.Frame("f") { - t.Fatal("frame mismatch") + if f.Field != index.Field("f") { + t.Fatal("field mismatch") } } -func TestIndex_CreateFrame(t *testing.T) { - // Ensure time quantum can be set appropriately on a new frame. +func TestIndex_CreateField(t *testing.T) { + // Ensure time quantum can be set appropriately on a new field. t.Run("TimeQuantum", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with explicit quantum. - f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) + // Create field with explicit quantum. + f, err := index.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeTime, + TimeQuantum: pilosa.TimeQuantum("YMDH"), + }) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected frame time quantum: %s", q) + t.Fatalf("unexpected field time quantum: %s", q) } }) }) - // Ensure frame can include range columns. + // Ensure field can include range columns. t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame with schema and verify it exists. - if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }, + // Create field with schema and verify it exists. + if f, err := index.CreateField("f", pilosa.FieldOptions{ + Type: pilosa.FieldTypeInt, + Min: 10, + Max: 20, }); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields: %#v", f.Fields()) + } else if !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { + t.Fatalf("unexpected type: %#v", f.Type()) } // Reopen the index & verify the fields are loaded. if err := index.Reopen(); err != nil { t.Fatal(err) - } else if f := index.Frame("f"); !reflect.DeepEqual(f.Fields(), []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, - {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, - }) { - t.Fatalf("unexpected fields after reopen: %#v", f.Fields()) + } else if f := index.Field("f"); !reflect.DeepEqual(f.Type(), pilosa.FieldTypeInt) { + t.Fatalf("unexpected type after reopen: %#v", f.Type()) } }) - t.Run("ErrRangeCacheAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + // TODO: These errors don't apply here. Instead, we need these tests + // on field creation FieldOptions validation. + /* + t.Run("ErrRangeCacheAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeRanked, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + CacheType: pilosa.CacheTypeRanked, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), - }); err != nil { - t.Fatal(err) - } - }) + t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() + if _, err := index.CreateField("f", pilosa.FieldOptions{ + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFieldFieldsAllowed", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt}, - }, - }); err != nil { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt}, + }, + }); err != nil { + t.Fatal(err) + } + }) - t.Run("ErrFieldNameRequired", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrFieldNameRequired", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "", Type: pilosa.FieldTypeInt}, - }, - }); err != pilosa.ErrFieldNameRequired { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "", Type: pilosa.FieldTypeInt}, + }, + }); err != pilosa.ErrFieldNameRequired { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldType", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidFieldType", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: "bad_type"}, - }, - }); err != pilosa.ErrInvalidFieldType { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: "bad_type"}, + }, + }); err != pilosa.ErrInvalidFieldType { + t.Fatal(err) + } + }) - t.Run("ErrInvalidFieldRange", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() + t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) { + index := test.MustOpenIndex() + defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - Fields: []*pilosa.Field{ - {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, - }, - }); err != pilosa.ErrInvalidFieldRange { - t.Fatal(err) - } - }) + if _, err := index.CreateField("f", pilosa.FieldOptions{ + Fields: []*pilosa.Field{ + {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, + }, + }); err != pilosa.ErrInvalidBSIGroupRange { + t.Fatal(err) + } + }) + */ }) } -// Ensure index can delete a frame. -func TestIndex_DeleteFrame(t *testing.T) { +// Ensure index can delete a field. +func TestIndex_DeleteField(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + // Create field. + if _, err := index.CreateFieldIfNotExists("f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Delete frame & verify it's gone. - if err := index.DeleteFrame("f"); err != nil { + // Delete field & verify it's gone. + if err := index.DeleteField("f"); err != nil { t.Fatal(err) - } else if index.Frame("f") != nil { - t.Fatal("expected nil frame") + } else if index.Field("f") != nil { + t.Fatal("expected nil field") } // Delete again to make sure it doesn't error. - if err := index.DeleteFrame("f"); err != nil { + if err := index.DeleteField("f"); err != nil { t.Fatal(err) } } -// Ensure index can delete a frame. +// Ensure index can delete a field. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") if err != nil { diff --git a/inmem/translator.go b/inmem/translator.go new file mode 100644 index 000000000..b620b2d03 --- /dev/null +++ b/inmem/translator.go @@ -0,0 +1,215 @@ +package inmem + +import ( + "context" + "io" + "sync" + + "github.com/pilosa/pilosa" +) + +// Ensure type implements interface. +var _ pilosa.TranslateStore = &TranslateStore{} + +// TranslateStore is an in-memory storage engine for translating string-to-uint64 values. +type TranslateStore struct { + mu sync.RWMutex + + cols map[string]*translateIndex + rows map[frameKey]*translateIndex +} + +// NewTranslateStore returns a new instance of TranslateStore. +func NewTranslateStore() *TranslateStore { + return &TranslateStore{ + cols: make(map[string]*translateIndex), + rows: make(map[frameKey]*translateIndex), + } +} + +// Reader returns an error because it is not supported by the inmem store. +func (s *TranslateStore) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + return nil, pilosa.ErrReplicationNotSupported +} + +// TranslateColumnsToUint64 converts value to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.cols[index] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateStore) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.lookup[values[i]] + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newTranslateIndex() + s.rows[key] = idx + } + + // Add new identifiers. + for i := range values { + if ret[i] != 0 { + continue + } + + idx.seq++ + v := idx.seq + ret[i] = v + idx.lookup[values[i]] = v + idx.reverse[v] = values[i] + } + + return ret, nil +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.reverse[value]; ok { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + return "", nil +} + +type frameKey struct { + index string + frame string +} + +type translateIndex struct { + seq uint64 + lookup map[string]uint64 + reverse map[uint64]string +} + +func newTranslateIndex() *translateIndex { + return &translateIndex{ + lookup: make(map[string]uint64), + reverse: make(map[uint64]string), + } +} diff --git a/inmem/translator_test.go b/inmem/translator_test.go new file mode 100644 index 000000000..d4d232566 --- /dev/null +++ b/inmem/translator_test.go @@ -0,0 +1,132 @@ +package inmem_test + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/pilosa/pilosa/inmem" +) + +func TestTranslateStore_TranslateColumn(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func TestTranslateStore_TranslateRow(t *testing.T) { + s := inmem.NewTranslateStore() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } +} + +func BenchmarkTranslateStore_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/1000) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateStore_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := inmem.NewTranslateStore() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/private.pb.go b/internal/private.pb.go index 35b452dce..c3dadb455 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -9,7 +9,7 @@ It has these top-level messages: IndexMeta - FrameMeta + FieldOptions ImportResponse BlockDataRequest BlockDataResponse @@ -18,11 +18,9 @@ CreateSliceMessage DeleteIndexMessage CreateIndexMessage - CreateFrameMessage - DeleteFrameMessage CreateFieldMessage DeleteFieldMessage - Frame + Field Schema Index URI @@ -31,7 +29,7 @@ NodeEventMessage NodeStatus ClusterStatus - Field + BSIGroup CreateViewMessage DeleteViewMessage ResizeInstruction @@ -62,6 +60,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { + Keys bool `protobuf:"varint,3,opt,name=Keys,proto3" json:"Keys,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -69,44 +68,75 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -type FrameMeta struct { - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` +func (m *IndexMeta) GetKeys() bool { + if m != nil { + return m.Keys + } + return false } -func (m *FrameMeta) Reset() { *m = FrameMeta{} } -func (m *FrameMeta) String() string { return proto.CompactTextString(m) } -func (*FrameMeta) ProtoMessage() {} -func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } +type FieldOptions struct { + Type string `protobuf:"bytes,8,opt,name=Type,proto3" json:"Type,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + Min int64 `protobuf:"varint,9,opt,name=Min,proto3" json:"Min,omitempty"` + Max int64 `protobuf:"varint,10,opt,name=Max,proto3" json:"Max,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Keys bool `protobuf:"varint,11,opt,name=Keys,proto3" json:"Keys,omitempty"` +} -func (m *FrameMeta) GetCacheType() string { +func (m *FieldOptions) Reset() { *m = FieldOptions{} } +func (m *FieldOptions) String() string { return proto.CompactTextString(m) } +func (*FieldOptions) ProtoMessage() {} +func (*FieldOptions) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } + +func (m *FieldOptions) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *FieldOptions) GetCacheType() string { if m != nil { return m.CacheType } return "" } -func (m *FrameMeta) GetCacheSize() uint32 { +func (m *FieldOptions) GetCacheSize() uint32 { if m != nil { return m.CacheSize } return 0 } -func (m *FrameMeta) GetTimeQuantum() string { +func (m *FieldOptions) GetMin() int64 { + if m != nil { + return m.Min + } + return 0 +} + +func (m *FieldOptions) GetMax() int64 { + if m != nil { + return m.Max + } + return 0 +} + +func (m *FieldOptions) GetTimeQuantum() string { if m != nil { return m.TimeQuantum } return "" } -func (m *FrameMeta) GetFields() []*Field { +func (m *FieldOptions) GetKeys() bool { if m != nil { - return m.Fields + return m.Keys } - return nil + return false } type ImportResponse struct { @@ -127,7 +157,7 @@ func (m *ImportResponse) GetErr() string { type BlockDataRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"` Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"` Block uint64 `protobuf:"varint,3,opt,name=Block,proto3" json:"Block,omitempty"` @@ -145,9 +175,9 @@ func (m *BlockDataRequest) GetIndex() string { return "" } -func (m *BlockDataRequest) GetFrame() string { +func (m *BlockDataRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -293,72 +323,16 @@ func (m *CreateIndexMessage) GetMeta() *IndexMeta { return nil } -type CreateFrameMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` -} - -func (m *CreateFrameMessage) Reset() { *m = CreateFrameMessage{} } -func (m *CreateFrameMessage) String() string { return proto.CompactTextString(m) } -func (*CreateFrameMessage) ProtoMessage() {} -func (*CreateFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } - -func (m *CreateFrameMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *CreateFrameMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - -func (m *CreateFrameMessage) GetMeta() *FrameMeta { - if m != nil { - return m.Meta - } - return nil -} - -type DeleteFrameMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` -} - -func (m *DeleteFrameMessage) Reset() { *m = DeleteFrameMessage{} } -func (m *DeleteFrameMessage) String() string { return proto.CompactTextString(m) } -func (*DeleteFrameMessage) ProtoMessage() {} -func (*DeleteFrameMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } - -func (m *DeleteFrameMessage) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *DeleteFrameMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - type CreateFieldMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field *Field `protobuf:"bytes,3,opt,name=Field" json:"Field,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Meta *FieldOptions `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"` } func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} -func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } +func (*CreateFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{10} } func (m *CreateFieldMessage) GetIndex() string { if m != nil { @@ -367,30 +341,29 @@ func (m *CreateFieldMessage) GetIndex() string { return "" } -func (m *CreateFieldMessage) GetFrame() string { +func (m *CreateFieldMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } -func (m *CreateFieldMessage) GetField() *Field { +func (m *CreateFieldMessage) GetMeta() *FieldOptions { if m != nil { - return m.Field + return m.Meta } return nil } type DeleteFieldMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` - Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` } func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} -func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } +func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{11} } func (m *DeleteFieldMessage) GetIndex() string { if m != nil { @@ -399,13 +372,6 @@ func (m *DeleteFieldMessage) GetIndex() string { return "" } -func (m *DeleteFieldMessage) GetFrame() string { - if m != nil { - return m.Frame - } - return "" -} - func (m *DeleteFieldMessage) GetField() string { if m != nil { return m.Field @@ -413,32 +379,32 @@ func (m *DeleteFieldMessage) GetField() string { return "" } -type Frame struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Meta *FrameMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` - Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` +type Field struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Meta *FieldOptions `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"` + Views []string `protobuf:"bytes,3,rep,name=Views" json:"Views,omitempty"` } -func (m *Frame) Reset() { *m = Frame{} } -func (m *Frame) String() string { return proto.CompactTextString(m) } -func (*Frame) ProtoMessage() {} -func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{12} } -func (m *Frame) GetName() string { +func (m *Field) GetName() string { if m != nil { return m.Name } return "" } -func (m *Frame) GetMeta() *FrameMeta { +func (m *Field) GetMeta() *FieldOptions { if m != nil { return m.Meta } return nil } -func (m *Frame) GetViews() []string { +func (m *Field) GetViews() []string { if m != nil { return m.Views } @@ -452,7 +418,7 @@ type Schema struct { func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} -func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } +func (*Schema) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} } func (m *Schema) GetIndexes() []*Index { if m != nil { @@ -463,13 +429,13 @@ func (m *Schema) GetIndexes() []*Index { type Index struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"` + Fields []*Field `protobuf:"bytes,4,rep,name=Fields" json:"Fields,omitempty"` } func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} -func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } +func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} } func (m *Index) GetName() string { if m != nil { @@ -478,9 +444,9 @@ func (m *Index) GetName() string { return "" } -func (m *Index) GetFrames() []*Frame { +func (m *Index) GetFields() []*Field { if m != nil { - return m.Frames + return m.Fields } return nil } @@ -494,7 +460,7 @@ type URI struct { func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} -func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } +func (*URI) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} } func (m *URI) GetScheme() string { if m != nil { @@ -526,7 +492,7 @@ type Node struct { func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } +func (*Node) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{16} } func (m *Node) GetID() string { if m != nil { @@ -557,7 +523,7 @@ type NodeStateMessage struct { func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} -func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } +func (*NodeStateMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{17} } func (m *NodeStateMessage) GetNodeID() string { if m != nil { @@ -581,7 +547,7 @@ type NodeEventMessage struct { func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} -func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } +func (*NodeEventMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{18} } func (m *NodeEventMessage) GetEvent() uint32 { if m != nil { @@ -606,7 +572,7 @@ type NodeStatus struct { func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} -func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } +func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{19} } func (m *NodeStatus) GetNode() *Node { if m != nil { @@ -638,7 +604,7 @@ type ClusterStatus struct { func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} -func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } +func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{20} } func (m *ClusterStatus) GetClusterID() string { if m != nil { @@ -661,40 +627,40 @@ func (m *ClusterStatus) GetNodes() []*Node { return nil } -type Field struct { +type BSIGroup struct { Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` Type string `protobuf:"bytes,2,opt,name=Type,proto3" json:"Type,omitempty"` Min int64 `protobuf:"varint,3,opt,name=Min,proto3" json:"Min,omitempty"` Max int64 `protobuf:"varint,4,opt,name=Max,proto3" json:"Max,omitempty"` } -func (m *Field) Reset() { *m = Field{} } -func (m *Field) String() string { return proto.CompactTextString(m) } -func (*Field) ProtoMessage() {} -func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } +func (m *BSIGroup) Reset() { *m = BSIGroup{} } +func (m *BSIGroup) String() string { return proto.CompactTextString(m) } +func (*BSIGroup) ProtoMessage() {} +func (*BSIGroup) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{21} } -func (m *Field) GetName() string { +func (m *BSIGroup) GetName() string { if m != nil { return m.Name } return "" } -func (m *Field) GetType() string { +func (m *BSIGroup) GetType() string { if m != nil { return m.Type } return "" } -func (m *Field) GetMin() int64 { +func (m *BSIGroup) GetMin() int64 { if m != nil { return m.Min } return 0 } -func (m *Field) GetMax() int64 { +func (m *BSIGroup) GetMax() int64 { if m != nil { return m.Max } @@ -703,14 +669,14 @@ func (m *Field) GetMax() int64 { type CreateViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} -func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } +func (*CreateViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{22} } func (m *CreateViewMessage) GetIndex() string { if m != nil { @@ -719,9 +685,9 @@ func (m *CreateViewMessage) GetIndex() string { return "" } -func (m *CreateViewMessage) GetFrame() string { +func (m *CreateViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -735,14 +701,14 @@ func (m *CreateViewMessage) GetView() string { type DeleteViewMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,3,opt,name=View,proto3" json:"View,omitempty"` } func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} -func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } +func (*DeleteViewMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{23} } func (m *DeleteViewMessage) GetIndex() string { if m != nil { @@ -751,9 +717,9 @@ func (m *DeleteViewMessage) GetIndex() string { return "" } -func (m *DeleteViewMessage) GetFrame() string { +func (m *DeleteViewMessage) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -777,7 +743,7 @@ type ResizeInstruction struct { func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} -func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{26} } +func (*ResizeInstruction) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{24} } func (m *ResizeInstruction) GetJobID() int64 { if m != nil { @@ -824,7 +790,7 @@ func (m *ResizeInstruction) GetClusterStatus() *ClusterStatus { type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,3,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` Slice uint64 `protobuf:"varint,5,opt,name=Slice,proto3" json:"Slice,omitempty"` } @@ -832,7 +798,7 @@ type ResizeSource struct { func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} -func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } +func (*ResizeSource) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{25} } func (m *ResizeSource) GetNode() *Node { if m != nil { @@ -848,9 +814,9 @@ func (m *ResizeSource) GetIndex() string { return "" } -func (m *ResizeSource) GetFrame() string { +func (m *ResizeSource) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -879,7 +845,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptorPrivate, []int{28} + return fileDescriptorPrivate, []int{26} } func (m *ResizeInstructionComplete) GetJobID() int64 { @@ -910,7 +876,7 @@ type SetCoordinatorMessage struct { func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} -func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } +func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{27} } func (m *SetCoordinatorMessage) GetNew() *Node { if m != nil { @@ -926,7 +892,7 @@ type UpdateCoordinatorMessage struct { func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessage{} } func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} -func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } +func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{28} } func (m *UpdateCoordinatorMessage) GetNew() *Node { if m != nil { @@ -943,7 +909,7 @@ type Topology struct { func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} -func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{31} } +func (*Topology) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{29} } func (m *Topology) GetClusterID() string { if m != nil { @@ -965,11 +931,11 @@ type RecalculateCaches struct { func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} -func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{32} } +func (*RecalculateCaches) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{30} } func init() { proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta") - proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta") + proto.RegisterType((*FieldOptions)(nil), "internal.FieldOptions") proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") @@ -978,11 +944,9 @@ func init() { proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage") proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage") proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage") - proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage") - proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage") proto.RegisterType((*CreateFieldMessage)(nil), "internal.CreateFieldMessage") proto.RegisterType((*DeleteFieldMessage)(nil), "internal.DeleteFieldMessage") - proto.RegisterType((*Frame)(nil), "internal.Frame") + proto.RegisterType((*Field)(nil), "internal.Field") proto.RegisterType((*Schema)(nil), "internal.Schema") proto.RegisterType((*Index)(nil), "internal.Index") proto.RegisterType((*URI)(nil), "internal.URI") @@ -991,7 +955,7 @@ func init() { proto.RegisterType((*NodeEventMessage)(nil), "internal.NodeEventMessage") proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus") proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus") - proto.RegisterType((*Field)(nil), "internal.Field") + proto.RegisterType((*BSIGroup)(nil), "internal.BSIGroup") proto.RegisterType((*CreateViewMessage)(nil), "internal.CreateViewMessage") proto.RegisterType((*DeleteViewMessage)(nil), "internal.DeleteViewMessage") proto.RegisterType((*ResizeInstruction)(nil), "internal.ResizeInstruction") @@ -1017,10 +981,20 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Keys { + dAtA[i] = 0x18 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } -func (m *FrameMeta) Marshal() (dAtA []byte, err error) { +func (m *FieldOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1030,7 +1004,7 @@ func (m *FrameMeta) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { +func (m *FieldOptions) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1052,17 +1026,31 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } - if len(m.Fields) > 0 { - for _, msg := range m.Fields { - dAtA[i] = 0x3a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n + if len(m.Type) > 0 { + dAtA[i] = 0x42 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Type))) + i += copy(dAtA[i:], m.Type) + } + if m.Min != 0 { + dAtA[i] = 0x48 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Min)) + } + if m.Max != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintPrivate(dAtA, i, uint64(m.Max)) + } + if m.Keys { + dAtA[i] = 0x58 + i++ + if m.Keys { + dAtA[i] = 1 + } else { + dAtA[i] = 0 } + i++ } return i, nil } @@ -1112,11 +1100,11 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Block != 0 { dAtA[i] = 0x18 @@ -1345,76 +1333,6 @@ func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *CreateFrameMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - if m.Meta != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n8, err := m.Meta.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n8 - } - return i, nil -} - -func (m *DeleteFrameMessage) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Index) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) - i += copy(dAtA[i:], m.Index) - } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } - return i, nil -} - func (m *CreateFieldMessage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1436,21 +1354,21 @@ func (m *CreateFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } - if m.Field != nil { + if m.Meta != nil { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Field.Size())) - n9, err := m.Field.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) + n8, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n8 } return i, nil } @@ -1476,14 +1394,8 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) - } if len(m.Field) > 0 { - dAtA[i] = 0x1a + dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) i += copy(dAtA[i:], m.Field) @@ -1491,7 +1403,7 @@ func (m *DeleteFieldMessage) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Frame) Marshal() (dAtA []byte, err error) { +func (m *Field) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1501,7 +1413,7 @@ func (m *Frame) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Frame) MarshalTo(dAtA []byte) (int, error) { +func (m *Field) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1516,11 +1428,11 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Meta.Size())) - n10, err := m.Meta.MarshalTo(dAtA[i:]) + n9, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n9 } if len(m.Views) > 0 { for _, s := range m.Views { @@ -1591,8 +1503,8 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } - if len(m.Frames) > 0 { - for _, msg := range m.Frames { + if len(m.Fields) > 0 { + for _, msg := range m.Fields { dAtA[i] = 0x22 i++ i = encodeVarintPrivate(dAtA, i, uint64(msg.Size())) @@ -1666,11 +1578,11 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.URI.Size())) - n11, err := m.URI.MarshalTo(dAtA[i:]) + n10, err := m.URI.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n10 } if m.IsCoordinator { dAtA[i] = 0x18 @@ -1739,11 +1651,11 @@ func (m *NodeEventMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n12, err := m.Node.MarshalTo(dAtA[i:]) + n11, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n11 } return i, nil } @@ -1767,31 +1679,31 @@ func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n13, err := m.Node.MarshalTo(dAtA[i:]) + n12, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n12 } if m.MaxSlices != nil { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.MaxSlices.Size())) - n14, err := m.MaxSlices.MarshalTo(dAtA[i:]) + n13, err := m.MaxSlices.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n13 } if m.Schema != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n15, err := m.Schema.MarshalTo(dAtA[i:]) + n14, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n15 + i += n14 } return i, nil } @@ -1838,7 +1750,7 @@ func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Field) Marshal() (dAtA []byte, err error) { +func (m *BSIGroup) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -1848,7 +1760,7 @@ func (m *Field) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Field) MarshalTo(dAtA []byte) (int, error) { +func (m *BSIGroup) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1899,11 +1811,11 @@ func (m *CreateViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1935,11 +1847,11 @@ func (m *DeleteViewMessage) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x1a @@ -1974,21 +1886,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n16, err := m.Node.MarshalTo(dAtA[i:]) + n15, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n15 } if m.Coordinator != nil { dAtA[i] = 0x1a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Coordinator.Size())) - n17, err := m.Coordinator.MarshalTo(dAtA[i:]) + n16, err := m.Coordinator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n16 } if len(m.Sources) > 0 { for _, msg := range m.Sources { @@ -2006,21 +1918,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + n17, err := m.Schema.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n17 } if m.ClusterStatus != nil { dAtA[i] = 0x32 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n18 } return i, nil } @@ -2044,11 +1956,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n20, err := m.Node.MarshalTo(dAtA[i:]) + n19, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n20 + i += n19 } if len(m.Index) > 0 { dAtA[i] = 0x12 @@ -2056,11 +1968,11 @@ func (m *ResizeSource) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if len(m.View) > 0 { dAtA[i] = 0x22 @@ -2100,11 +2012,11 @@ func (m *ResizeInstructionComplete) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Node.Size())) - n21, err := m.Node.MarshalTo(dAtA[i:]) + n20, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n20 } if len(m.Error) > 0 { dAtA[i] = 0x1a @@ -2134,11 +2046,11 @@ func (m *SetCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n22, err := m.New.MarshalTo(dAtA[i:]) + n21, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n21 } return i, nil } @@ -2162,11 +2074,11 @@ func (m *UpdateCoordinatorMessage) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPrivate(dAtA, i, uint64(m.New.Size())) - n23, err := m.New.MarshalTo(dAtA[i:]) + n22, err := m.New.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n23 + i += n22 } return i, nil } @@ -2240,10 +2152,13 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l + if m.Keys { + n += 2 + } return n } -func (m *FrameMeta) Size() (n int) { +func (m *FieldOptions) Size() (n int) { var l int _ = l l = len(m.CacheType) @@ -2257,11 +2172,18 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Fields) > 0 { - for _, e := range m.Fields { - l = e.Size() - n += 1 + l + sovPrivate(uint64(l)) - } + l = len(m.Type) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if m.Min != 0 { + n += 1 + sovPrivate(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovPrivate(uint64(m.Max)) + } + if m.Keys { + n += 2 } return n } @@ -2283,7 +2205,7 @@ func (m *BlockDataRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2384,38 +2306,6 @@ func (m *CreateIndexMessage) Size() (n int) { return n } -func (m *CreateFrameMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - if m.Meta != nil { - l = m.Meta.Size() - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - -func (m *DeleteFrameMessage) Size() (n int) { - var l int - _ = l - l = len(m.Index) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } - return n -} - func (m *CreateFieldMessage) Size() (n int) { var l int _ = l @@ -2423,12 +2313,12 @@ func (m *CreateFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.Field != nil { - l = m.Field.Size() + if m.Meta != nil { + l = m.Meta.Size() n += 1 + l + sovPrivate(uint64(l)) } return n @@ -2441,10 +2331,6 @@ func (m *DeleteFieldMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -2452,7 +2338,7 @@ func (m *DeleteFieldMessage) Size() (n int) { return n } -func (m *Frame) Size() (n int) { +func (m *Field) Size() (n int) { var l int _ = l l = len(m.Name) @@ -2491,8 +2377,8 @@ func (m *Index) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if len(m.Frames) > 0 { - for _, e := range m.Frames { + if len(m.Fields) > 0 { + for _, e := range m.Fields { l = e.Size() n += 1 + l + sovPrivate(uint64(l)) } @@ -2599,7 +2485,7 @@ func (m *ClusterStatus) Size() (n int) { return n } -func (m *Field) Size() (n int) { +func (m *BSIGroup) Size() (n int) { var l int _ = l l = len(m.Name) @@ -2626,7 +2512,7 @@ func (m *CreateViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2644,7 +2530,7 @@ func (m *DeleteViewMessage) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2697,7 +2583,7 @@ func (m *ResizeSource) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } @@ -2812,6 +2698,26 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -2833,7 +2739,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { } return nil } -func (m *FrameMeta) Unmarshal(dAtA []byte) error { +func (m *FieldOptions) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2856,10 +2762,10 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: FrameMeta: wiretype end group for non-group") + return fmt.Errorf("proto: FieldOptions: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FieldOptions: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 3: @@ -2939,11 +2845,11 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 7: + case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPrivate @@ -2953,23 +2859,79 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPrivate } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex > l { return io.ErrUnexpectedEOF } - m.Fields = append(m.Fields, &Field{}) - if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Type = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3130,7 +3092,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3155,7 +3117,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3977,255 +3939,6 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: CreateFrameMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: CreateFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Meta == nil { - m.Meta = &FrameMeta{} - } - if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: DeleteFrameMessage: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: DeleteFrameMessage: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Index = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -4286,7 +3999,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -4311,11 +4024,11 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Meta", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4339,10 +4052,10 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Field == nil { - m.Field = &Field{} + if m.Meta == nil { + m.Meta = &FieldOptions{} } - if err := m.Field.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4426,35 +4139,6 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } @@ -4504,7 +4188,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { } return nil } -func (m *Frame) Unmarshal(dAtA []byte) error { +func (m *Field) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4527,10 +4211,10 @@ func (m *Frame) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Frame: wiretype end group for non-group") + return fmt.Errorf("proto: Field: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4589,7 +4273,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Meta == nil { - m.Meta = &FrameMeta{} + m.Meta = &FieldOptions{} } if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -4786,7 +4470,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frames", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -4810,8 +4494,8 @@ func (m *Index) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frames = append(m.Frames, &Frame{}) - if err := m.Frames[len(m.Frames)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Fields = append(m.Fields, &Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -5593,7 +5277,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { } return nil } -func (m *Field) Unmarshal(dAtA []byte) error { +func (m *BSIGroup) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -5616,10 +5300,10 @@ func (m *Field) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Field: wiretype end group for non-group") + return fmt.Errorf("proto: BSIGroup: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: BSIGroup: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -5799,7 +5483,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5824,7 +5508,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -5936,7 +5620,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -5961,7 +5645,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { @@ -6338,7 +6022,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -6363,7 +6047,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 4: if wireType != 2 { @@ -6997,70 +6681,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1035 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, - 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, - 0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9, - 0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85, - 0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, - 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce, - 0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40, - 0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1, - 0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b, - 0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90, - 0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c, - 0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c, - 0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64, - 0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c, - 0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68, - 0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e, - 0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc, - 0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d, - 0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f, - 0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa, - 0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9, - 0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12, - 0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2, - 0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1, - 0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1, - 0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4, - 0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce, - 0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09, - 0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b, - 0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99, - 0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f, - 0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb, - 0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88, - 0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39, - 0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56, - 0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b, - 0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4, - 0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07, - 0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6, - 0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3, - 0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a, - 0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1, - 0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8, - 0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd, - 0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf, - 0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3, - 0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c, - 0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d, - 0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8, - 0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69, - 0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08, - 0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33, - 0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac, - 0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d, - 0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b, - 0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34, - 0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19, - 0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53, - 0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f, - 0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96, - 0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31, - 0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58, - 0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f, - 0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00, - 0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00, + // 1028 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x72, 0x1c, 0x35, + 0x17, 0xfe, 0xfb, 0x32, 0xe3, 0x99, 0xe3, 0x8c, 0x7f, 0x5b, 0x01, 0xd3, 0xa1, 0x28, 0x67, 0x50, + 0xa5, 0x2a, 0x26, 0x0b, 0x57, 0x48, 0x36, 0xdc, 0x52, 0xe5, 0xb2, 0xc7, 0x40, 0x03, 0x36, 0xa0, + 0xb6, 0xb3, 0xcb, 0x42, 0x99, 0x51, 0x25, 0x5d, 0xee, 0x69, 0x35, 0xdd, 0x6a, 0xdb, 0x93, 0x05, + 0x5b, 0xd8, 0xb0, 0xa7, 0x78, 0x12, 0x1e, 0x81, 0x25, 0x8f, 0x40, 0x99, 0x17, 0xa1, 0x74, 0xa4, + 0xbe, 0xd8, 0x33, 0x8e, 0x53, 0x86, 0x9d, 0xce, 0xfd, 0xd3, 0xd1, 0x77, 0x24, 0xc1, 0x20, 0xcb, + 0xe3, 0x13, 0xae, 0xc4, 0x56, 0x96, 0x4b, 0x25, 0x49, 0x2f, 0x4e, 0x95, 0xc8, 0x53, 0x9e, 0xd0, + 0xbb, 0xd0, 0x0f, 0xd3, 0x89, 0x38, 0xdb, 0x17, 0x8a, 0x13, 0x02, 0xfe, 0xd7, 0x62, 0x56, 0x04, + 0xde, 0xd0, 0xd9, 0xec, 0x31, 0x5c, 0xd3, 0xdf, 0x1d, 0xb8, 0xf5, 0x79, 0x2c, 0x92, 0xc9, 0xb7, + 0x99, 0x8a, 0x65, 0x5a, 0x90, 0xf7, 0xa0, 0xbf, 0xcb, 0xc7, 0x2f, 0xc5, 0xe1, 0x2c, 0x13, 0xe8, + 0xd9, 0x67, 0x8d, 0xa2, 0xb6, 0x46, 0xf1, 0x2b, 0x11, 0xf8, 0x43, 0x67, 0x73, 0xc0, 0x1a, 0x05, + 0x19, 0xc2, 0xf2, 0x61, 0x3c, 0x15, 0xdf, 0x97, 0x3c, 0x55, 0xe5, 0x34, 0xe8, 0x60, 0x74, 0x5b, + 0xa5, 0x21, 0x60, 0xe2, 0x1e, 0x9a, 0x70, 0x4d, 0x56, 0xc1, 0xdb, 0x8f, 0xd3, 0xa0, 0x3f, 0x74, + 0x36, 0x3d, 0xa6, 0x97, 0xa8, 0xe1, 0x67, 0x01, 0x58, 0x0d, 0x3f, 0xab, 0xa1, 0x2f, 0xb7, 0xa0, + 0x53, 0x58, 0x09, 0xa7, 0x99, 0xcc, 0x15, 0x13, 0x45, 0x26, 0xd3, 0x02, 0x33, 0xed, 0xe5, 0x79, + 0xe0, 0x60, 0x72, 0xbd, 0xa4, 0x3f, 0xc2, 0xea, 0x4e, 0x22, 0xc7, 0xc7, 0x23, 0xae, 0x38, 0x13, + 0x3f, 0x94, 0xa2, 0x50, 0xe4, 0x2d, 0xe8, 0x60, 0x4f, 0xac, 0x9f, 0x11, 0xb4, 0x16, 0xfb, 0x10, + 0xb8, 0x46, 0x8b, 0x82, 0xd6, 0x62, 0x3c, 0x76, 0xc2, 0x67, 0x46, 0xd0, 0xda, 0x28, 0x89, 0xc7, + 0xa6, 0x03, 0x3e, 0x33, 0x82, 0xc6, 0xf8, 0x34, 0x16, 0xa7, 0x76, 0xdb, 0xb8, 0xa6, 0x21, 0xac, + 0xb5, 0xea, 0x5b, 0x98, 0xeb, 0xd0, 0x65, 0xf2, 0x34, 0x1c, 0x15, 0x81, 0x33, 0xf4, 0x36, 0x7d, + 0x66, 0x25, 0x6c, 0xae, 0x4c, 0xca, 0x69, 0xaa, 0x4d, 0x2e, 0x9a, 0x1a, 0x05, 0xbd, 0x03, 0x1d, + 0xec, 0xb4, 0xde, 0x65, 0x13, 0xab, 0x97, 0xf4, 0x27, 0x07, 0xfa, 0xfb, 0xfc, 0x0c, 0x61, 0x14, + 0xe4, 0x09, 0xf4, 0x22, 0xc5, 0xd3, 0x09, 0xcf, 0x27, 0xe8, 0xb4, 0xfc, 0xe8, 0xfd, 0xad, 0x8a, + 0x10, 0x5b, 0xb5, 0xdb, 0x56, 0xe5, 0xb3, 0x97, 0xaa, 0x7c, 0xc6, 0xea, 0x90, 0x77, 0x3f, 0x85, + 0xc1, 0x05, 0x93, 0xae, 0x77, 0x2c, 0x66, 0x55, 0x57, 0x8f, 0xc5, 0x4c, 0xef, 0xff, 0x84, 0x27, + 0xa5, 0xc0, 0x5e, 0xf9, 0xcc, 0x08, 0x9f, 0xb8, 0x1f, 0x39, 0x74, 0x1b, 0xc8, 0x6e, 0x2e, 0xb8, + 0x12, 0x58, 0x64, 0x5f, 0x14, 0x05, 0x7f, 0x21, 0xae, 0xee, 0xb8, 0xe9, 0xa2, 0xdb, 0xea, 0x22, + 0x7d, 0x00, 0x64, 0x24, 0x12, 0xa1, 0x84, 0xe5, 0xed, 0x6b, 0x32, 0xd0, 0xa8, 0xaa, 0x76, 0xbd, + 0x2f, 0xb9, 0x0f, 0xbe, 0x1e, 0x02, 0x2c, 0xb6, 0xfc, 0xe8, 0x76, 0xd3, 0x91, 0x7a, 0x3e, 0x18, + 0x3a, 0xd0, 0xa4, 0x4a, 0x8a, 0x0c, 0xb8, 0x76, 0x0b, 0x0b, 0x48, 0xf3, 0xc0, 0x96, 0xf2, 0xb0, + 0xd4, 0x7a, 0x53, 0xaa, 0x3d, 0x68, 0xb6, 0xda, 0x76, 0xb5, 0xdd, 0x9b, 0x56, 0xa3, 0xcf, 0xac, + 0x56, 0xf3, 0xef, 0x80, 0x4f, 0x85, 0x8d, 0xc1, 0x75, 0x0d, 0xc5, 0xbd, 0x1e, 0x8a, 0x4e, 0xaf, + 0x39, 0xab, 0xef, 0x07, 0x4f, 0xa7, 0x47, 0x81, 0x3e, 0x86, 0x6e, 0x34, 0x7e, 0x29, 0xa6, 0x9c, + 0x7c, 0x00, 0x4b, 0x88, 0x43, 0x14, 0x96, 0x56, 0xff, 0xbf, 0xd4, 0x44, 0x56, 0xd9, 0xe9, 0xc8, + 0xe2, 0x5f, 0x88, 0xe9, 0x3e, 0x74, 0xb1, 0x7a, 0x11, 0xf8, 0x97, 0xd3, 0xa0, 0x9e, 0x59, 0x33, + 0xdd, 0x03, 0xef, 0x88, 0x85, 0x7a, 0x5c, 0x10, 0x41, 0x95, 0xc5, 0x4a, 0x3a, 0xf7, 0x97, 0xb2, + 0x50, 0xb6, 0x1b, 0xb8, 0xd6, 0xba, 0xef, 0x64, 0xae, 0xb0, 0xf5, 0x03, 0x86, 0x6b, 0xfa, 0x0c, + 0xfc, 0x03, 0x39, 0x11, 0x64, 0x05, 0xdc, 0x70, 0x64, 0x73, 0xb8, 0xe1, 0x88, 0xdc, 0xc5, 0xf4, + 0xb6, 0x35, 0x83, 0x06, 0xc4, 0x11, 0x0b, 0x19, 0x16, 0xbe, 0x07, 0x83, 0xb0, 0xd8, 0x95, 0x32, + 0x9f, 0xc4, 0x29, 0x57, 0x32, 0xb7, 0x17, 0xe7, 0x45, 0x25, 0xdd, 0x86, 0x55, 0x9d, 0x3e, 0x52, + 0x5c, 0xd5, 0x84, 0x5f, 0x87, 0xae, 0xd6, 0xd5, 0xe5, 0xac, 0x84, 0x94, 0xd7, 0x7e, 0xd5, 0x09, + 0xa2, 0x40, 0xbf, 0x31, 0x19, 0xf6, 0x4e, 0x44, 0xaa, 0x5a, 0x0c, 0x40, 0x19, 0x13, 0x0c, 0x98, + 0x11, 0x08, 0x35, 0x5b, 0xb1, 0x98, 0x57, 0x1a, 0xcc, 0x5a, 0xcb, 0xd0, 0x46, 0x7f, 0x71, 0x00, + 0x2a, 0x40, 0x65, 0x51, 0x87, 0x38, 0x57, 0x87, 0x90, 0x0f, 0x5b, 0xd7, 0xc7, 0xfc, 0x80, 0xd4, + 0x26, 0xd6, 0xba, 0x64, 0x36, 0x2b, 0x5a, 0x58, 0x96, 0xaf, 0x36, 0xfe, 0x46, 0x6f, 0x8f, 0x89, + 0xd3, 0x18, 0x06, 0xbb, 0x49, 0x59, 0x28, 0x91, 0x5b, 0x44, 0xfa, 0x9a, 0x33, 0x8a, 0xba, 0x3f, + 0x8d, 0x62, 0x71, 0x8b, 0xc8, 0x3d, 0xe8, 0x68, 0xa4, 0x86, 0x9b, 0xf3, 0xdb, 0x30, 0x46, 0xfa, + 0x14, 0x7a, 0x3b, 0x51, 0xf8, 0x45, 0x2e, 0xcb, 0x6c, 0x21, 0xf3, 0xaa, 0xd7, 0xc7, 0x9d, 0x7f, + 0x7d, 0xbc, 0xb9, 0xd7, 0xc7, 0xaf, 0x5f, 0x1f, 0x1a, 0xc1, 0x9a, 0xb9, 0x12, 0xf4, 0x48, 0xdc, + 0xe4, 0x46, 0xa8, 0x9e, 0x06, 0xaf, 0xf5, 0x34, 0x44, 0xb0, 0x66, 0x26, 0xff, 0xbf, 0x4c, 0xfa, + 0x9b, 0x0b, 0x6b, 0x4c, 0x14, 0xf1, 0x2b, 0x11, 0xa6, 0x85, 0xca, 0xcb, 0xb1, 0x1e, 0x70, 0x1d, + 0xff, 0x95, 0x7c, 0x6e, 0xbb, 0xed, 0x31, 0x23, 0xbc, 0x09, 0x99, 0xc8, 0x43, 0x58, 0xbe, 0x3c, + 0x00, 0xf3, 0xae, 0x6d, 0x17, 0xf2, 0x10, 0x96, 0x22, 0x59, 0xe6, 0x9a, 0x49, 0x66, 0xbc, 0x5b, + 0x97, 0x8e, 0x41, 0x66, 0xcc, 0xac, 0x72, 0x6b, 0x51, 0xa9, 0xf3, 0x7a, 0x2a, 0x91, 0x27, 0x97, + 0xa8, 0x14, 0x74, 0x31, 0xe0, 0x9d, 0x26, 0xe0, 0x82, 0x99, 0x5d, 0xf4, 0xa6, 0x3f, 0x3b, 0x70, + 0xab, 0x0d, 0xe1, 0x8d, 0x66, 0xa3, 0x3e, 0x11, 0x77, 0xe1, 0x89, 0x78, 0x8b, 0x4e, 0xc4, 0x6f, + 0x4e, 0xa4, 0x79, 0xe5, 0x3a, 0xed, 0x57, 0xee, 0x18, 0xee, 0xcc, 0x1d, 0xd3, 0xae, 0x9c, 0x66, + 0x9a, 0x0f, 0xff, 0xe2, 0xb8, 0xf4, 0xad, 0x91, 0xe7, 0xf6, 0xa0, 0xfa, 0xcc, 0x08, 0xf4, 0x63, + 0x78, 0x3b, 0x12, 0xaa, 0x75, 0x48, 0x15, 0xdb, 0x86, 0xe0, 0x1d, 0x88, 0xd3, 0x2b, 0xb6, 0xaf, + 0x4d, 0xf4, 0x33, 0x08, 0x8e, 0xb2, 0x09, 0x57, 0xe2, 0x46, 0xd1, 0x3b, 0xd0, 0x3b, 0x94, 0x99, + 0x4c, 0xe4, 0x8b, 0xd9, 0x35, 0x53, 0x1f, 0xc0, 0x92, 0xb9, 0x22, 0xcd, 0xc7, 0xa7, 0xcf, 0x2a, + 0x91, 0xde, 0xd6, 0x84, 0x1e, 0xf3, 0x64, 0x5c, 0x26, 0x1a, 0x86, 0xfe, 0x01, 0x15, 0x3b, 0xab, + 0x7f, 0x9c, 0x6f, 0x38, 0x7f, 0x9e, 0x6f, 0x38, 0x7f, 0x9d, 0x6f, 0x38, 0xbf, 0xfe, 0xbd, 0xf1, + 0xbf, 0xe7, 0x5d, 0xfc, 0xf9, 0x3e, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa3, 0x25, 0x40, 0x21, + 0x0a, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 52e587f4b..23bb4886c 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,13 +3,17 @@ syntax = "proto3"; package internal; message IndexMeta { + bool Keys = 3; } -message FrameMeta { +message FieldOptions { + string Type = 8; string CacheType = 3; uint32 CacheSize = 4; + int64 Min = 9; + int64 Max = 10; string TimeQuantum = 5; - repeated Field Fields = 7; + bool Keys = 11; } message ImportResponse { @@ -18,7 +22,7 @@ message ImportResponse { message BlockDataRequest { string Index = 1; - string Frame = 2; + string Field = 2; string View = 5; uint64 Slice = 4; uint64 Block = 3; @@ -51,32 +55,20 @@ message CreateIndexMessage { IndexMeta Meta = 2; } -message CreateFrameMessage { - string Index = 1; - string Frame = 2; - FrameMeta Meta = 3; -} - -message DeleteFrameMessage { - string Index = 1; - string Frame = 2; -} - message CreateFieldMessage { string Index = 1; - string Frame = 2; - Field Field = 3; + string Field = 2; + FieldOptions Meta = 3; } message DeleteFieldMessage { string Index = 1; - string Frame = 2; - string Field = 3; + string Field = 2; } -message Frame { +message Field { string Name = 1; - FrameMeta Meta = 2; + FieldOptions Meta = 2; repeated string Views = 3; } @@ -86,7 +78,7 @@ message Schema { message Index { string Name = 1; - repeated Frame Frames = 4; + repeated Field Fields = 4; } message URI { @@ -123,7 +115,7 @@ message ClusterStatus { repeated Node Nodes = 3; } -message Field { +message BSIGroup { string Name = 1; string Type = 2; int64 Min = 3; @@ -132,13 +124,13 @@ message Field { message CreateViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } message DeleteViewMessage { string Index = 1; - string Frame = 2; + string Field = 2; string View = 3; } @@ -154,7 +146,7 @@ message ResizeInstruction { message ResizeSource { Node Node = 1; string Index = 2; - string Frame = 3; + string Field = 3; string View = 4; uint64 Slice = 5; } diff --git a/internal/public.pb.go b/internal/public.pb.go index 0dab2c831..3cb6fa270 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -27,7 +27,7 @@ import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" -import binary "encoding/binary" +import encoding_binary "encoding/binary" import io "io" @@ -412,7 +412,7 @@ func (m *QueryResult) GetChanged() bool { type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` @@ -433,9 +433,9 @@ func (m *ImportRequest) GetIndex() string { return "" } -func (m *ImportRequest) GetFrame() string { +func (m *ImportRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -484,9 +484,8 @@ func (m *ImportRequest) GetTimestamps() []int64 { type ImportValueRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` ColumnKeys []string `protobuf:"bytes,7,rep,name=ColumnKeys" json:"ColumnKeys,omitempty"` Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` @@ -504,9 +503,9 @@ func (m *ImportValueRequest) GetIndex() string { return "" } -func (m *ImportValueRequest) GetFrame() string { +func (m *ImportValueRequest) GetField() string { if m != nil { - return m.Frame + return m.Field } return "" } @@ -518,13 +517,6 @@ func (m *ImportValueRequest) GetSlice() uint64 { return 0 } -func (m *ImportValueRequest) GetField() string { - if m != nil { - return m.Field - } - return "" -} - func (m *ImportValueRequest) GetColumnIDs() []uint64 { if m != nil { return m.ColumnIDs @@ -808,7 +800,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } return i, nil @@ -1064,11 +1056,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Slice != 0 { dAtA[i] = 0x18 @@ -1181,23 +1173,17 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) i += copy(dAtA[i:], m.Index) } - if len(m.Frame) > 0 { + if len(m.Field) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Frame))) - i += copy(dAtA[i:], m.Frame) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) } if m.Slice != 0 { dAtA[i] = 0x18 i++ i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) } - if len(m.Field) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) - i += copy(dAtA[i:], m.Field) - } if len(m.ColumnIDs) > 0 { dAtA14 := make([]byte, len(m.ColumnIDs)*10) var j13 int @@ -1472,7 +1458,7 @@ func (m *ImportRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } @@ -1522,17 +1508,13 @@ func (m *ImportValueRequest) Size() (n int) { if l > 0 { n += 1 + l + sovPublic(uint64(l)) } - l = len(m.Frame) + l = len(m.Field) if l > 0 { n += 1 + l + sovPublic(uint64(l)) } if m.Slice != 0 { n += 1 + sovPublic(uint64(m.Slice)) } - l = len(m.Field) - if l > 0 { - n += 1 + l + sovPublic(uint64(l)) - } if len(m.ColumnIDs) > 0 { l = 0 for _, e := range m.ColumnIDs { @@ -2335,7 +2317,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 m.FloatValue = float64(math.Float64frombits(v)) default: @@ -3067,7 +3049,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -3092,7 +3074,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Frame = string(dAtA[iNdEx:postIndex]) + m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { @@ -3437,54 +3419,6 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { m.Index = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPublic - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Frame = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) - } - m.Slice = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Slice |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) } @@ -3513,6 +3447,25 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } m.Field = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } case 5: if wireType == 0 { var v uint64 @@ -3795,50 +3748,49 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 709 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c, - 0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba, - 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4, - 0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd, - 0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64, - 0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8, - 0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82, - 0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69, - 0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f, - 0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, - 0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e, - 0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e, - 0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62, - 0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf, - 0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8, - 0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77, - 0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e, - 0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35, - 0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88, - 0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59, - 0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7, - 0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c, - 0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd, - 0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85, - 0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3, - 0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18, - 0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3, - 0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4, - 0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96, - 0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60, - 0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec, - 0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd, - 0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e, - 0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c, - 0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41, - 0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62, - 0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9, - 0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b, - 0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82, - 0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8, - 0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd, - 0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7, - 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7, - 0xa4, 0x5a, 0x06, 0x00, 0x00, + // 699 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0x65, 0x62, 0xe7, 0x75, 0xd3, 0x84, 0x6a, 0x04, 0xc5, 0x42, 0x28, 0x58, 0x16, 0x42, 0x5e, + 0xa5, 0x52, 0xd8, 0x83, 0xe8, 0x4b, 0x8a, 0x2a, 0x2a, 0xb8, 0x2d, 0x45, 0x2c, 0xdd, 0x66, 0x54, + 0x2c, 0x39, 0x9e, 0x60, 0x8f, 0x95, 0xe6, 0x3b, 0xd8, 0xf0, 0x09, 0x2c, 0xf8, 0x08, 0x96, 0x5d, + 0xf2, 0x09, 0x50, 0x7e, 0x04, 0xcd, 0x1d, 0x4f, 0xec, 0xa6, 0x52, 0xc5, 0x82, 0xdd, 0x9c, 0x73, + 0x66, 0xee, 0xcc, 0x99, 0x39, 0xd7, 0x86, 0x8d, 0x79, 0x71, 0x96, 0xc4, 0xe7, 0xa3, 0x79, 0x26, + 0x95, 0xe4, 0x9d, 0x38, 0x55, 0x22, 0x4b, 0xa3, 0x24, 0xf8, 0x08, 0x0e, 0xca, 0x05, 0xf7, 0xa0, + 0xbd, 0x2b, 0x93, 0x62, 0x96, 0xe6, 0x1e, 0xf3, 0x9d, 0xd0, 0x45, 0x0b, 0xf9, 0x33, 0x68, 0xbe, + 0x56, 0x2a, 0xcb, 0xbd, 0x86, 0xef, 0x84, 0xbd, 0xf1, 0x60, 0x64, 0x97, 0x8e, 0x34, 0x8d, 0x46, + 0xe4, 0x1c, 0xdc, 0x43, 0xb1, 0xcc, 0x3d, 0xc7, 0x77, 0xc2, 0x2e, 0xd2, 0x38, 0x78, 0x09, 0xee, + 0xdb, 0x28, 0xce, 0xf8, 0x00, 0x1a, 0x93, 0x3d, 0x8f, 0xf9, 0x2c, 0x74, 0xb1, 0x31, 0xd9, 0xe3, + 0x0f, 0xa0, 0xb9, 0x2b, 0x8b, 0x54, 0x79, 0x0d, 0xa2, 0x0c, 0xe0, 0x9b, 0xe0, 0x1c, 0x8a, 0xa5, + 0xe7, 0xf8, 0x2c, 0xec, 0xa2, 0x1e, 0x06, 0x63, 0xe8, 0x9c, 0x46, 0xc9, 0x4a, 0x3d, 0x8d, 0x12, + 0x2a, 0xe2, 0xa0, 0x1e, 0xde, 0xac, 0xe2, 0x94, 0x55, 0x82, 0xf7, 0xe0, 0xec, 0xc4, 0x4a, 0x8b, + 0x28, 0x17, 0xab, 0x5d, 0x0d, 0xe0, 0x8f, 0xa1, 0x63, 0x5c, 0x4d, 0xf6, 0xca, 0xbd, 0x57, 0x98, + 0x3f, 0x81, 0xee, 0x49, 0x3c, 0x13, 0xb9, 0x8a, 0x66, 0x73, 0x3a, 0x84, 0x83, 0x15, 0x11, 0x7c, + 0x80, 0xbe, 0x99, 0xa9, 0xdd, 0x1e, 0x0b, 0x75, 0xcb, 0xd3, 0xbf, 0xdd, 0xd2, 0x6d, 0x8f, 0xdf, + 0x18, 0xb8, 0x5a, 0xb3, 0x12, 0x5b, 0x49, 0xfa, 0x4a, 0x4f, 0x96, 0x73, 0x51, 0x9e, 0x94, 0xc6, + 0xdc, 0x87, 0xde, 0xb1, 0xca, 0xe2, 0xf4, 0xe2, 0x34, 0x4a, 0x0a, 0x51, 0x16, 0xaa, 0x53, 0xda, + 0xe3, 0x24, 0x55, 0x46, 0x76, 0xc9, 0xc6, 0x0a, 0x6b, 0x8f, 0x3b, 0x52, 0x26, 0x46, 0x6c, 0xfa, + 0x2c, 0xec, 0x60, 0x45, 0xf0, 0x21, 0xc0, 0x41, 0x22, 0xa3, 0x72, 0x6d, 0xcb, 0x67, 0x21, 0xc3, + 0x1a, 0x13, 0x6c, 0x43, 0x5b, 0x9f, 0xf4, 0x4d, 0x34, 0xaf, 0xdc, 0xb2, 0x3b, 0xdc, 0x06, 0x57, + 0x0c, 0x36, 0xde, 0x15, 0x22, 0x5b, 0xa2, 0xf8, 0x5c, 0x88, 0x9c, 0x5e, 0x85, 0x70, 0xe9, 0xd2, + 0x00, 0xbe, 0x05, 0xad, 0xe3, 0x24, 0x3e, 0x17, 0xe6, 0xee, 0x5c, 0x2c, 0x91, 0xf6, 0x5a, 0xdd, + 0x79, 0x4e, 0x5e, 0x3b, 0x58, 0xa7, 0xf4, 0x4a, 0x14, 0x33, 0xa9, 0xac, 0x99, 0x12, 0xf1, 0x10, + 0xee, 0xef, 0x5f, 0x9e, 0x27, 0xc5, 0x54, 0xa0, 0x5c, 0x98, 0xd5, 0x2d, 0x9a, 0xb0, 0x4e, 0xf3, + 0xe7, 0x30, 0x28, 0x29, 0x9b, 0xfe, 0x36, 0x4d, 0x5c, 0x63, 0x83, 0x2f, 0x0c, 0xfa, 0xa5, 0x95, + 0x7c, 0x2e, 0xd3, 0x5c, 0xe8, 0xf7, 0xda, 0xcf, 0x32, 0xfb, 0x5e, 0xfb, 0x59, 0xc6, 0xb7, 0xa1, + 0x8d, 0x22, 0x2f, 0x12, 0x65, 0x43, 0xf0, 0xb0, 0xba, 0x16, 0xbb, 0xb6, 0x48, 0x14, 0xda, 0x59, + 0xfc, 0x15, 0x0c, 0x6e, 0x84, 0xca, 0x74, 0x4f, 0x6f, 0xfc, 0xa8, 0x5a, 0x77, 0x43, 0xc7, 0xb5, + 0xe9, 0xc1, 0x0f, 0x06, 0xbd, 0x5a, 0x65, 0xfe, 0x94, 0x7a, 0x99, 0xce, 0xd4, 0x1b, 0xf7, 0xab, + 0x2a, 0x28, 0x17, 0x48, 0x5d, 0xbe, 0x01, 0xec, 0xa8, 0xcc, 0x13, 0x3b, 0xd2, 0xaf, 0xa8, 0xfb, + 0xd3, 0x6e, 0x5b, 0x7b, 0x45, 0x4d, 0xa3, 0x11, 0xe9, 0xcb, 0xf0, 0x29, 0x4a, 0x2f, 0xc4, 0x94, + 0xf2, 0xd4, 0x41, 0x0b, 0xf9, 0xa8, 0xea, 0x4f, 0x7a, 0x80, 0xde, 0x98, 0x57, 0x25, 0xac, 0x82, + 0x55, 0x0f, 0xdb, 0x40, 0xeb, 0xb7, 0xe8, 0x9b, 0x40, 0x07, 0xbf, 0x19, 0xf4, 0x27, 0xb3, 0xb9, + 0xcc, 0x54, 0x2d, 0x24, 0x93, 0x74, 0x2a, 0x2e, 0x6d, 0x48, 0x08, 0x68, 0xf6, 0x20, 0x16, 0xc9, + 0x94, 0x4e, 0xdf, 0x45, 0x03, 0x34, 0x4b, 0x61, 0xa1, 0x70, 0xb8, 0x68, 0x00, 0xc5, 0x42, 0xf7, + 0x7b, 0xee, 0xb9, 0x26, 0x50, 0x06, 0xe9, 0xf8, 0xdb, 0x76, 0xcf, 0xbd, 0x26, 0x49, 0x15, 0xa1, + 0xe3, 0xbf, 0xea, 0x77, 0x9d, 0x17, 0x27, 0x74, 0xb0, 0xc6, 0xe8, 0x7b, 0x40, 0xb9, 0xa0, 0x8f, + 0x5c, 0x9b, 0x3e, 0x72, 0x16, 0xea, 0x95, 0xa6, 0x0c, 0x89, 0x1d, 0x12, 0x6b, 0x4c, 0xf0, 0x9d, + 0x01, 0x37, 0x1e, 0xa9, 0x91, 0xfe, 0x9f, 0xd1, 0xbb, 0x0d, 0x6d, 0x41, 0x8b, 0xf6, 0xb3, 0x66, + 0x4a, 0xb4, 0x76, 0xdc, 0xf6, 0xfa, 0x71, 0x77, 0x36, 0xaf, 0xae, 0x87, 0xec, 0xe7, 0xf5, 0x90, + 0xfd, 0xba, 0x1e, 0xb2, 0xaf, 0x7f, 0x86, 0xf7, 0xce, 0x5a, 0xf4, 0xd3, 0x78, 0xf1, 0x37, 0x00, + 0x00, 0xff, 0xff, 0x71, 0x71, 0xa2, 0x0c, 0x44, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 9207d3a67..cd34b88ff 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -70,7 +70,7 @@ message QueryResult { message ImportRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; repeated uint64 RowIDs = 4; repeated uint64 ColumnIDs = 5; @@ -81,9 +81,8 @@ message ImportRequest { message ImportValueRequest { string Index = 1; - string Frame = 2; + string Field = 2; uint64 Slice = 3; - string Field = 4; repeated uint64 ColumnIDs = 5; repeated string ColumnKeys = 7; repeated int64 Values = 6; diff --git a/internal/test/querygenerator.go b/internal/test/querygenerator.go new file mode 100644 index 000000000..32720cb68 --- /dev/null +++ b/internal/test/querygenerator.go @@ -0,0 +1,190 @@ +package test + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type Args map[string]interface{} + +type Calls []*pql.Call + +func PQL(calls ...*pql.Call) *pql.Query { + return &pql.Query{Calls: calls} +} + +func Row(frame string, row int) *pql.Call { + return &pql.Call{ + Name: "Row", + Args: Args{ + "frame": frame, + "row": row, + }, + } +} + +func mutationArgs(args ...interface{}) Args { + rargs := make(Args) + for _, arg := range args { + switch v := arg.(type) { + case int: + rargs["column"] = v + case string: + if strings.Contains(v, "=") { + parts := strings.Split(v, "=") + rargs["frame"] = parts[0] + i, _ := strconv.ParseInt(parts[1], 10, 64) + rargs["value"] = i + } else { + rargs["timestamp"] = v + } + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs +} + +func Set(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Set", Args: mutationArgs(args...)} +} + +func Clear(args ...interface{}) *pql.Call { + return &pql.Call{Name: "Clear", Args: mutationArgs(args...)} +} + +func magic(args ...interface{}) (Args, Calls) { + var ( + rargs Args + calls Calls + ) + + for _, arg := range args { + switch v := arg.(type) { + case Args: + rargs = v + case []*pql.Call: + calls = append(calls, v...) + default: + fmt.Printf("wat %T!\n", v) + } + } + + return rargs, calls +} +func Count(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Count", Args: kvargs, Children: children} +} + +func Union(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Union", Args: kvargs, Children: children} +} + +func Intersect(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Intersect", Args: kvargs, Children: children} +} + +func Difference(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Difference", Args: kvargs, Children: children} +} +func Xor(args ...*pql.Call) *pql.Call { + kvargs, children := magic(args) + return &pql.Call{Name: "Xor", Args: kvargs, Children: children} +} + +func Between(frame string, min, max int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.BETWEEN, + "Value": []int{min, max}, + }, + } +} +func Lt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LT, + "Value": column, + }, + } +} +func Lte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.LTE, + "Value": column, + }, + } +} +func Gt(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GT, + "Value": column, + }, + } +} + +func Gte(frame string, column int) *pql.Call { + return &pql.Call{ + Name: "Range", + Args: Args{ + "Op": pql.GTE, + "Value": column, + }, + } +} +func CompareCall(a, b *pql.Call) bool { + if a.Name != b.Name { + return false + } + for k, i := range a.Args { + switch v := i.(type) { + case []int: + bside := b.Args[k] + for j := range v { + if v[j] != bside.([]int)[j] { + return false + } + + } + default: + if b.Args[k] != i { + return false + } + } + } + + if len(a.Children) == len(b.Children) { + for i := range a.Children { + if !CompareCall(a.Children[i], b.Children[i]) { + return false + } + } + } else { + return false + } + return true +} + +func Compare(a, b *pql.Query) bool { + for i := range a.Calls { + if !CompareCall(a.Calls[i], b.Calls[i]) { + return false + } + + } + return true +} diff --git a/internal/test/querygenerator_test.go b/internal/test/querygenerator_test.go new file mode 100644 index 000000000..7af764602 --- /dev/null +++ b/internal/test/querygenerator_test.go @@ -0,0 +1,219 @@ +package test + +import ( + "testing" + "github.com/pilosa/pilosa/pql" +) + + +func TestPQL_Generator(t *testing.T) { + t.Run("pql.Query generator", func(t *testing.T) { + for _, u := range []struct { + pql string + calc *pql.Query + exp *pql.Query + }{ + { + pql: "Union(Row(aaa=10),Row(bbb=9))", + calc: PQL(Union(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Row(aaa=10),Row(bbb=9))", + calc: PQL(Intersect(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Difference(Row(aaa=10),Row(bbb=9))", + calc: PQL(Difference(Row("aaa", 10), Row("bbb", 9))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Difference", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + }, + }, + }, + { + pql: "Range(bbb > 20)", + calc: PQL(Gt("bbb", 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.GT, + "Value": 20, + }, + }, + }, + }, + }, + { + pql: "Range(10 < bbb < 20)", + calc: PQL(Between("bbb", 10, 20)), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Range", + Args: map[string]interface{}{ + "Op": pql.BETWEEN, + "Value": []int{10, 20}, + }, + }, + }, + }, + }, + { + pql: "Set(10, aaa=9)", + calc: PQL(Set(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: "Clear(10, aaa=10)", + calc: PQL(Clear(10, "aaa=9")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Clear", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + }, + }, + }, + }, + }, + { + pql: `Set(10, aaa=10, "2017-03-02T03:00")`, + calc: PQL(Set(10, "aaa=9", "2017-03-02T03:00")), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Set", + Args: map[string]interface{}{ + "frame": "aaa", + "value": int64(9), + "column": 10, + "timestamp": "2017-03-02T03:00", + }, + }, + }, + }, + }, + { + pql: `Count(Row(aaa=10))`, + calc: PQL(Count(Row("aaa", 10))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Count", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + }, + }, + }, + }, + }, + { + pql: "Intersect(Union(Row(aaa=10),Row(bbb=9)), Row(aaa=12))", + calc: PQL(Intersect(Union(Row("aaa", 10), Row("bbb", 9)), Row("aaa", 12))), + exp: &pql.Query{ + Calls: []*pql.Call{ + { + Name: "Intersect", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Union", + Args: map[string]interface{}{}, + Children: []*pql.Call{ + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 10}, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "bbb", "row": 9}, + }, + }, + }, + { + Name: "Row", + Args: map[string]interface{}{"frame": "aaa", "row": 12}, + }, + }, + }, + }, + }, + }, + } { + + if !Compare(u.calc, u.exp) { + t.Fatalf("Not Equal. expected: %v, got %v for %s", u.exp, u.calc, u.pql) + } + } + }) + +} diff --git a/mock/mock.go b/mock/mock.go new file mode 100644 index 000000000..46469c2f9 --- /dev/null +++ b/mock/mock.go @@ -0,0 +1,14 @@ +package mock + +type ReadCloser struct { + ReadFunc func(p []byte) (int, error) + CloseFunc func() error +} + +func (rc *ReadCloser) Read(p []byte) (int, error) { + return rc.ReadFunc(p) +} + +func (rc *ReadCloser) Close() error { + return rc.CloseFunc() +} diff --git a/mock/translator.go b/mock/translator.go new file mode 100644 index 000000000..3f815b89f --- /dev/null +++ b/mock/translator.go @@ -0,0 +1,38 @@ +package mock + +import ( + "context" + "io" + + "github.com/pilosa/pilosa" +) + +var _ pilosa.TranslateStore = (*TranslateStore)(nil) + +type TranslateStore struct { + TranslateColumnsToUint64Func func(index string, values []string) ([]uint64, error) + TranslateColumnToStringFunc func(index string, values uint64) (string, error) + TranslateRowsToUint64Func func(index, frame string, values []string) ([]uint64, error) + TranslateRowToStringFunc func(index, frame string, values uint64) (string, error) + ReaderFunc func(ctx context.Context, off int64) (io.ReadCloser, error) +} + +func (s *TranslateStore) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + return s.TranslateColumnsToUint64Func(index, values) +} + +func (s *TranslateStore) TranslateColumnToString(index string, values uint64) (string, error) { + return s.TranslateColumnToStringFunc(index, values) +} + +func (s *TranslateStore) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + return s.TranslateRowsToUint64Func(index, frame, values) +} + +func (s *TranslateStore) TranslateRowToString(index, frame string, value uint64) (string, error) { + return s.TranslateRowToStringFunc(index, frame, value) +} + +func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, error) { + return s.ReaderFunc(ctx, off) +} diff --git a/pilosa.go b/pilosa.go index 7ddea28eb..bff167a7d 100644 --- a/pilosa.go +++ b/pilosa.go @@ -16,9 +16,7 @@ package pilosa import ( "errors" - "net" "regexp" - "strings" "github.com/pilosa/pilosa/internal" ) @@ -31,26 +29,26 @@ var ( ErrIndexExists = errors.New("index already exists") ErrIndexNotFound = errors.New("index not found") - // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") + // ErrFieldRequired is returned when no field is specified. + ErrFieldRequired = errors.New("field required") + ErrFieldExists = errors.New("field already exists") + ErrFieldNotFound = errors.New("field not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrBSIGroupNotFound = errors.New("bsigroup not found") + ErrBSIGroupExists = errors.New("bsigroup already exists") + ErrBSIGroupNameRequired = errors.New("bsigroup name required") + ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") + ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") + ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") + ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") + ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") - ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]") + ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. @@ -63,6 +61,8 @@ var ( ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") + + ErrNotImplemented = errors.New("not implemented") ) // ApiMethodNotAllowedError wraps an error value indicating that a particular @@ -78,27 +78,28 @@ type BadRequestError struct { error } -// Regular expression to validate index and frame names. +// Regular expression to validate index and field names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. type ColumnAttrSet struct { ID uint64 `json:"id"` + Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeColumnAttrSets converts a into its internal representation. -func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { +// EncodeColumnAttrSets converts a into its internal representation. +func EncodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = encodeColumnAttrSet(a[i]) + other[i] = EncodeColumnAttrSet(a[i]) } return other } -// encodeColumnAttrSet converts set into its internal representation. -func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { +// EncodeColumnAttrSet converts set into its internal representation. +func EncodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { return &internal.ColumnAttrSet{ ID: set.ID, Attrs: encodeAttrs(set.Attrs), @@ -108,26 +109,16 @@ func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { // TimeFormat is the go-style time format used to parse string dates. const TimeFormat = "2006-01-02T15:04" -// ValidateName ensures that the name is a valid format. -func ValidateName(name string) error { +// validateName ensures that the name is a valid format. +func validateName(name string) error { if !nameRegexp.Match([]byte(name)) { return ErrName } return nil } -// StringInSlice checks for substring a in the slice. -func StringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -// StringSlicesAreEqual determines if two string slices are equal. -func StringSlicesAreEqual(a, b []string) bool { +// stringSlicesAreEqual determines if two string slices are equal. +func stringSlicesAreEqual(a, b []string) bool { if a == nil && b == nil { return true @@ -150,54 +141,6 @@ func StringSlicesAreEqual(a, b []string) bool { return true } -// SliceDiff returns the difference between two uint64 slices. -func SliceDiff(a, b []uint64) []uint64 { - m := make(map[uint64]uint64) - - for _, y := range b { - m[y]++ - } - - var ret []uint64 - for _, x := range a { - if m[x] > 0 { - m[x]-- - continue - } - ret = append(ret, x) - } - - return ret -} - -// ContainsSubstring checks to see if substring a is contained in any string in the slice. -func ContainsSubstring(a string, list []string) bool { - for _, b := range list { - if strings.Contains(b, a) { - return true - } - } - return false -} - -// HostToIP converts host to an IP4 address based on net.LookupIP(). -func HostToIP(host string) string { - // if host is not an IP addr, check net.LookupIP() - if net.ParseIP(host) == nil { - hosts, err := net.LookupIP(host) - if err != nil { - return host - } - for _, h := range hosts { - // this restricts pilosa to IP4 - if h.To4() != nil { - return h.String() - } - } - } - return host -} - // AddressWithDefaults converts addr into a valid address, // using defaults when necessary. func AddressWithDefaults(addr string) (*URI, error) { diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go new file mode 100644 index 000000000..139d5c2b7 --- /dev/null +++ b/pilosa_internal_test.go @@ -0,0 +1,43 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "testing" +) + +func TestValidateName(t *testing.T) { + names := []string{ + "a", "ab", "ab1", "b-c", "d_e", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, name := range names { + if validateName(name) != nil { + t.Fatalf("Should be valid index name: %s", name) + } + } +} + +func TestValidateNameInvalid(t *testing.T) { + names := []string{ + "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, name := range names { + if validateName(name) == nil { + t.Fatalf("Should be invalid index name: %s", name) + } + } +} diff --git a/pilosa_test.go b/pilosa_test.go index 41b0d7098..1c98686e5 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -22,54 +22,6 @@ import ( _ "github.com/pilosa/pilosa/test" ) -func TestValidateName(t *testing.T) { - names := []string{ - "a", "ab", "ab1", "b-c", "d_e", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - } - for _, name := range names { - if pilosa.ValidateName(name) != nil { - t.Fatalf("Should be valid index name: %s", name) - } - } -} - -func TestValidateNameInvalid(t *testing.T) { - names := []string{ - "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - } - for _, name := range names { - if pilosa.ValidateName(name) == nil { - t.Fatalf("Should be invalid index name: %s", name) - } - } -} - -func TestStringInSlice(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "localhost:10101" - if !pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s in %v", substr, list) - } - substr = "10101" - if pilosa.StringInSlice(substr, list) { - t.Fatalf("Expected substring %s not in %v", substr, list) - } -} - -func TestContainsSubstring(t *testing.T) { - list := []string{"localhost:10101", "localhost:10102", "localhost:10103"} - substr := "10101" - if !pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s contained in %v", substr, list) - } - substr = "4000" - if pilosa.ContainsSubstring(substr, list) { - t.Fatalf("Expected substring %s in not contained in %v", substr, list) - } -} - func TestAddressWithDefaults(t *testing.T) { tests := []struct { addr string diff --git a/pql/ast.go b/pql/ast.go index c3deff9dd..2d3f59e58 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -40,6 +40,23 @@ func (q *Query) WriteCallN() int { return n } +// HasKeys returns true if any call in the query uses keys and requires translation to ids. +func (q *Query) HasKeys() bool { + for _, call := range q.Calls { + if call.Args["col"] != nil { + if _, ok := call.Args["col"].(string); ok { + return true + } + } + if call.Args["row"] != nil { + if _, ok := call.Args["row"].(string); ok { + return true + } + } + } + return false +} + // String returns a string representation of the query. func (q *Query) String() string { a := make([]string, len(q.Calls)) @@ -100,6 +117,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// StringArg is for reading the value at key from call.Args as a string. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. An error is returned if the value is not a string. +func (c *Call) StringArg(key string) (string, bool, error) { + val, ok := c.Args[key] + if !ok { + return "", false, nil + } + switch tval := val.(type) { + case string: + return tval, true, nil + default: + return "", true, fmt.Errorf("could not convert %v of type %T to string in Call.StringArg", tval, tval) + } +} + // Keys returns a list of argument keys in sorted order. func (c *Call) Keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/pql/ast_test.go b/pql/ast_test.go index f5d75e9de..4b130c610 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -33,11 +33,11 @@ func TestCall_String(t *testing.T) { c := &pql.Call{ Name: "Range", Args: map[string]interface{}{ - "frame": "f", + "other": "f", "field0": &pql.Condition{Op: pql.GTE, Value: 10}, }, } - if s := c.String(); s != `Range(field0 >= 10, frame="f")` { + if s := c.String(); s != `Range(field0 >= 10, other="f")` { t.Fatalf("unexpected string: %s", s) } }) diff --git a/pql/parser_test.go b/pql/parser_test.go index 0e2a5c17d..411406815 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -135,7 +135,7 @@ func TestParser_Parse(t *testing.T) { // Parse with both child calls and arguments. t.Run("ChildrenAndArguments", func(t *testing.T) { - q, err := pql.ParseString(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`) + q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], @@ -143,9 +143,9 @@ func TestParser_Parse(t *testing.T) { Name: "TopN", Children: []*pql.Call{{ Name: "Bitmap", - Args: map[string]interface{}{"id": int64(100), "frame": "other"}, + Args: map[string]interface{}{"id": int64(100), "field": "other"}, }}, - Args: map[string]interface{}{"n": int64(3), "frame": "f"}, + Args: map[string]interface{}{"n": int64(3), "field": "f"}, }, ) { t.Fatalf("unexpected call: %#v", q.Calls[0]) @@ -154,14 +154,14 @@ func TestParser_Parse(t *testing.T) { // Parse a list argument. t.Run("ListArgument", func(t *testing.T) { - q, err := pql.ParseString(`TopN(frame="f", ids=[0,10,30])`) + q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) if err != nil { t.Fatal(err) } else if !reflect.DeepEqual(q.Calls[0], &pql.Call{ Name: "TopN", Args: map[string]interface{}{ - "frame": "f", + "field": "f", "ids": []interface{}{int64(0), int64(10), int64(30)}, }, }, diff --git a/row.go b/row.go index ec2c42726..a59bd73e0 100644 --- a/row.go +++ b/row.go @@ -27,6 +27,9 @@ import ( type Row struct { segments []RowSegment + // String keys translated to/from segment columns. + Keys []string + // Attributes associated with the row. Attrs map[string]interface{} } @@ -166,6 +169,11 @@ func (r *Row) ClearBit(i uint64) (changed bool) { return s.ClearBit(i) } +// Segments returns a list of all segments in the row. +func (r *Row) Segments() []RowSegment { + return r.segments +} + // segment returns a segment for a given slice. // Returns nil if segment does not exist. func (r *Row) segment(slice uint64) *RowSegment { @@ -241,8 +249,10 @@ func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` Columns []uint64 `json:"columns"` + Keys []string `json:"keys,omitempty"` } o.Columns = r.Columns() + o.Keys = r.Keys o.Attrs = r.Attrs if o.Attrs == nil { @@ -261,8 +271,8 @@ func (r *Row) Columns() []uint64 { return a } -// encodeRow converts r into its internal representation. -func encodeRow(r *Row) *internal.Row { +// EncodeRow converts r into its internal representation. +func EncodeRow(r *Row) *internal.Row { if r == nil { return nil } @@ -273,8 +283,8 @@ func encodeRow(r *Row) *internal.Row { } } -// decodeRow converts r from its internal representation. -func decodeRow(pr *internal.Row) *Row { +// DecodeRow converts r from its internal representation. +func DecodeRow(pr *internal.Row) *Row { if pr == nil { return nil } diff --git a/server.go b/server.go index 320fd174c..dd26a2c0b 100644 --- a/server.go +++ b/server.go @@ -19,7 +19,6 @@ import ( "fmt" "log" "net" - "net/http" "os" "os/exec" "path/filepath" @@ -53,17 +52,16 @@ type Server struct { closing chan struct{} // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - executor *Executor + Holder *Holder + Cluster *Cluster + TranslateFile *TranslateFile + diagnostics *DiagnosticsCollector + executor *Executor // External - handler *Handler + handler Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver - Gossiper Gossiper - remoteClient *http.Client systemInfo SystemInfo gcNotifier GCNotifier NewAttrStore func(string) AttrStore @@ -77,6 +75,8 @@ type Server struct { diagnosticInterval time.Duration maxWritesPerRequest int + primaryTranslateStore TranslateStore + defaultClient InternalClient dataDir string } @@ -127,7 +127,7 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { } } -func OptServerHandler(h *Handler) ServerOption { +func OptServerHandler(h Handler) ServerOption { return func(s *Server) error { s.handler = h return nil @@ -162,12 +162,18 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { } } -func OptServerRemoteClient(c *http.Client) ServerOption { +func OptServerInternalClient(c InternalClient) ServerOption { return func(s *Server) error { - s.executor = NewExecutor(c) - s.remoteClient = c - s.defaultClient = NewInternalHTTPClientFromURI(nil, c) - s.Cluster.RemoteClient = c + s.executor = NewExecutor(OptExecutorInternalQueryClient(c)) + s.defaultClient = c + s.Cluster.InternalClient = c + return nil + } +} + +func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { + return func(s *Server) error { + s.primaryTranslateStore = store return nil } } @@ -203,15 +209,10 @@ func OptServerURI(uri *URI) ServerOption { // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { - handler, err := NewHandler() - if err != nil { - return nil, errors.Wrap(err, "initializing handler") - } s := &Server{ closing: make(chan struct{}), Cluster: NewCluster(), Holder: NewHolder(), - handler: handler, Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), @@ -249,6 +250,14 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder + // Initialize translation database. + s.TranslateFile = NewTranslateFile() + s.TranslateFile.Path = filepath.Join(path, "keys") + s.TranslateFile.PrimaryTranslateStore = s.primaryTranslateStore + if err := s.TranslateFile.Open(); err != nil { + return nil, err + } + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -267,8 +276,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.executor.Holder = s.Holder s.executor.Node = node s.executor.Cluster = s.Cluster + s.executor.TranslateStore = s.TranslateFile s.executor.MaxWritesPerRequest = s.maxWritesPerRequest - s.handler.API.Executor = s.executor + s.handler.GetAPI().Executor = s.executor + s.handler.GetAPI().TranslateStore = s.TranslateFile return s, nil } @@ -300,27 +311,18 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.handler.API.Holder = s.Holder - s.handler.API.Broadcaster = s.Broadcaster - s.handler.API.BroadcastHandler = s - s.handler.API.StatusHandler = s - s.handler.API.Cluster = s.Cluster + api := s.handler.GetAPI() + api.Holder = s.Holder + api.Broadcaster = s.Broadcaster + api.BroadcastHandler = s + api.StatusHandler = s + api.Cluster = s.Cluster // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster - // Serve HTTP. - go func() { - server := &http.Server{Handler: s.handler} - go func() { - <-s.closing - server.Close() - }() - err := server.Serve(s.ln) - if err != nil && err.Error() != "http: Server closed" { - s.logger.Printf("HTTP handler terminated with error: %s\n", err) - } - }() + // Serve handler. + go s.handler.Serve(s.ln, s.closing) // Start the BroadcastReceiver. if err := s.BroadcastReceiver.Start(s); err != nil { @@ -328,7 +330,7 @@ func (s *Server) Open() error { } // Open Cluster management. - if err := s.Cluster.Open(); err != nil { + if err := s.Cluster.open(); err != nil { return fmt.Errorf("opening Cluster: %v", err) } @@ -336,7 +338,7 @@ func (s *Server) Open() error { if err := s.Holder.Open(); err != nil { return fmt.Errorf("opening Holder: %v", err) } - if err := s.Cluster.SetNodeState(NodeStateReady); err != nil { + if err := s.Cluster.setNodeState(NodeStateReady); err != nil { return fmt.Errorf("setting nodeState: %v", err) } @@ -345,7 +347,7 @@ func (s *Server) Open() error { // the cluster without waiting for data to load on the coordinator. Before // this starts, the joins are queued up in the Cluster.joiningLeavingNodes // buffered channel. - s.Cluster.ListenForJoins() + s.Cluster.listenForJoins() // Start background monitoring. s.wg.Add(3) @@ -366,11 +368,14 @@ func (s *Server) Close() error { s.ln.Close() } if s.Cluster != nil { - s.Cluster.Close() + s.Cluster.close() } if s.Holder != nil { s.Holder.Close() } + if s.TranslateFile != nil { + s.TranslateFile.Close() + } return nil } @@ -420,7 +425,6 @@ func (s *Server) monitorAntiEntropy() { syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. @@ -455,71 +459,60 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if err := s.Holder.DeleteIndex(obj.Index); err != nil { return err } - case *internal.CreateFrameMessage: + case *internal.CreateFieldMessage: idx := s.Holder.Index(obj.Index) if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - opt := decodeFrameOptions(obj.Meta) - _, err := idx.CreateFrame(obj.Frame, *opt) + opt := decodeFieldOptions(obj.Meta) + _, err := idx.CreateField(obj.Field, *opt) if err != nil { return err } - case *internal.DeleteFrameMessage: - idx := s.Holder.Index(obj.Index) - if err := idx.DeleteFrame(obj.Frame); err != nil { - return err - } - case *internal.CreateFieldMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - field := decodeField(obj.Field) - if err := f.CreateField(field); err != nil { - return err - } case *internal.DeleteFieldMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) - if err := f.DeleteField(obj.Field); err != nil { + idx := s.Holder.Index(obj.Index) + if err := idx.DeleteField(obj.Field); err != nil { return err } case *internal.CreateViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } _, _, err := f.createViewIfNotExistsBase(obj.View) if err != nil { return err } case *internal.DeleteViewMessage: - f := s.Holder.Frame(obj.Index, obj.Frame) + f := s.Holder.Field(obj.Index, obj.Field) if f == nil { - return fmt.Errorf("Local Frame not found: %s", obj.Frame) + return fmt.Errorf("Local Field not found: %s", obj.Field) } err := f.DeleteView(obj.View) if err != nil { return err } case *internal.ClusterStatus: - err := s.Cluster.MergeClusterStatus(obj) + err := s.Cluster.mergeClusterStatus(obj) if err != nil { return err } case *internal.ResizeInstruction: - err := s.Cluster.FollowResizeInstruction(obj) + err := s.Cluster.followResizeInstruction(obj) if err != nil { return err } case *internal.ResizeInstructionComplete: - err := s.Cluster.MarkResizeInstructionComplete(obj) + err := s.Cluster.markResizeInstructionComplete(obj) if err != nil { return err } case *internal.SetCoordinatorMessage: - s.Cluster.SetCoordinator(DecodeNode(obj.New)) + s.Cluster.setCoordinator(DecodeNode(obj.New)) case *internal.UpdateCoordinatorMessage: - s.Cluster.UpdateCoordinator(DecodeNode(obj.New)) + s.Cluster.updateCoordinator(DecodeNode(obj.New)) case *internal.NodeStateMessage: - err := s.Cluster.ReceiveNodeState(obj.NodeID, obj.State) + err := s.Cluster.receiveNodeState(obj.NodeID, obj.State) if err != nil { return err } @@ -553,7 +546,7 @@ func (s *Server) SendSync(pb proto.Message) error { // SendAsync represents an implementation of Broadcaster. func (s *Server) SendAsync(pb proto.Message) error { - return s.Gossiper.SendAsync(pb) + return ErrNotImplemented } // SendTo represents an implementation of Broadcaster. @@ -657,7 +650,7 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) - s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) + s.diagnostics.Set("Cluster", strings.Join(s.Cluster.nodeIDs(), ",")) s.diagnostics.Set("NumNodes", len(s.Cluster.Nodes)) s.diagnostics.Set("NumCPU", runtime.NumCPU()) s.diagnostics.Set("NodeID", s.NodeID) @@ -666,7 +659,7 @@ func (s *Server) monitorDiagnostics() { // Flush the diagnostics metrics at startup, then on each tick interval flush := func() { - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() if err == nil { s.diagnostics.Set("OpenFiles", openFiles) } @@ -723,7 +716,7 @@ func (s *Server) monitorRuntime() { // Record the number of go routines. s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0) - openFiles, err := CountOpenFiles() + openFiles, err := countOpenFiles() // Open File handles. if err == nil { s.Holder.Stats.Gauge("OpenFiles", float64(openFiles), 1.0) @@ -739,8 +732,8 @@ func (s *Server) monitorRuntime() { } } -// CountOpenFiles on operating systems that support lsof. -func CountOpenFiles() (int, error) { +// countOpenFiles on operating systems that support lsof. +func countOpenFiles() (int, error) { switch runtime.GOOS { case "darwin", "linux", "unix", "freebsd": // -b option avoid kernel blocks @@ -754,9 +747,9 @@ func CountOpenFiles() (int, error) { return len(lines), nil case "windows": // TODO: count open file handles on windows - return 0, errors.New("CountOpenFiles() on Windows is not supported") + return 0, errors.New("countOpenFiles() on Windows is not supported") default: - return 0, errors.New("CountOpenFiles() on this OS is not supported") + return 0, errors.New("countOpenFiles() on this OS is not supported") } } diff --git a/server/cluster_test.go b/server/cluster_test.go index e6777d90c..3dfc6c4d9 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -42,7 +42,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Expected indexes and Frames + // Expected indexes and Fields expected := map[string][]string{ "i": []string{"f"}, } @@ -51,14 +51,14 @@ func TestMain_SendReceiveMessage(t *testing.T) { client0 := m0.Client() client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Make sure node0 knows about the index and frame created. + // Make sure node0 knows about the index and field created. schema0, err := client0.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -66,15 +66,15 @@ func TestMain_SendReceiveMessage(t *testing.T) { received0 := map[string][]string{} for _, idx := range schema0 { received0[idx.Name] = []string{} - for _, frame := range idx.Frames { - received0[idx.Name] = append(received0[idx.Name], frame.Name) + for _, field := range idx.Fields { + received0[idx.Name] = append(received0[idx.Name], field.Name) } } if !reflect.DeepEqual(received0, expected) { t.Fatalf("unexpected schema on node0: %s", received0) } - // Make sure node1 knows about the index and frame created. + // Make sure node1 knows about the index and field created. schema1, err := client1.Schema(context.Background()) if err != nil { t.Fatal(err) @@ -82,8 +82,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { received1 := map[string][]string{} for _, idx := range schema1 { received1[idx.Name] = []string{} - for _, frame := range idx.Frames { - received1[idx.Name] = append(received1[idx.Name], frame.Name) + for _, field := range idx.Fields { + received1[idx.Name] = append(received1[idx.Name], field.Name) } } if !reflect.DeepEqual(received1, expected) { @@ -92,8 +92,8 @@ func TestMain_SendReceiveMessage(t *testing.T) { // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -206,10 +206,10 @@ func TestClusterResize_AddNode(t *testing.T) { // Create a client for each node. client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -250,17 +250,17 @@ func TestClusterResize_AddNode(t *testing.T) { client0 := m0.Client() //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } // Write data on first node. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=1300000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=1300000) `); err != nil { t.Fatal(err) } @@ -302,17 +302,17 @@ func TestClusterResize_AddNode(t *testing.T) { client0 := m0.Client() //client1 := m1.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } // Write data on first node. Note that no data is placed on slice 1. if _, err := m0.Query("i", "", ` - SetBit(row=1, frame="f", col=1) - SetBit(row=1, frame="f", col=2400000) + SetBit(row=1, field="f", col=1) + SetBit(row=1, field="f", col=2400000) `); err != nil { t.Fatal(err) } @@ -455,10 +455,10 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Run("ErrorRemoveWithoutReplicas", func(t *testing.T) { client0 := m0.Client() - // Create indexes and frames on one node. + // Create indexes and fields on one node. if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { + } else if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } @@ -466,7 +466,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { // TODO: Deterministic node IDs would ensure consistent results setColumns := "" for i := 0; i < 20; i++ { - setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("SetBit(row=1, field=\"f\", col=%d) ", i*pilosa.SliceWidth) } if _, err := m0.Query("i", "", setColumns); err != nil { diff --git a/server/config.go b/server/config.go index 663e9e18c..1b74b177b 100644 --- a/server/config.go +++ b/server/config.go @@ -61,7 +61,7 @@ type Config struct { Handler struct { // CORS Allowed Origins AllowedOrigins []string `toml:"allowed-origins"` - } + } `toml:"handler"` // TLS TLS TLSConfig `toml:"tls"` @@ -78,6 +78,11 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` + // Translation config supports translation store replication. + Translation struct { + PrimaryURL string `toml:"primary-url"` + } + AntiEntropy struct { Interval toml.Duration `toml:"interval"` } `toml:"anti-entropy"` diff --git a/server/server.go b/server/server.go index 345807ec2..b6bf6e864 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,6 @@ import ( "log" "math/rand" "net" - "net/http" "os" "os/signal" "strconv" @@ -39,7 +38,7 @@ import ( "github.com/pilosa/pilosa/gcnotify" "github.com/pilosa/pilosa/gopsutil" "github.com/pilosa/pilosa/gossip" - "github.com/pilosa/pilosa/statik" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/statsd" "github.com/pkg/errors" ) @@ -165,14 +164,17 @@ func (m *Command) SetupServer() error { } m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime) - handler, err := pilosa.NewHandler(pilosa.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins)) + api := pilosa.NewAPI() + api.Logger = m.logger + + handler, err := http.NewHandler( + http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), + http.OptHandlerAPI(api), + http.OptHandlerLogger(m.logger), + ) if err != nil { return errors.Wrap(err, "wrapping handler") } - handler.Logger = m.logger - handler.FileSystem = &statik.FileSystem{} - handler.API = pilosa.NewAPI() - handler.API.Logger = m.logger uri, err := pilosa.AddressWithDefaults(m.Config.Bind) if err != nil { @@ -213,8 +215,13 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "getting listener") } - c := GetHTTPClient(TLSConfig) - handler.API.RemoteClient = c + c := http.GetHTTPClient(TLSConfig) + + // Setup connection to primary store if this is a replica. + var primaryTranslateStore pilosa.TranslateStore + if m.Config.Translation.PrimaryURL != "" { + primaryTranslateStore = http.NewTranslateStore(m.Config.Translation.PrimaryURL) + } m.Server, err = pilosa.NewServer( pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), @@ -233,32 +240,13 @@ func (m *Command) SetupServer() error { pilosa.OptServerStatsClient(statsClient), pilosa.OptServerListener(ln), pilosa.OptServerURI(uri), - pilosa.OptServerRemoteClient(c), + pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), + pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore), ) return errors.Wrap(err, "new server") } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} - // SetupNetworking sets up internode communication based on the configuration. func (m *Command) SetupNetworking() error { @@ -280,7 +268,6 @@ func (m *Command) SetupNetworking() error { m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.MemberSet = pilosa.NewStaticMemberSet(m.Server.Cluster.Nodes) m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver - m.Server.Gossiper = pilosa.NopGossiper return nil } @@ -325,7 +312,6 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.MemberSet = gossipMemberSet m.Server.Broadcaster = m.Server m.Server.BroadcastReceiver = gossipMemberSet - m.Server.Gossiper = gossipMemberSet return nil } @@ -355,7 +341,7 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { case "nop", "none": return pilosa.NopStatsClient, nil default: - return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].", name) } } diff --git a/server/server_test.go b/server/server_test.go index 9092212b2..971795156 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -21,14 +21,14 @@ import ( "io/ioutil" "math/rand" "reflect" - "runtime" "sort" "strings" "testing" "testing/quick" - "github.com/BurntSushi/toml" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -44,27 +44,27 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) + client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } // Execute SetBit() commands. for _, cmd := range cmds { - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && !strings.Contains(err.Error(), "index already exists") { + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) } - if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && !strings.Contains(err.Error(), "frame already exists") { + if err := client.CreateField(context.Background(), "i", cmd.Field, pilosa.FieldOptions{}); err != nil && err != pilosa.ErrFieldExists { t.Fatal(err) } - if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, frame=%q, col=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { + if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(row=%d, field=%q, col=%d)`, cmd.ID, cmd.Field, cmd.ColumnID)); err != nil { t.Fatal(err) } } // Validate data. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -73,7 +73,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -86,8 +86,8 @@ func TestMain_Set_Quick(t *testing.T) { } // Validate data after reopening. - for frame, frameSet := range SetCommands(cmds).Frames() { - for id, columnIDs := range frameSet { + for field, fieldSet := range SetCommands(cmds).Fields() { + for id, columnIDs := range fieldSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ @@ -96,7 +96,7 @@ func TestMain_Set_Quick(t *testing.T) { }, }, }) + "\n" - if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, frame=%q)`, id, frame)); err != nil { + if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(row=%d, field=%q)`, id, field)); err != nil { t.Fatal(err) } else if res != exp { t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp) @@ -119,49 +119,49 @@ func TestMain_SetRowAttrs(t *testing.T) { m := test.MustRunMain() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "z", pilosa.FieldOptions{}); err != nil { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "neg", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } - // Set columns on different rows in different frames. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + // Set columns on different rows in different fields. + if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=2, frame="z", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=2, field="z", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=3, frame="neg", col=100)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=3, field="neg", col=100)`); err != nil { t.Fatal(err) } // Set row attributes. - if _, err := m.Query("i", "", `SetRowAttrs(row=1, frame="x", x=100)`); err != nil { + if _, err := m.Query("i", "", `SetRowAttrs(row=1, field="x", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="x", x=-200)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="x", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=2, field="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("i", "", `SetRowAttrs(row=3, field="neg", x=-0.44)`); err != nil { t.Fatal(err) } // Query row x/1. - if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -172,19 +172,19 @@ func TestMain_SetRowAttrs(t *testing.T) { } // Query rows after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, field="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. - if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { + if res, err := m.Query("i", "", `Bitmap(row=2, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -196,18 +196,18 @@ func TestMain_SetColumnAttrs(t *testing.T) { m := test.MustRunMain() defer m.Close() - // Create frames. + // Create fields. client := m.Client() if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal(err) - } else if err := client.CreateFrame(context.Background(), "i", "x", pilosa.FrameOptions{}); err != nil { + } else if err := client.CreateField(context.Background(), "i", "x", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } // Set columns on row. - if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { + if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { + } else if _, err := m.Query("i", "", `SetBit(row=1, field="x", col=101)`); err != nil { t.Fatal(err) } @@ -217,7 +217,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) @@ -228,7 +228,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { } // Query row after reopening. - if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { + if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, field="x")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) @@ -262,21 +262,6 @@ func tempMkdir(t *testing.T) string { return dir } -// Ensure the file handle count is working -func TestCountOpenFiles(t *testing.T) { - // Windows is not supported yet - if runtime.GOOS == "windows" { - t.Skip("Skipping unsupported CountOpenFiles test on Windows.") - } - count, err := pilosa.CountOpenFiles() - if err != nil { - t.Errorf("CountOpenFiles failed: %s", err) - } - if count == 0 { - t.Error("CountOpenFiles returned invalid value 0.") - } -} - func TestMain_RecalculateHashes(t *testing.T) { const clusterSize = 5 cluster := test.MustRunMainWithCluster(t, clusterSize) @@ -286,15 +271,15 @@ func TestMain_RecalculateHashes(t *testing.T) { if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } - if err := client0.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{CacheType: "ranked"}); err != nil { - t.Fatal("create frame:", err) + if err := client0.CreateField(context.Background(), "i", "f", pilosa.FieldOptions{CacheType: "ranked"}); err != nil { + t.Fatal("create field:", err) } // Set some columns data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { - data = append(data, fmt.Sprintf(`SetBit(row=%d, frame="f", col=%d)`, rowID, columnID)) + data = append(data, fmt.Sprintf(`SetBit(row=%d, field="f", col=%d)`, rowID, columnID)) } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { @@ -311,7 +296,7 @@ func TestMain_RecalculateHashes(t *testing.T) { // Run a TopN query on all nodes. The result should be the same as the target. for _, m := range cluster { - res, err := m.Query("i", "", `TopN(frame="f")`) + res, err := m.Query("i", "", `TopN(field="f")`) if err != nil { t.Fatal(err) } @@ -325,37 +310,37 @@ func TestMain_RecalculateHashes(t *testing.T) { // SetCommand represents a command to set a column. type SetCommand struct { ID uint64 - Frame string + Field string ColumnID uint64 } type SetCommands []SetCommand -// Frames returns the set of column ids for each frame/row. -func (a SetCommands) Frames() map[string]map[uint64][]uint64 { +// Fields returns the set of column ids for each field/row. +func (a SetCommands) Fields() map[string]map[uint64][]uint64 { // Create a set of unique commands. m := make(map[SetCommand]struct{}) for _, cmd := range a { m[cmd] = struct{}{} } - // Build unique ids for each frame & row. - frames := make(map[string]map[uint64][]uint64) + // Build unique ids for each field & row. + fields := make(map[string]map[uint64][]uint64) for cmd := range m { - if frames[cmd.Frame] == nil { - frames[cmd.Frame] = make(map[uint64][]uint64) + if fields[cmd.Field] == nil { + fields[cmd.Field] = make(map[uint64][]uint64) } - frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID) + fields[cmd.Field][cmd.ID] = append(fields[cmd.Field][cmd.ID], cmd.ColumnID) } // Sort each set of column ids. - for _, frame := range frames { - for id := range frame { - sort.Sort(uint64Slice(frame[id])) + for _, field := range fields { + for id := range field { + sort.Sort(uint64Slice(field[id])) } } - return frames + return fields } // GenerateSetCommands generates random SetCommand objects. @@ -364,7 +349,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { for i := range cmds { cmds[i] = SetCommand{ ID: uint64(rand.Intn(1000)), - Frame: "x", + Field: "x", ColumnID: uint64(rand.Intn(10)), } } @@ -374,7 +359,7 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { // ParseConfig parses s into a Config. func ParseConfig(s string) (server.Config, error) { var c server.Config - _, err := toml.Decode(s, &c) + err := toml.Unmarshal([]byte(s), &c) return c, err } diff --git a/test/client.go b/server_internal_test.go similarity index 60% rename from test/client.go rename to server_internal_test.go index 9e391e88b..e64f80a0f 100644 --- a/test/client.go +++ b/server_internal_test.go @@ -12,24 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package test +package pilosa import ( - "net/http" - - "github.com/pilosa/pilosa" + "runtime" + "testing" ) -// Client represents a test wrapper for pilosa.Client. -type Client struct { - *pilosa.InternalHTTPClient -} - -// MustNewClient returns a new instance of Client. Panic on error. -func MustNewClient(host string, h *http.Client) *Client { - c, err := pilosa.NewInternalHTTPClient(host, h) - if err != nil { - panic(err) +// Ensure the file handle count is working +func TestCountOpenFiles(t *testing.T) { + // Windows is not supported yet + if runtime.GOOS == "windows" { + t.Skip("Skipping unsupported countOpenFiles test on Windows.") + } + count, err := countOpenFiles() + if err != nil { + t.Errorf("countOpenFiles failed: %s", err) + } + if count == 0 { + t.Error("countOpenFiles returned invalid value 0.") } - return &Client{InternalHTTPClient: c} } diff --git a/server_test.go b/server_test.go index 9e10374d7..2f1003592 100644 --- a/server_test.go +++ b/server_test.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pilosa_test import ( @@ -19,9 +33,9 @@ func TestMonitorAntiEntropy(t *testing.T) { if err != nil { t.Fatalf("creating index: %v", err) } - err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) + err = client.CreateField(context.Background(), "balh", "fralh", pilosa.FieldOptions{}) if err != nil { - t.Fatalf("creating frame: %v", err) + t.Fatalf("creating field: %v", err) } time.Sleep(time.Millisecond * 2) diff --git a/statik/.gitignore b/statik/.gitignore deleted file mode 100644 index 485c0c57d..000000000 --- a/statik/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/statik.go diff --git a/statik/filesystem.go b/statik/filesystem.go deleted file mode 100644 index e3bf95cb1..000000000 --- a/statik/filesystem.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//go:generate statik -src=../webui -dest=.. -// -// Package statik contains static assets for the Web UI. `go generate` or -// `make generate-statik` will produce statik.go, which is ignored by git. -package statik - -import ( - "net/http" - - "github.com/pilosa/pilosa" - "github.com/rakyll/statik/fs" -) - -// Ensure nopFileSystem implements interface. -var _ pilosa.FileSystem = &FileSystem{} - -// FileSystem represents a static FileSystem. -type FileSystem struct{} - -// New is a statik implementation of FileSystem New method. -func (s *FileSystem) New() (http.FileSystem, error) { - return fs.New() -} diff --git a/statik/statik.go b/statik/statik.go new file mode 100644 index 000000000..54ef98b8f --- /dev/null +++ b/statik/statik.go @@ -0,0 +1,10 @@ +package statik + +import ( + "github.com/rakyll/statik/fs" +) + +func init() { + data := "PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x17\x00\x00\x00assets/chevron-down.png\x89PNG\x0d\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x00\xc8\x00\x00\x00\xc8\x08\x06\x00\x00\x00\xadX\xae\x9e\x00\x00\x0e\x0eIDATx\xda\xed\xddy\x90\x14\xd5\x1d\x07\xf0\xc7\xce\xd5=}\xce\xec\x1c;\xb33\xb3;3{\xc0\x9e\xec\x01\xcb.\xbb\xec1\xbb\xa8A\xa3$h\xc5#\x1e \xb95\xa5Dc*\x95C<\"\xa5\xd1T\x02\xc6Jb\x89\xe6\x1f\xe3\x91hb\"\xa8\x89g\x8c\xa6\"\xc6\x8ax!`\x8c ^ \x88r\xaf\xc9\xef\x07\xa31D\x84\x85\xdd\x9973\xdfOUWQ\xcbL\xf7{\xef\xf7\xeb\xee7\xef\xf5!\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xe4\x8d\x93\x16\xc3\x8cu\xfby\xa1\x7f\xeb\xb48\xd0,p\x10\x8e\xfd\xf2\xc6\xc8\xe6RQ\xd1\\V\xbc\xd3\xa1X\x0b\xed\x9a\xcc\xb7yq(\xf6\x99n_\xba\x89\xfe\xcf\x83\x1c\x80\x03\xf0p\x8eP\xae\xcc\x7f?o\xca<\xd6\xe78\x978\xa7\x8a\xa5\x92\x15j\xb0a\x9eQ9m\xb9Z^\xb7\xdeW;\xf2o^\xd4@\xfdsf|\xc62oE\xdb1\xf4\x19\x1fr\x01\xf6\xe3\xe3\xdc\xa0\x1c\xb9\x96r\xe5\x85\x0f\xf2\xa6\xbc\xee\xa5\xbd\xb9D9\xc5\xb9U\xe8\x95\xacR}\xa9\xaf\x1bU=\xcf\xbc_\xc1\xfd\x17\xb3\xba\xef/\xde\xc0\xe4\x85\xf4\xd90r\x02\xb2\xc2\x9c\x13\x9c\x1b\x07\xca\x1b\xce)\xd5\x97\xbe\x90s\xac\x10+\xa8*v\xb2\xca\xa1\x96/;P\x05\xf7_\x9cZ\xf8\"::T\xa2\xcbU\xda]*\xce\x01\xce\x85C\xcd\x1b\xca\xb1\xa5\x8a/\x95\xa0\xef*\x85RIEx\xb4\xd9\xde\xd0\x94{\x0f\xb5\x92\xef/\xde\x8a\xe6[\x85jt\xd3:\xdc\xc8\x95\x92\xe3\xa6\xd8\xf7p\x0e\x8c9o(\xd7(\xe7F\ne'9\xdf\xa8\x9a\xf9\x8a]32:\xd6\x8a\xf2w\xcc\xea\xde\xf5\xb4\x8e3\x91/%\xe7,\x8e\xfd\xe1\xe6\x0d\xe7\x1c\xad\xe3<\x99+\x18sx\xfd\xd7\xd9\xe9\xc1\xcdv\xcd\xf0\x9e\xb1V\xf2\xbf\x95\x1d\xdeC\xeb\xd8\xe4\xd4\x02W\xd2:#\xc8\x9b\xa2\x17\xa1X_\xc51\x1f\x87\xbc\xd9\xcc9\xc8\xb9([%\x8f\xd3\xc2\xcd\xbf\xb1\x92\xb36\x1dn\x05\xf7_\xacd\xff\x9bz\xb8\xf5\x97\xb4\xee\xa3\x91CE\xebh\x8e1\xc7z\xfc\xf2f\xd6&\xceEZ\xf7\xb12T\xb0\x8c\x96\x0b\x8d\xea\x9e\xa7\xecd\xff\xd6\xf1\xaa\xe4\x07G\x85T\xff\xdbf\xf5\xcc'\xb9\xdb\x96\xdd\x16\x14\x07\x8e\xe5\"\x8e-\xc7x\xdc\xf3\x86r\x91s\x92\xb6qA\xbe\xf2f\x12-\x8dn\xbb\xfab\xbd\xb2}-\x9d\xde\xde\x1b\xefJ~\xe8\xd4\xf9\x1emc\x0do\x8b\xb6\xd9\x94\xdd6\x14&\x8e]s6o^\xccA\xde\xac\xcd\xe6Mc.\xf3\x86\xf7\xc8\x11\xad\xa2e\xb9\x11\x9f\xbea\xa2*\xf8\x7f\xe3\xde\xb4-\xda\xe6\x8d\xb4\xed\x13\x04.Q)D|y\xc8\\\x8a\xe1My\xc8\x9b\xe5\x9c\xb3\xb9:\x9b\xd4z\xc3M+\xa9\xdf\xf8N\xae*\xf9\xa1\xfe\xe5\xbbj\xa8\xe1a\xe12\xe6S9\xca\x91s\x05\xa3\x9ccF\xb1{\x84c\x98\xfb\xbc\xe9\x7f\xc7[\xd1\xb4\x82\xca\x91\xceEeO\xceu\x05\xf7_\x94@\xed:\x8f]\xcdCz\xad\xa2\x08/\\+\xb2\xb3F+\xc7\x8ac\x96\xef\xbc\xa1\xb2\x9c\x94\x8bJ\x9f\x91\xef\x8a\xf2\xa2G\xdb7\xba\xcd\x04\x9f:3\x02\xb3\xef2\xe2\x98\x0cs\x8c8V2\xe4\x0c\x95\xe7\xb4\\T\xfdR*[\x1dr\xfb\x88\xd5s[r\x9b\xca\x12_=\xd6\xf9\x00\xe7\x1c\xe7^\xc1\x1ca<\xbe\xf41z\xb4\xfdf:\xfdJ1\x14l\xa7\x07\xff\xa9'\xba~.\x9cJ\xaf\xc0\xec\xfb\xe1\xf5\x10\xa8\xed\xb8\x0d\xb9-%\xe9Fo\xe7\x1c\xe3\\\x93\xb1Ku(\xda\x8dD\xcfO\xadT\xff\x1aY\x8e6\xdeH\xcb\x1djp\xf2\xa7\x05n\xc6\x1a\x8b\x08\xb7\x19\xb7\x9d4\xbd\x82T\xff\x0b\x9c[\x9cc\x85\xde\xb8 \xad\xb2c\x91Y\xdd\xfb'Y\x1a\xd7\x8cM_\xa5\x84\x1b.\xe2\xee\x02r\xff\xa0&s[q\x9bI\x13?\xca%\xce)\xce\xadbid\x8f\xd3\xe3\x9fm\xa5\x06\x1e\xb3k2[\xe48=\x0fn\xf1X\x89\x1f\xbb\xb50\xff\xa8\xf3b?\xf8?^n\x1bj\xa3\xeb\xb8\xad\xe4\x18t\xc9l\xe1\x1c\xe2\\\x12Ez\xfd]\xbd\x91\xec\xbb\xc5J\x0fm\x90\xe5h\xa4\xf8k\xeeq\x9a\x91\xa3\xa8l~\xec\x13\x1f\xf0s\x9b(\xfe\xf4\xbd\x12\xcdmm\xe0\xdc)\x85\x81\x16\x9f\x1an9\x97\x8e\x06odg\xbf\xf3>\xebj\xc4\xa7\xaf+s\x1b\x17e\x87\x82K\xf9\x8eE\xae\xbb\x8f\xdb\x82\xdbD\x92\xab#\xf8\x92\x917\xd4p\xeb9\xa56T?\x85*\xbe\xd5JgFe\xd8I\xf6^\x0e=\xc9\xc9\x0f\x890Kx\x071\xa9\x0d~%\xc9\x1c\xd6{Vjh\x94s\x84\x7f\x07\x95b0\x1c\x1e#Y\xef\xb1\x93?SC\x0d\xb2\xfc.\xd9A\xdd\x8a\x87\xa9l\xc7\x8b\xd2\xba*\x98\xebz<\xd5\xfd\x11n\x03\x19b\xc19\xe11\xa2?\xe1\x1c\x11%|\x85\xb6C\xa8\xfe\xb8C\xf1\x9d\xea\xd4C\xaf\xc8ry\xb4\x11\xebz\x8e\xca\xb6\x84\x16\xb5\x04b\xc0u\\\xc2u\x96\xe5\xf6\x05\xce\x05\xca\x89S(7b\x02\xb7/\xec\xed\xf7Z\xc2\xe9\x99\xed\xd2\xc3\xbf7\xabfn\xcb\xff\xe9}h\xb7\x91\x98\xb1\x81\x82t\x13\x95\x8dG\xb9\x8a\xf1\xf2y\xaeS3\xd7\x91\xeb\xcau\xce\xfb\xf0-\xc5\x9er\xe0>\xca\x85\x91\xbd9\x81'\xd8\xfc\x0f~\x1eo\x1fu\xb9\xae\xf2\x86\x9b\xa4\x18\xe52b\xd3\xb6\xa8\x81\xfa;\xa9\\\xa7\x17\xd9P0\xd7\xe5t\xae\x1b\xd7Q\x8a \\\x8a9\xc7\x9es@\xe0V\x85\x8f=\xaa\xd5\xba\x8c\xd8\x02\xc5\x9fz^\x86\x1f\xeftT\xe3;\xd2VQ\xb9.+\x92\xa1`\xae\xc3e\\'\xae\x9b\x0c?\xc6)\xd6\xcfr\xcc9\xf6\x027\xbb\x1d\x12>\x82,\xd0\"\xad+(\x88\x9b\xa4\xf8\xd1X\x9e\xde\xa8\xf8R\x97;\xfd\xe9\x1eQ\x98\x93T\x8a\x12\xe9\xea\xe5:p]\xa4\x98\x11\xa7\xd8r\x8c\xa9l\xf3q\xd68\xbc\xb3IF\x0dMY\xaaWv\xbe,\xc5U\xa3\x95\x1d\xdb\xf4X\xe7\xedf\xa2\xfb\xf4l\x1f\xb9PX\xf4;\xe3\x0c39\xeb\x0e\xae\x83\x1cm\xd9\xf92\xc7\x96\xca6\x84\xb3\xc6\x91\xa9t*\xfe\xefP\x80\x9f\xb1\xd2C;$\x18\xe5\x1a\xb5\xd3C/\xe9\xd1\xf6/ 5\x1a\x97|\x94\xc5\xc1e\xe4\xb2r\x99%\x99\x11\xdf\xc1\xb1\xa4\x98~K\xe0\x8e\xcfq\x1d\xe9\x9aO\x0d\xfb\x90\x95\xec\x7fK\x9a\xfb\x10\xa2m\x17k\xc1\x86\xa9\x92\x0e\x07\xab\\6=\xd2\xb6X\x9a\xcbE(v\x1cC*\xdbY\x18\xa1\x9a\x18\xd3\x94\xf2\xba\xe5\xd4\xd0\x1b\xe5\xb9\xb2t\xe6\n\x8f?9[\xb2Q.\xaf\xc7_}\x14\x95m\xa5<;\xc7\xc0\x9b\x14\xbb\x1b8\x86H\xe3\x89\xee6\x08\xf1M_:\xf3\x8e];\xbc[\x8e\xcb\"Fv\n\xb7>O\x88j%\xcfG\xc6I{\xcb\xe0\xd6O\xe42I\xd16\x1c#\x8a\x15\x95\xed\x1b\x02\x93~9\xdcI\x1c\x8e\xe3\x8d\xf8\xf4?\xcbr\xdd\x10-\xbb\xd4P\xe3\xd5\"\x87O\xef\xfb\x08I5\xd8x\x0d\x97E\x96\xeb\xdb(F\x8fS\xac\x8e\xc3\xce\x91\x87aK\xbe_A-\xaf[&K7\xc2\xae\xc9\xbcmT\xf5\xf2\xb5\\#yh\x8f\xd9\xbcm.\x83,\xedA\xb1Y\xea\xd6\xc3\xfcN\x17\x05\xe9\x9a\xaf\xf9\x125V\xe9\xb1\x13_\xb5\xd2\x83\xafK\xb2\x93l\xb7R\x03k\xca\xbc\xe5_\x16\xb9\xb92\xd8\xe4m\xf16y\xdb\x92\\\xf4\xf9:\xc5\xe4\\\xe1\x8d\xf3(\x15\xee\xff\x97@P 5\x9f\xaa'\xbaWH\xf3\xe3\xbd\xaa\xf7E-\xdcr\x0d\x95\xadc\x02\xeb\xdd\xc9\xdb\xe0m\xc9Ro\xa3\xaa\xfbn\x8e\x05\xc7\x04i)\x17C\x89\xb4\xf6j\x91\xf6k$\x1a\xd6|\xd5H\xcc\xf8-\x95m\xde\x04\xd4w\x9e\x11\x9f\xf1;\xde\x864O4\x8c\xb4_\xcd1\xe0X \x1d\xa5\x9d/\xf1\xa4\x8dD\xcf\x12:\xcd\xaf\x91\xa4\xcb\xb5K\x8b\xb4>\xe62bgS\xf9\xe2\xe3P\xc78\xadk!\xaf\x93\x1f\xce,I\x97\x8a\x9f0\xb2\x84\xda>\x85\xf9\x8d\x02\xe9r\xe9\x89\x19\xe7\xd0\xd1u\xa5<\x93\x8a\xed\xcf{\x83\x0d|\xc1c\xf7a&\x11\x7f\xa7\x9b\xd6q9\xad\xeb\x05\x89\xce\x92+\xf5\xaa\xee\xaf\x08\x89\x9e\x83\x0b\x878\xca\xa5T4\x0eZ\xa9\xfe\xdb\xcdD\xcf\xdbv:3*\xc1\xe3j6\xa9\xe5\xb5\xb7 \xa7\x87\xdf\xfd>\x96\x89E/\x7f\x87\xbf\xcb\xeb\xc8\xffC\xf82\xa3Fl\xda[Vj\xe06\xa5\xa2i\x00\xa3T\x85-\xa2\xf8jn\xd2+;\x9e\xa5\xe4\x92b\xf2\xccm'V\xb9\xed\xd8\xdcl\x97k\xd2A\xce\x1aq\xfe,}\xe7II\x9eI\xb5\x93\xdb\xd2\xa5\x85\xaf\xa7\xb2\x85\x91^\xc5\x81\xdf\x11\xb1P\xf1\xa7\x1f5\xaaz\xa4\x18\n\xf5V4\xff\xc3\xa9\x07/\xa1rM\x11\x1f=\x89\xc6\x7f\x9bB\x9f\xb9\x94?+\xc7\x08U\xcf6j\xc3?R\xb9\x16d\xdb\x14\x8a\x08_R]/\x1c\xca\x83vr`\xab\x0c].\xea\xfe\xed\x98\xe4R\xff N\x97Z\xd1z\x85\x11\xefZ]*;\x07\xd5\xf5i5\xd2\xfa=\x81\xdba\xe1\x10\xc5\x14\x7f\xf2l\xbd\xb2\x83\xdf\xe2\xba\xa7Xw\x0c\xae\x9b\x1e\xebxH\xf1\xa7\xf8\xb9T1\x84\x1d\xc6j\x8eY5\xf3\x1e+5\xf0j\xd1u\xa9\xa8N\xfcP\x07\xaa\xe31\x083\x1c\x89\x16o\xb4\xe3\x87\xd4G_[<;\xc7\xac\x17\xb5\xca\x0e\x9e\xf8kFxa<\x98.#r\xb6\x9d\x1a\\O\xdd\x92\x1d\x05\xdc\xa5\xda\xc1up\x19Q\xeeR\x99\x08+\x8c\xb7\x99fr\xd6\xe3\xbe\x9a\xcc\xb6\x82\xdbA\xa8\xcc\\v\xb1\xef~\x14\x80 \x93\xd0\"m\xdf/\xb8\xc9\xbfH\x1b\xbfZ \x8e\xf0\xc1D+\x13\x81z\xc3\xe1\x0d|\xb2Pv\x0e*\xeb\x1c.\xb3\xc0C\xa2!\x87\xbcZ\xb0\xe1h\xfa\xf1~\x9f\xbc\x93\x7f\xfd\xf7j\xa1&\xd9\x1e\x8b\n%D\xf1VL\x9b\xa6\xc7g\\\xa2E;\xb6\xcb\xb2cx\xc3M[\xf5x\xf7bodz\xa7(\xccw\x99@\x11q(\xbe\x86\x84\xc7W\xfb\x05\xb7\x9dXMg\x94]y\xbc\\d\x97\xdb\x8e\xff\x8d\x9fz\xa2\xfa\x9bd\x7f5\x03\x94\x18\xbe\x99\xe8\x145P\xf3\x0b=\xda\x96\xf3\xf7\xfe\xe9\x95m\x9bi\xdb7S\x19>C\x8b\x86p\x80\x8c\xf8E\xa4\xddn+~\xb9\x1a\x9c\x92\xb3\xd9w\xda\xd6\x06\xda\xe6\xa5\xb4\xed.\x81W\x99A\x01\xf0 Q\xb6X\xaf\xec|\xd2\xac\xee\x9b\xb09\x13\xab\xba\xef]\xda\xc6*\xda\xd6wi\x9b6\x9a\x1d\n\xedl\xf2Yo\xb0\xf1\xd7f\xa2{\xdc\xdf\x8aE\xeb\xdc\xe4\x0d5\xf2+\xaaO\xcbn\x0b\xa0\xe0\xf0\xbcC\xa3K\x0b\xde`V\xf5\xbcj\xa73{\xc6\xe1 #{x]\xb4N~.U\x83\xc0\xdc\x06\x14\x01\xfe]p\x99Y5s\xed\x91\xbc\x88\x94\xbe\xbb\x9d\xd7A\xeb\xe2\xdba\xf1Z\x01(*|o\xf7\xb1j\xb0\x9e_r\xb3k\xec\x17\x1afv\xa9\xc1\xc9\x0f\xd2:\xe6\xa0)\xa1X9\x84\xd0\x83en\xed\xc6\xb1\xee \xf4\x9d\xeb\xf9\xbb\x02s\x1bP\x02g\x92r\xa7\xea_d$\xba\x0f\xfa\xe2\x1b\xfa\xcc\x1a\x97\xea?\x8f\xbf#\xf0\x84\x11(!\x01\x8f\xafj\x8e7\xd8p\xd7\x01/\x19 5\xdeI\x9f\xf9Dv\xe7\x00(9\x9aK\xabl\xd5BM\x8b= 0) {\n output_string += `
\n
\n Just getting started? Try this:
\n :create index test
\n :use test
\n :create frame foo
\n SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit\n `\n }\n }\n }\n }\n\n\n var markup =`\n
\n
\n
\n
\n
Input
\n       \n Source: ${res.indexname}\n
\n
\n ${res.input}\n
\n
\n
\n
\n
output
\n       \n ${res.querytime_ms} ms\n
\n
\n ${output_string}\n
\n
Expand
\n \n
\n
\n
\n
\n \n Export raw .csv\n
\n
\n
\n `\n node.innerHTML = markup;\n this.output.insertBefore(node, this.output.firstChild);\n\n // Expand when overflow\n var element = this.output.firstChild.getElementsByClassName(result_class)[0];\n var expand = this.output.firstChild.getElementsByClassName(\"expand\")[0];\n if (element.clientHeight < element.scrollHeight) {\n expand.style.display = 'block';\n } else {\n expand.style.display = 'none';\n }\n expand.onclick = function () {\n element.style.height = element.scrollHeight + \"px\";\n expand.style.display = 'none';\n return false;\n };\n }\n\n populate_index_dropdown() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/schema')\n var select = document.getElementById('index-dropdown')\n\n xhr.onload = function() {\n var schema = JSON.parse(xhr.responseText)\n for(var i=0; i 0) {\n select.value = 1;\n }\n }\n xhr.send(null)\n }\n\n}\n\nfunction populate_version() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/version')\n var node = document.getElementById('server-version')\n\n xhr.onload = function() {\n var version = JSON.parse(xhr.responseText)['version']\n var version_major_minor = /(v\\d+\\.\\d+)/.exec(version)[0]\n var doc_link = document.getElementById('nav-documentation')\n doc_link.onclick = function() {\n window.open('https://www.pilosa.com/docs/' + version_major_minor + '/introduction/')\n }\n node.innerHTML = version\n }\n xhr.send(null)\n}\n\nfunction handle_nav_click(e) {\n // e.id = \"nav-xxx\"\n name = e.id.substring(4)\n set_active_pane_by_name(name)\n window.location.hash = name\n}\n\nfunction set_active_pane_by_name(name) {\n // toggle the nav buttons\n document.getElementsByClassName(\"nav-active\")[0].classList.remove(\"nav-active\")\n document.getElementById(\"nav-\" + name).classList.add(\"nav-active\")\n\n // toggle the main interface content divs\n document.getElementsByClassName(\"interface-active\")[0].classList.remove(\"interface-active\")\n document.getElementById('interface-' + name).classList.add(\"interface-active\")\n\n // hack hack\n switch(name) {\n case \"cluster\":\n update_cluster_status()\n break\n case \"documentation\":\n open_external_docs()\n break\n }\n}\n\n\nfunction update_cluster_status() {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', '/status')\n status_node = document.getElementById('status')\n xhr.onload = function() {\n var status = JSON.parse(xhr.responseText)\n render_status(status)\n }\n xhr.send(null)\n}\n\nfunction render_status(status) {\n // render node table\n var nodes_div = document.getElementById(\"status-nodes\")\n while (nodes_div.firstChild) {\n nodes_div.removeChild(nodes_div.firstChild);\n }\n\n var nodes = status[\"status\"][\"Nodes\"]\n table = document.createElement(\"table\")\n tbody = document.createElement(\"tbody\")\n table.appendChild(tbody)\n var caption = document.createElement(\"caption\")\n caption.innerHTML = \"(\" + nodes.length + \")\"\n table.appendChild(caption)\n\n var header = document.createElement('tr')\n markup = `Host\n State`\n header.innerHTML = markup\n tbody.appendChild(header)\n for(var n=0; n${nodes[n][\"Host\"]}\n ${nodes[n][\"State\"]}`\n row.innerHTML = markup\n tbody.appendChild(row)\n }\n nodes_div.appendChild(table)\n\n // render index tables\n var indexes_div = document.getElementById(\"status-indexes\")\n while (indexes_div.firstChild) {\n indexes_div.removeChild(indexes_div.firstChild);\n }\n\n var indexes = nodes[0][\"Indexes\"] // TODO currently comes from only node 0\n for(var n=0; nName\n Cache Type\n Cache Size`\n header.innerHTML = markup\n tbody.appendChild(header)\n\n var frames = indexes[n][\"Frames\"]\n if(frames) {\n for(var m=0; m${frames[m][\"Name\"]}\n ${frames[m][\"Meta\"][\"CacheType\"]}\n ${frames[m][\"Meta\"][\"CacheSize\"]}`\n tbody.appendChild(row)\n }\n }\n indexes_div.appendChild(table)\n }\n\n // render slice tables\n // TODO enable when Slices element is present in status response\n /*\n var slices_div = document.getElementById(\"status-slices\")\n data = \"\"\n for(var n=0; n\"\n }\n }\n slices_div.innerHTML = data\n */\n\n}\n\nfunction open_external_docs() {\n window.open(\"https://www.pilosa.com/docs\");\n}\n\nfunction check_anchor_uri() {\n var pane_names = {\"console\": 0, \"cluster\": 0, \"documentation\": 0}\n var anchor = window.location.hash.substr(1);\n if(anchor in pane_names) {\n set_active_pane_by_name(anchor)\n }\n}\n\nDate.prototype.today = function () {\n return this.getFullYear() +\"/\"+ (((this.getMonth()+1) < 10)?\"0\":\"\") + (this.getMonth()+1) +\"/\"+ ((this.getDate() < 10)?\"0\":\"\") + this.getDate();\n}\n\nDate.prototype.timeNow = function () {\n return ((this.getHours() < 10)?\"0\":\"\") + this.getHours() +\":\"+ ((this.getMinutes() < 10)?\"0\":\"\") + this.getMinutes() +\":\"+ ((this.getSeconds() < 10)?\"0\":\"\") + this.getSeconds();\n}\n\npopulate_version()\n\n\nclass Autocompleter {\n constructor(input, output) {\n this.input = input\n this.output = output\n this.keyword_map = this.static_keywords\n this.init_dynamic_keywords()\n }\n\n get static_keywords() {\n return {\n // keyword: length of substring that comes after cursor\n \"SetBit()\": 1,\n \"ClearBit()\": 1,\n \"SetRowAttrs()\": 1,\n \"SetColumnAttrs()\": 1,\n \"Bitmap()\": 1,\n \"Union()\": 1,\n \"Intersect()\": 1,\n \"Difference()\": 1,\n \"Count()\": 1,\n \"Range()\": 1,\n \"TopN()\": 1,\n \"frame=\": 0,\n }\n }\n\n complete() {\n var completer = this\n // extract word fragment ending at cursor. a word fragment:\n // - starts with last nonalpha character before cursor (or beginning of string)\n // - ends at cursor\n var word_start = completer.input.selectionEnd-1\n while(word_start>0) {\n var c = completer.input.value.charCodeAt(word_start)\n if(!((c>64 && c<91) || (c>96 && c<123))) {\n word_start++\n break\n }\n word_start--\n }\n var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)\n\n // check for keyword match and insert if exactly one match\n var matches = []\n for(var keyword in this.keyword_map) {\n if(keyword.startsWith(input_word)){\n matches.push(keyword)\n }\n }\n if(matches.length > 1) {\n // completer.output.innerHTML = whatever\n }\n\n if(matches.length == 1) {\n // completer.output.innerHTML = \"\"\n var cursor_pos = completer.input.selectionEnd\n var completion = matches[0].substring(input_word.length)\n var before = completer.input.value.substring(0, cursor_pos)\n var after = completer.input.value.substring(cursor_pos)\n completer.input.value = before + completion + after\n var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]\n completer.input.setSelectionRange(new_pos, new_pos)\n }\n }\n\n init_dynamic_keywords() {\n // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list\n }\n\n add_keyword() {\n // call when index or frame created in webui\n }\n\n remove_keyword() {\n // call when index or frame deleted in webui\n // issue: if e.g. multiple indexes have same frame, removing one removes all.\n // solution: maintain count. requires more elaborate representation of keywords.\n }\n}\n\nvar input = document.getElementById('query')\nvar output = document.getElementById('outputs')\nvar button = document.getElementById('query-btn')\nvar autocomplete_output = document.getElementById('autocomplete-container')\n\nautocompleter = new Autocompleter(input, autocomplete_output)\nrepl = new REPL(input, output, button, autocompleter)\nrepl.populate_index_dropdown()\nrepl.bind_events()\n\ninput.focus()\n\ncheck_anchor_uri()\n\nfunction isJSON(str) {\n try {\n JSON.parse(str)\n } catch (e) {\n return false\n }\n return true\n}\n\nfunction parse_query(query, indexname) {\n var keys = query.replace(/\\s+/g, \" \").split(\" \");\n var command = keys[0];\n var command_type = keys[1];\n var command_name = keys[2];\n var option_str = keys.slice(3, keys.length)\n var options = parse_options(option_str);\n if (command !== \":use\") {\n if (!command_name){\n return {}\n }\n }\n\n var parsed_query = {};\n parsed_query[\"command\"] = command.substr(1, command.length);\n parsed_query[\"command_name\"] = command_name;\n switch (command) {\n case \":create\":\n parsed_query[\"request\"] = \"POST\";\n if(Object.keys(options).length === 0) {\n parsed_query[\"data\"] = \"\";\n } else {\n var opts = {\"options\":{}};\n for (var o in options) {\n opts.options[o] = options[o]\n }\n parsed_query[\"data\"] = JSON.stringify(opts);\n }\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n break\n }\n break;\n case \":delete\":\n parsed_query[\"request\"] = \"DELETE\";\n switch (command_type){\n case \"index\":\n parsed_query[\"url\"] = '/index/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n case \"frame\":\n parsed_query[\"url\"] = '/index/' + indexname + '/frame/' + command_name;\n parsed_query[\"data\"] = \"\";\n break;\n }\n break;\n case \":use\":\n parsed_query[\"command_name\"] = keys[1];\n break;\n default:\n return {}\n }\n return parsed_query;\n}\n\nfunction parse_options(option_str) {\n var int_keys = [\"cacheSize\"];\n var bool_keys = [\"inverseEnabled\"];\n var options = {};\n for (var i = 0; i < option_str.length; i++) {\n var parts = option_str[i].split('=');\n if (int_keys.indexOf(parts[0]) !== -1 ){\n options[parts[0]] = Number(parts[1])\n } else if (bool_keys.indexOf(parts[0]) !== -1){\n options[parts[0]] = (parts[1] == \"true\")\n } else {\n options[parts[0]] = parts[1]\n }\n }\n return options;\n}PK\x07\x08\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-cluster-active.svgnav_cluster_1\nPK\x07\x08\xc1J\xead \x02\x00\x00 \x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-cluster.svgnav_cluster_1PK\x07\x08\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1d\x00\x00\x00assets/nav-console-active.svgnav_consolePK\x07\x08\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x00\x00assets/nav-console.svgnav_console\nPK\x07\x08\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00#\x00\x00\x00assets/nav-documentation-active.svgdocumentation\nPK\x07\x08\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00assets/nav-documentation.svgdocumentationPK\x07\x08\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xc1n\xa3J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00assets/nav_item1.svgnav_item1PK\x07\x08+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00\xa0~\xe6J\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00assets/style.css*{\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n}\nbody{\n font-family: sans-serif;\n background-color: #fbfcfd;\n margin: 0;\n color: #102445;\n}\nh2{\n margin-bottom: 30px;\n}\n\nh5{\n text-transform: uppercase;\n letter-spacing: 2px;\n line-height: 1.21;\n margin: 0;\n}\na{\n line-height: 1.38;\n letter-spacing: 0.2px;\n text-decoration: none;\n color: #102445;\n}\n\n\na:hover{\n color: #1db598;\n}\n\ntextarea{\n width: 100%;\n margin-bottom: 10px;\n border-radius: 2px;\n background-color: #fbfcfd;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n outline: none;\n padding: 30px;\n}\n\n\nselect{\n /*-webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n background: url(\"img/chevron-down.png\") no-repeat calc(100% - 10px) !important;*/\n border-radius: 3px;\n background-color: #fbfcfd;\n width: 187px;\n height: 50px;\n border: solid 1.5px #e4eff4;\n font-size: 18px;\n font-weight: bold;\n line-height: 1.39;\n letter-spacing: 0.2px;\n color: #102445;\n padding: 10.5px;\n\n}\n\nbutton{\n width: 165px;\n height: 50px;\n border-radius: 3px;\n background-color: #1db598;\n outline: none;\n border: none;\n font-size: 16px;\n color: white;\n}\n\nem{\n font-style: normal;\n opacity: 0.5;\n font-size: 14px;\n font-weight: 500;\n letter-spacing: 0.2px;\n color: #102445;\n}\n\n.header{\n height: 92px;\n display: flex;\n align-items: center;\n justify-content: space-between;\n width: 90%;\n margin: auto;\n}\n\n.container{\n display: flex;\n height:100%;\n min-height: 100vh;\n}\n.nav{\n color: white;\n display: flex;\n flex-direction: column;\n width: 150px;\n background: #3c5f8d;\n}\n\n.nav-item{\n height:150px;\n display: flex;\n align-items: center;\n justify-content: center;\n flex-direction: column;\n border-bottom: 3px solid #2a4871;\n cursor: pointer;\n}\n\n.nav-active{\n background: #f2f7f9;\n font-weight: bold;\n color: #1db598;\n}\n\n.nav-item > .nav-image {\n display: flex;\n}\n\n.nav-item > .nav-image-active {\n display: none;\n}\n\n.nav-active > .nav-image {\n display: none;\n}\n\n.nav-active > .nav-image-active {\n display: flex;\n}\n\n\n.interface{\n display: none;\n flex: 1;\n flex-direction: column;\n align-items: center;\n background: #f2f7f9;\n}\n\n.interface-active{\n display: flex;\n}\n\n.query{\n margin-bottom: 30px;\n}\n.query,\n.output-container,\n.status-container{\n width: 75%;\n}\n\n.output{\n margin-bottom: 30px;\n}\n\n.input-controls{\n display: flex;\n justify-content: flex-end;\n}\n\n.tabs{\n display: flex;\n background: #eaf2f6;\n}\n.active-tab{\n background: white;\n font-weight: bold;\n color: #1db598;\n\n}\n\n.tab{\n height:60px;\n width: 100px;\n border-top-right-radius: 5px;\n display: flex;\n align-items: center;\n justify-content: center;\n visibility: visible;\n cursor: pointer;\n\n}\n\n.pane{\n background: white;\n padding: 30px;\n display: none;\n}\n\n.active{\n display: block;\n}\n\n.result-io-header{\n display: flex;\n align-items: center;\n margin-bottom: 15px;\n}\n\n.result-input,\n.result-output,\n.result-error{\n height: 60px;\n border-radius: 2px;\n background-color: #fafafa;\n border: solid 1.5px #e4eff4;\n font-family: monospace;\n font-size: 16px;\n line-height: 1.5;\n letter-spacing: 1.1px;\n color: #102445;\n padding: 15px;\n margin-bottom: 15px;\n word-break: break-all;\n overflow-wrap: break-word;\n overflow:hidden;\n}\n\n\n.result-output{\n background-color: #edf9f7;\n border-left: solid 4px #1db598;\n}\n\n.result-error{\n background-color: #fbf1f0;\n border-left: solid 4px #fa3035;\n color: #fa3035;\n}\n\n.raw{\n height: 253px;\n display: flex;\n align-items: center;\n justify-content: center;\n}\n\n\n.result-table > table {\n border-left: solid 4px #1db598;\n}\n\ntable{\n border: solid 0.5px #e0e0e0;\n width: 100%;\n margin-bottom: 30px;\n /*color:#3c5f8d;*/\n}\ncaption{\n text-align:left;\n font-size: 16px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n text-align: left;\n}\nth{\n font-size: 14px;\n font-weight: bold;\n line-height: 1.21;\n letter-spacing: 2px;\n color: #102445;\n text-transform: uppercase;\n text-align: left;\n padding: 21px 30px;\n background-color: white;\n}\ntr{\n border: solid 0.5px #e0e0e0;\n background-color: white;\n}\ntr:nth-child(even) {\n background-color: #f2f7f9;\n}\ntd{\n padding: 21px 30px;\n}\n\n.expand {\n text-align: center;\n}\n\n.query h2 {\n display: inline-block;\n}\n\n.query-tooltip {\n position: relative;\n display: inline;\n color: #000;\n margin-left: 5px;\n}\n\n.query-tooltip:hover {\n color: #000;\n}\n\n.query-tooltip-content {\n background-color: rgb(250, 250, 250);\n border: solid 1.5px #e4eff4;\n color: #102445;\n border-radius: 2px;\n padding: 15px;\n margin-bottom: 15px;\n\n position: absolute;\n left: 80px;\n top: -30px;\n z-index: 1;\n}\n\n.query-tooltip-container {\n position: relative;\n visibility: hidden;\n}\n\n.query-tooltip:hover+.query-tooltip-container{\n visibility: visible;\n}\n\n.code{\n font-family: monospace;\n}\n\nPK\x07\x08\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00PK\x03\x04\x14\x00\x08\x00\x00\x00cz\xbfJ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00index.html\n\n\n \n \n \n \n Pilosa WebUI\n \n\n\n
\n \"\"\n
\n
\n
\n
\n
\n \"\"\n \"\"\n Console\n
\n
\n \"\"\n \"\"\n Cluster Admin\n
\n
\n \"\"\n \"\"\n Documentation\n
\n
\n
\n\n
\n

Query

\n ?\n
\n
\n
PQL
\n
\n SetBit(frame=foo, rowID=0, columnID=0)
\n ClearBit(frame=foo, rowID=0, columnID=0)
\n SetRowAttrs(frame=foo, rowID=0, color=\"blue\")
\n SetColumnAttrs(frame=foo, columnID=0, shape=\"circle\")
\n Bitmap(frame=foo, rowID=0)
\n Range(frame=foo, rowID=0, start=\"2010-01\", end=\"2017-03\")
\n Count(<BITMAP_CALL>)
\n TopN([BITMAP_CALL], frame=foo, n=20)
\n Union([BITMAP_CALL, ...])
\n Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
\n Difference(<BITMAP_CALL>, <BITMAP_CALL>)\n
\n
\n
Special commands
\n
\n :create index test [columnLabel=column]
\n :use test
\n :create frame foo [rowLabel=row]
\n :delete index test
\n :delete frame foo\n
\n
\n <tab>: autocomplete
\n <up>/<down>: history
\n
\n
\n \n
\n
\n \n    \n \n
\n
\n
\n\n
\n

Output

\n
\n \n
\n
\n\n
\n\n
\n
\n

Nodes

\n
\n
\n
\n
\n

Indexes

\n
\n
\n
\n
\n \n
\n\n
\n\n
\n docs!\n
\n\n
\n \n\n\nPK\x07\x08\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3JJ\x1c\xff\xa8G\x0e\x00\x00G\x0e\x00\x00\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00assets/chevron-down.pngPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\x96\x84jK\xfa\x8b=\x1a\xcaH\x00\x00\xcaH\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x8c\x0e\x00\x00assets/main.jsPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc1J\xead \x02\x00\x00 \x02\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x92W\x00\x00assets/nav-cluster-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xc4\x07\xec\x0b\x05\x02\x00\x00\x05\x02\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xe6Y\x00\x00assets/nav-cluster.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xf2\x90\xe75\xa0\x01\x00\x00\xa0\x01\x00\x00\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81/\\\x00\x00assets/nav-console-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xfb\xc8\xea\xb0\x9e\x01\x00\x00\x9e\x01\x00\x00\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x1a^\x00\x00assets/nav-console.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe5\x95\x86\x82\xec\x01\x00\x00\xec\x01\x00\x00#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xfc_\x00\x00assets/nav-documentation-active.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J\xe18\x81J\xe8\x01\x00\x00\xe8\x01\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x819b\x00\x00assets/nav-documentation.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xc1n\xa3J+\xd4\xf31\xa2\x01\x00\x00\xa2\x01\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81kd\x00\x00assets/nav_item1.svgPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00\xa0~\xe6J\xec[\xd0\xfe=\x13\x00\x00=\x13\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81Of\x00\x00assets/style.cssPK\x01\x02\x14\x03\x14\x00\x08\x00\x00\x00cz\xbfJ\x8dC\xf8\xe1\xef\x0f\x00\x00\xef\x0f\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\xcay\x00\x00index.htmlPK\x05\x06\x00\x00\x00\x00\x0b\x00\x0b\x00\xf2\x02\x00\x00\xf1\x89\x00\x00\x00\x00" + fs.Register(data) +} diff --git a/stats.go b/stats.go index 313708fbc..23c3e0bc8 100644 --- a/stats.go +++ b/stats.go @@ -110,7 +110,7 @@ func (c *ExpvarStatsClient) WithTags(tags ...string) StatsClient { return &ExpvarStatsClient{ m: m, - tags: UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), } } @@ -249,8 +249,8 @@ func (a MultiStatsClient) Close() error { return nil } -// UnionStringSlice returns a sorted set of tags which combine a & b. -func UnionStringSlice(a, b []string) []string { +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { // Sort both sets first. sort.Strings(a) sort.Strings(b) diff --git a/stats_test.go b/stats_test.go index 611b7b2b6..6644786cf 100644 --- a/stats_test.go +++ b/stats_test.go @@ -36,45 +36,45 @@ func TestMultiStatClient_Expvar(t *testing.T) { ms[0] = c hldr.Stats = ms - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).ClearBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, SliceWidth) + hldr.SetBit("d", "f", 0, SliceWidth+2) + hldr.ClearBit("d", "f", 0, 1) - if pilosa.Expvar.String() != `{"index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Gauge creates a unique key, subsequent Gauge calls will overwrite hldr.Stats.Gauge("g", 5, 1.0) hldr.Stats.Gauge("g", 8, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Set creates a unique key, subsequent sets will overwrite hldr.Stats.Set("s", "4", 1.0) hldr.Stats.Set("s", "7", 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7"}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Record timing duration and a uniquely Set key/value dur, _ := time.ParseDuration("123us") hldr.Stats.Timing("tt", dur, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } // Expvar histogram is implemented as a gauge hldr.Stats.Histogram("hh", 3, 1.0) - if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"frame:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { + if pilosa.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": {"field:f": {"view:standard": {"slice:0": {"clearBit": 1, "rows": 0, "setBit": 2}, "slice:1": {"rows": 0, "setBit": 2}}}}, "s": "7", "tt": 123µs}` { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } @@ -88,14 +88,14 @@ func TestStatsCount_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, SliceWidth) + hldr.SetBit("d", "f", 0, SliceWidth+2) // Execute query. called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "TopN" { @@ -109,7 +109,7 @@ func TestStatsCount_TopN(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(field=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -121,10 +121,10 @@ func TestStatsCount_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) e.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, tags []string) { if name != "Bitmap" { @@ -138,7 +138,7 @@ func TestStatsCount_Bitmap(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, row=0)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(field=f, row=0)`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -150,17 +150,17 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - frame := e.Holder.Frame("d", "f") - if frame == nil { - t.Fatal("frame not found") + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) + field := e.Holder.Field("d", "f") + if field == nil { + t.Fatal("field not found") } - frame.Stats = &MockStats{ + field.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "SetRowAttrs" { t.Errorf("Expected SetRowAttrs, Results %s", name) @@ -168,7 +168,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(row=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -180,11 +180,11 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0) - hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1) + hldr.SetBit("d", "f", 10, 0) + hldr.SetBit("d", "f", 10, 1) called := false - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + e := test.NewExecutor(hldr.Holder, pilosa.NewTestCluster(1)) idx := e.Holder.Index("d") if idx == nil { t.Fatal("idex not found") @@ -199,7 +199,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) { called = true }, } - if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(col=10, field=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } if !called { @@ -257,7 +257,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { } } -func TestStatsCount_CreateFrame(t *testing.T) { +func TestStatsCount_CreateField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -272,8 +272,8 @@ func TestStatsCount_CreateFrame(t *testing.T) { called := false s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "createFrame" { - t.Errorf("Expected createFrame, Results %s", name) + if name != "createField" { + t.Errorf("Expected createField, Results %s", name) } if index[0] != "index:i" { t.Errorf("Expected index:i, Results %s", index) @@ -282,13 +282,13 @@ func TestStatsCount_CreateFrame(t *testing.T) { called = true }, } - http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil)) + http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil)) if !called { t.Error("Count isn't called") } } -func TestStatsCount_DeleteFrame(t *testing.T) { +func TestStatsCount_DeleteField(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -298,13 +298,13 @@ func TestStatsCount_DeleteFrame(t *testing.T) { called := false // Create index. indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { + if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil { t.Fatal(err) } s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != "deleteFrame" { - t.Errorf("Expected deleteFrame, Results %s", name) + if name != "deleteField" { + t.Errorf("Expected deleteField, Results %s", name) } if index[0] != "index:i" { t.Errorf("Expected index:i, Results %s", index) @@ -313,7 +313,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { called = true }, } - http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader(""))) if !called { t.Error("Count isn't called") } diff --git a/statsd/statsd.go b/statsd/statsd.go index 0cec63718..81cfbb2b4 100644 --- a/statsd/statsd.go +++ b/statsd/statsd.go @@ -15,6 +15,7 @@ package statsd import ( + "sort" "time" "github.com/DataDog/datadog-go/statsd" @@ -72,7 +73,7 @@ func (c *StatsClient) Tags() []string { func (c *StatsClient) WithTags(tags ...string) pilosa.StatsClient { return &StatsClient{ client: c.client, - tags: pilosa.UnionStringSlice(c.tags, tags), + tags: unionStringSlice(c.tags, tags), logger: c.logger, } } @@ -124,3 +125,38 @@ func (c *StatsClient) Timing(name string, value time.Duration, rate float64) { func (c *StatsClient) SetLogger(logger pilosa.Logger) { c.logger = logger } + +// unionStringSlice returns a sorted set of tags which combine a & b. +func unionStringSlice(a, b []string) []string { + // Sort both sets first. + sort.Strings(a) + sort.Strings(b) + + // Find size of largest slice. + n := len(a) + if len(b) > n { + n = len(b) + } + + // Exit if both sets are empty. + if n == 0 { + return nil + } + + // Iterate over both in order and merge. + other := make([]string, 0, n) + for len(a) > 0 || len(b) > 0 { + if len(a) == 0 { + other, b = append(other, b[0]), b[1:] + } else if len(b) == 0 { + other, a = append(other, a[0]), a[1:] + } else if a[0] < b[0] { + other, a = append(other, a[0]), a[1:] + } else if b[0] < a[0] { + other, b = append(other, b[0]), b[1:] + } else { + other, a, b = append(other, a[0]), a[1:], b[1:] + } + } + return other +} diff --git a/test/attr.go b/test/attr.go deleted file mode 100644 index 16e8ff334..000000000 --- a/test/attr.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/boltdb" -) - -// AttrStore represents a test wrapper for pilosa.AttrStore. -type AttrStore struct { - pilosa.AttrStore -} - -// NewAttrStore returns a new instance of AttrStore. -func NewAttrStore(string) pilosa.AttrStore { - f, err := ioutil.TempFile("", "pilosa-attr-") - if err != nil { - panic(err) - } - f.Close() - os.Remove(f.Name()) - - return &AttrStore{boltdb.NewAttrStore(f.Name())} -} - -func BenchmarkAttrStore_Duplicate(b *testing.B) { - s := MustOpenAttrStore() - defer s.Close() - - // Set attributes. - const n = 5 - for i := 0; i < n; i++ { - if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - - // Update attributes with an existing subset. - cpuN := runtime.GOMAXPROCS(0) - var wg sync.WaitGroup - for i := 0; i < cpuN; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < b.N/cpuN; j++ { - if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { - b.Fatal(err) - } - } - }() - } - wg.Wait() -} - -// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. -func MustOpenAttrStore() pilosa.AttrStore { - s := NewAttrStore("") - if err := s.Open(); err != nil { - panic(err) - } - return s -} - -// Close closes the database and removes the underlying data. -func (s *AttrStore) Close() error { - defer os.RemoveAll(s.Path()) - return s.AttrStore.Close() -} diff --git a/test/cluster.go b/test/cluster.go index 77cf496a8..d1ee6af80 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -15,17 +15,10 @@ package test import ( - "bufio" - "bytes" "fmt" "io/ioutil" - "path/filepath" - "sync" - "time" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/internal" ) // NewCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -37,14 +30,14 @@ func NewCluster(n int) *pilosa.Cluster { c := pilosa.NewCluster() c.ReplicaN = 1 - c.Hasher = NewModHasher() + c.Hasher = newModHasher() c.Path = path c.Topology = pilosa.NewTopology() for i := 0; i < n; i++ { c.Nodes = append(c.Nodes, &pilosa.Node{ ID: fmt.Sprintf("node%d", i), - URI: NewURI("http", fmt.Sprintf("host%d", i), uint16(0)), + URI: newURI("http", fmt.Sprintf("host%d", i), uint16(0)), }) } @@ -55,396 +48,19 @@ func NewCluster(n int) *pilosa.Cluster { return c } -// ModHasher represents a simple, mod-based hashing. -type ModHasher struct{} +// modHasher represents a simple, mod-based hashing. +type modHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. -func NewModHasher() *ModHasher { return &ModHasher{} } +// newModHasher returns a new instance of ModHasher with n buckets. +func newModHasher() *modHasher { return &modHasher{} } -func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } +func (*modHasher) Hash(key uint64, n int) int { return int(key) % n } -// ConstHasher represents hash that always returns the same index. -type ConstHasher struct { - i int -} - -// NewConstHasher returns a new instance of ConstHasher that always returns i. -func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} } - -func (h *ConstHasher) Hash(key uint64, n int) int { return h.i } - -// NewURI is a test URI creator that intentionally swallows errors. -func NewURI(scheme, host string, port uint16) pilosa.URI { +// newURI is a test URI creator that intentionally swallows errors. +func newURI(scheme, host string, port uint16) pilosa.URI { uri := pilosa.DefaultURI() uri.SetScheme(scheme) uri.SetHost(host) uri.SetPort(port) return *uri } - -func NewURIFromHostPort(host string, port uint16) pilosa.URI { - uri := pilosa.DefaultURI() - uri.SetHost(host) - uri.SetPort(port) - return *uri -} - -// TestCluster represents a cluster of test nodes, each of which -// has a pilosa.Cluster. -type TestCluster struct { - Clusters []*pilosa.Cluster - - common *commonClusterSettings - - mu sync.RWMutex - resizing bool - resizeDone chan struct{} -} - -type commonClusterSettings struct { - Nodes []*pilosa.Node -} - -func (t *TestCluster) CreateIndex(name string) error { - for _, c := range t.Clusters { - if _, err := c.Holder.CreateIndexIfNotExists(name, pilosa.IndexOptions{}); err != nil { - return err - } - } - return nil -} - -func (t *TestCluster) CreateFrame(index, frame string, opt pilosa.FrameOptions) error { - for _, c := range t.Clusters { - idx, err := c.Holder.CreateIndexIfNotExists(index, pilosa.IndexOptions{}) - if err != nil { - return err - } - if _, err := idx.CreateFrame(frame, opt); err != nil { - return err - } - } - return nil -} -func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { - // Determine which node should receive the SetBit. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := colID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetBit(view, rowID, colID, x) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { - // Determine which node should receive the SetFieldValue. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := columnID / pilosa.SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetFieldValue(columnID, name, value) - if err != nil { - return err - } - } - - return nil -} - -func (t *TestCluster) clusterByID(id string) *pilosa.Cluster { - for _, c := range t.Clusters { - if c.Node.ID == id { - return c - } - } - return nil -} - -// AddNode adds a node to the cluster and (potentially) starts a resize job. -func (t *TestCluster) AddNode(saveTopology bool) error { - id := len(t.Clusters) - - c, err := t.addCluster(id, saveTopology) - if err != nil { - return err - } - - // Send NodeJoin event to coordinator. - if id > 0 { - coord := t.Clusters[0] - ev := &pilosa.NodeEvent{ - Event: pilosa.NodeJoin, - Node: c.Node, - } - - if err := coord.ReceiveEvent(ev); err != nil { - return err - } - - // Wait for the AddNode job to finish. - if c.State() != pilosa.ClusterStateNormal { - t.resizeDone = make(chan struct{}) - t.mu.Lock() - t.resizing = true - t.mu.Unlock() - <-t.resizeDone - } - } - - return nil -} - -// WriteTopology writes the given topology to disk. -func (t *TestCluster) WriteTopology(path string, top *pilosa.Topology) error { - if buf, err := proto.Marshal(top.Encode()); err != nil { - return err - } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { - return err - } - return nil -} - -func (t *TestCluster) addCluster(i int, saveTopology bool) (*pilosa.Cluster, error) { - - id := fmt.Sprintf("node%d", i) - uri := NewURI("http", fmt.Sprintf("host%d", i), uint16(0)) - - node := &pilosa.Node{ - ID: id, - URI: uri, - } - - // add URI to common - //t.common.NodeIDs = append(t.common.NodeIDs, id) - //sort.Sort(t.common.NodeIDs) - - // add node to common - t.common.Nodes = append(t.common.Nodes, node) - - // create node-specific temp directory - path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) - if err != nil { - return nil, err - } - - // holder - h := pilosa.NewHolder() - h.Path = path - - // cluster - c := pilosa.NewCluster() - c.ReplicaN = 1 - c.Hasher = NewModHasher() - c.Path = path - c.Topology = pilosa.NewTopology() - c.Holder = h - c.MemberSet = pilosa.NewStaticMemberSet(c.Nodes) - c.Node = node - c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator - c.Broadcaster = t - - // add nodes - if saveTopology { - for _, n := range t.common.Nodes { - c.AddNode(n) - } - } - - // Add this node to the TestCluster. - t.Clusters = append(t.Clusters, c) - - return c, nil -} - -// NewTestCluster returns a new instance of test.Cluster. -func NewTestCluster(n int) *TestCluster { - - tc := &TestCluster{ - common: &commonClusterSettings{}, - } - - // add clusters - for i := 0; i < n; i++ { - _, err := tc.addCluster(i, true) - if err != nil { - panic(err) - } - } - return tc -} - -// SetState sets the state of the cluster on each node. -func (t *TestCluster) SetState(state string) { - for _, c := range t.Clusters { - c.SetState(state) - } -} - -// Open opens all clusters in the test cluster. -func (t *TestCluster) Open() error { - for _, c := range t.Clusters { - if err := c.Open(); err != nil { - return err - } - if err := c.Holder.Open(); err != nil { - return err - } - if err := c.SetNodeState(pilosa.NodeStateReady); err != nil { - return err - } - } - - // Start the listener on the coordinator. - if len(t.Clusters) == 0 { - return nil - } - t.Clusters[0].ListenForJoins() - - return nil -} - -// Close closes all clusters in the test cluster. -func (t *TestCluster) Close() error { - for _, c := range t.Clusters { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// TestCluster implements Broadcaster interface. - -// SendSync is a test implemenetation of Broadcaster SendSync method. -func (t *TestCluster) SendSync(pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ClusterStatus: - // Apply the send message to all nodes (except the coordinator). - for _, c := range t.Clusters { - c.MergeClusterStatus(obj) - } - t.mu.RLock() - if obj.State == pilosa.ClusterStateNormal && t.resizing { - close(t.resizeDone) - } - t.mu.RUnlock() - } - - return nil -} - -// SendAsync is a test implemenetation of Broadcaster SendAsync method. -func (t *TestCluster) SendAsync(pb proto.Message) error { - return nil -} - -// SendTo is a test implemenetation of Broadcaster SendTo method. -func (t *TestCluster) SendTo(to *pilosa.Node, pb proto.Message) error { - switch obj := pb.(type) { - case *internal.ResizeInstruction: - err := t.FollowResizeInstruction(obj) - if err != nil { - return err - } - case *internal.ResizeInstructionComplete: - coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) - } - return nil -} - -// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. -func (t *TestCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { - - // Prepare the return message. - complete := &internal.ResizeInstructionComplete{ - JobID: instr.JobID, - Node: instr.Node, - Error: "", - } - - // Stop processing on any error. - if err := func() error { - - // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) - instrNode := pilosa.DecodeNode(instr.Node) - destCluster := t.clusterByID(instrNode.ID) - - // Sync the schema received in the resize instruction. - if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { - return err - } - - for _, src := range instr.Sources { - srcNode := pilosa.DecodeNode(src.Node) - srcCluster := t.clusterByID(srcNode.ID) - - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - if destFragment == nil { - // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) - v := f.View(src.View) - var err error - destFragment, err = v.CreateFragmentIfNotExists(src.Slice) - if err != nil { - return err - } - } - - buf := bytes.NewBuffer(nil) - - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) - - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err - } - } - - return nil - }(); err != nil { - complete.Error = err.Error() - } - - node := pilosa.DecodeNode(instr.Coordinator) - if err := t.SendTo(node, complete); err != nil { - return err - } - - return nil -} diff --git a/test/executor.go b/test/executor.go index 8cd6391c3..c4af65b01 100644 --- a/test/executor.go +++ b/test/executor.go @@ -15,12 +15,13 @@ package test import ( - "net/http" + gohttp "net/http" "strings" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/inmem" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -28,19 +29,21 @@ type Executor struct { *pilosa.Executor } -var remoteClient *http.Client +var remoteClient *gohttp.Client func init() { - remoteClient = server.GetHTTPClient(nil) + remoteClient = http.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. // The executor always matches the uri of the first cluster node. func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor { - executor := pilosa.NewExecutor(remoteClient) + client := http.NewInternalClientFromURI(nil, remoteClient) + executor := pilosa.NewExecutor(pilosa.OptExecutorInternalQueryClient(client)) e := &Executor{Executor: executor} e.Holder = holder e.Cluster = cluster + e.TranslateStore = inmem.NewTranslateStore() e.Node = cluster.Nodes[0] return e } diff --git a/test/fragment.go b/test/fragment.go index cd1111750..6faf5d72d 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -15,9 +15,6 @@ package test import ( - "io/ioutil" - "os" - "github.com/pilosa/pilosa" ) @@ -29,113 +26,3 @@ type Fragment struct { *pilosa.Fragment RowAttrStore pilosa.AttrStore } - -// NewFragment returns a new instance of Fragment with a temporary path. -func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") - if err != nil { - panic(err) - } - file.Close() - - f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice), - RowAttrStore: MustOpenAttrStore(), - } - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - return f -} - -// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error. -func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment { - if cacheType == "" { - cacheType = pilosa.DefaultCacheType - } - f := NewFragment(index, frame, view, slice, cacheType) - - if err := f.Open(); err != nil { - panic(err) - } - return f -} - -// Close closes the fragment and removes all underlying data. -func (f *Fragment) Close() error { - defer os.Remove(f.Path()) - defer os.Remove(f.CachePath()) - defer f.RowAttrStore.Close() - return f.Fragment.Close() -} - -// Reopen closes the fragment and reopens it as a new instance. -func (f *Fragment) Reopen() error { - cacheType := f.Fragment.CacheType - path := f.Path() - if err := f.Fragment.Close(); err != nil { - return err - } - - f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice()) - f.Fragment.CacheType = cacheType - f.Fragment.RowAttrStore = f.RowAttrStore - if err := f.Open(); err != nil { - return err - } - return nil -} - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearColumns clears columns on a row. Panic on error. -func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := f.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// RowAttrStore provides simple storage for attributes. -type RowAttrStore struct { - attrs map[uint64]map[string]interface{} -} - -// NewRowAttrStore returns a new instance of RowAttrStore. -func NewRowAttrStore() *RowAttrStore { - return &RowAttrStore{ - attrs: make(map[uint64]map[string]interface{}), - } -} - -// RowAttrs returns the attributes set to a row id. -func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { - return s.attrs[id], nil -} - -// SetRowAttrs assigns a set of attributes to a row id. -func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { - s.attrs[id] = m -} - -// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk. -func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { - ipct := int(pct * 100) - for i := 0; i < SliceWidth*rowN; i++ { - if i%100 >= ipct { - continue - } - - rowIDs = append(rowIDs, uint64(i%SliceWidth)) - columnIDs = append(columnIDs, uint64(i/SliceWidth)) - } - return -} diff --git a/test/frame.go b/test/frame.go index e107b7d85..75dc5800d 100644 --- a/test/frame.go +++ b/test/frame.go @@ -23,48 +23,48 @@ import ( "github.com/pilosa/pilosa" ) -// Frame represents a test wrapper for pilosa.Frame. -type Frame struct { - *pilosa.Frame +// Field represents a test wrapper for pilosa.Field. +type Field struct { + *pilosa.Field } -// NewFrame returns a new instance of Frame d/0. -func NewFrame() *Frame { - path, err := ioutil.TempDir("", "pilosa-frame-") +// NewField returns a new instance of Field d/0. +func NewField(opt ...pilosa.FieldOption) *Field { + path, err := ioutil.TempDir("", "pilosa-field-") if err != nil { panic(err) } - frame, err := pilosa.NewFrame(path, "i", "f") + field, err := pilosa.NewField(path, "i", "f", opt...) if err != nil { panic(err) } - return &Frame{Frame: frame} + return &Field{Field: field} } -// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. -func MustOpenFrame() *Frame { - f := NewFrame() +// MustOpenField returns a new, opened field at a temporary path. Panic on error. +func MustOpenField(opt ...pilosa.FieldOption) *Field { + f := NewField(opt...) if err := f.Open(); err != nil { panic(err) } return f } -// Close closes the frame and removes the underlying data. -func (f *Frame) Close() error { +// Close closes the field and removes the underlying data. +func (f *Field) Close() error { defer os.RemoveAll(f.Path()) - return f.Frame.Close() + return f.Field.Close() } // Reopen closes the index and reopens it. -func (f *Frame) Reopen() error { +func (f *Field) Reopen() error { var err error - if err := f.Frame.Close(); err != nil { + if err := f.Field.Close(); err != nil { return err } path, index, name := f.Path(), f.Index(), f.Name() - f.Frame, err = pilosa.NewFrame(path, index, name) + f.Field, err = pilosa.NewField(path, index, name) if err != nil { return err } @@ -75,8 +75,8 @@ func (f *Frame) Reopen() error { return nil } -// MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { +// MustSetBit sets a bit on the field. Panic on error. +func (f *Field) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { panic(err) @@ -84,23 +84,23 @@ func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (c return changed } -// Ensure frame can set its cache -func TestFrame_SetCacheSize(t *testing.T) { - f := MustOpenFrame() +// Ensure field can set its cache +func TestField_SetCacheSize(t *testing.T) { + f := MustOpenField() defer f.Close() cacheSize := uint32(100) - // Set & retrieve frame cache size. + // Set & retrieve field cache size. if err := f.SetCacheSize(cacheSize); err != nil { t.Fatal(err) } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size: %d", q) + t.Fatalf("unexpected field cache size: %d", q) } - // Reload frame and verify that it is persisted. + // Reload field and verify that it is persisted. if err := f.Reopen(); err != nil { t.Fatal(err) } else if q := f.CacheSize(); q != cacheSize { - t.Fatalf("unexpected frame cache size (reopen): %d", q) + t.Fatalf("unexpected field cache size (reopen): %d", q) } } diff --git a/test/handler.go b/test/handler.go index 401fa64ab..27fa20503 100644 --- a/test/handler.go +++ b/test/handler.go @@ -19,25 +19,26 @@ import ( "encoding/json" "io" "io/ioutil" - "net/http" + gohttp "net/http" "net/http/httptest" "net/url" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) // Handler represents a test wrapper for pilosa.Handler. type Handler struct { - *pilosa.Handler + *http.Handler Executor HandlerExecutor } // NewHandler returns a new instance of Handler. -func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { - handler, err := pilosa.NewHandler(opts...) +func NewHandler(opts ...http.HandlerOption) (*Handler, error) { + handler, err := http.NewHandler(opts...) if err != nil { return nil, err } @@ -55,7 +56,7 @@ func NewHandler(opts ...pilosa.HandlerOption) (*Handler, error) { } // MustNewHandler returns a new instance of Handler. -func MustNewHandler(opts ...pilosa.HandlerOption) *Handler { +func MustNewHandler(opts ...http.HandlerOption) *Handler { h, err := NewHandler(opts...) if err != nil { panic(err) @@ -145,8 +146,8 @@ func MustParseURLHost(rawurl string) string { } // MustNewHTTPRequest creates a new HTTP request. Panic on error. -func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request { - req, err := http.NewRequest(method, urlStr, body) +func MustNewHTTPRequest(method, urlStr string, body io.Reader) *gohttp.Request { + req, err := gohttp.NewRequest(method, urlStr, body) if err != nil { panic(err) } diff --git a/test/holder.go b/test/holder.go index 97b399bea..4484850fd 100644 --- a/test/holder.go +++ b/test/holder.go @@ -80,19 +80,19 @@ func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOption return &Index{Index: idx} } -// MustCreateFrameIfNotExists returns a given frame. Panic on error. -func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame { - f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) +// MustCreateFieldIfNotExists returns a given field. Panic on error. +func (h *Holder) MustCreateFieldIfNotExists(index, field string) *Field { + f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } return f } -// MustCreateFragmentIfNotExists returns a given fragment. Panic on error. -func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. +func (h *Holder) MustCreateRankedFragmentIfNotExists(index, field, view string, slice uint64) *Fragment { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked}) if err != nil { panic(err) } @@ -107,20 +107,58 @@ func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice return &Fragment{Fragment: frag} } -// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error. -func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment { +// Row returns a Row for a given field. +func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row { idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) if err != nil { panic(err) } - v, err := f.CreateViewIfNotExists(view) + row, err := f.Row(rowID) if err != nil { panic(err) } - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - panic(err) - } - return &Fragment{Fragment: frag} + return row +} + +// ViewRow returns a Row for a given field and view. +func (h *Holder) ViewRow(index, field, view string, rowID uint64) *pilosa.Row { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + row, err := f.ViewRow(view, rowID) + if err != nil { + panic(err) + } + return row +} + +// SetBit clears a bit on the given field. +func (h *Holder) SetBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + f.SetBit(pilosa.ViewStandard, rowID, columnID, nil) +} + +// ClearBit clears a bit on the given field. +func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { + idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}) + f, err := idx.CreateFieldIfNotExists(field, pilosa.FieldOptions{}) + if err != nil { + panic(err) + } + f.ClearBit(pilosa.ViewStandard, rowID, columnID, nil) +} + +// MustSetBits sets columns on a row. Panic on error. +// This function does not accept a timestamp or quantum. +func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + h.SetBit(index, field, rowID, columnID) + } } diff --git a/test/index.go b/test/index.go index 5354c85f9..69c18682c 100644 --- a/test/index.go +++ b/test/index.go @@ -73,20 +73,20 @@ func (i *Index) Reopen() error { return nil } -// CreateFrame creates a frame with the given options. -func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrame(name, opt) +// CreateField creates a field with the given options. +func (i *Index) CreateField(name string, opt pilosa.FieldOptions) (*Field, error) { + f, err := i.Index.CreateField(name, opt) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Field{Field: f}, nil } -// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. -func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) { - f, err := i.Index.CreateFrameIfNotExists(name, opt) +// CreateFieldIfNotExists creates a field with the given options if it doesn't exist. +func (i *Index) CreateFieldIfNotExists(name string, opt pilosa.FieldOptions) (*Field, error) { + f, err := i.Index.CreateFieldIfNotExists(name, opt) if err != nil { return nil, err } - return &Frame{Frame: f}, nil + return &Field{Field: f}, nil } diff --git a/test/pilosa.go b/test/pilosa.go index 4a741ea8b..dca6243fb 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -19,15 +19,15 @@ import ( "fmt" "io" "io/ioutil" - "net/http" + gohttp "net/http" "os" "strings" "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" @@ -238,8 +238,8 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. -func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) +func (m *Main) Client() *http.InternalClient { + client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil)) if err != nil { panic(err) } @@ -249,7 +249,7 @@ func (m *Main) Client() *pilosa.InternalHTTPClient { // Query executes a query against the program through the HTTP API. func (m *Main) Query(index, rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query) - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != gohttp.StatusOK { return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) } return resp.Body, nil @@ -267,11 +267,11 @@ func (m *Main) RecalculateCaches() error { // MustDo executes http.Do() with an http.NewRequest(). Panic on error. func MustDo(method, urlStr string, body string) *httpResponse { - req, err := http.NewRequest(method, urlStr, strings.NewReader(body)) + req, err := gohttp.NewRequest(method, urlStr, strings.NewReader(body)) if err != nil { panic(err) } - resp, err := http.DefaultClient.Do(req) + resp, err := gohttp.DefaultClient.Do(req) if err != nil { panic(err) } @@ -287,6 +287,6 @@ func MustDo(method, urlStr string, body string) *httpResponse { // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { - *http.Response + *gohttp.Response Body string } diff --git a/time.go b/time.go index 517292071..def889304 100644 --- a/time.go +++ b/time.go @@ -79,8 +79,8 @@ func ParseTimeQuantum(v string) (TimeQuantum, error) { return q, nil } -// ViewByTimeUnit returns the view name for time with a given quantum unit. -func ViewByTimeUnit(name string, t time.Time, unit rune) string { +// viewByTimeUnit returns the view name for time with a given quantum unit. +func viewByTimeUnit(name string, t time.Time, unit rune) string { switch unit { case 'Y': return fmt.Sprintf("%s_%s", name, t.Format("2006")) @@ -95,11 +95,11 @@ func ViewByTimeUnit(name string, t time.Time, unit rune) string { } } -// ViewsByTime returns a list of views for a given timestamp. -func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { +// viewsByTime returns a list of views for a given timestamp. +func viewsByTime(name string, t time.Time, q TimeQuantum) []string { a := make([]string, 0, len(q)) for _, unit := range q { - view := ViewByTimeUnit(name, t, unit) + view := viewByTimeUnit(name, t, unit) if view == "" { continue } @@ -108,8 +108,8 @@ func ViewsByTime(name string, t time.Time, q TimeQuantum) []string { return a } -// ViewsByTimeRange returns a list of views to traverse to query a time range. -func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { +// viewsByTimeRange returns a list of views to traverse to query a time range. +func viewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start // Save flags for performance. @@ -127,7 +127,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextDayGTE(t, end) { break } else if t.Hour() != 0 { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) continue } @@ -138,7 +138,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextMonthGTE(t, end) { break } else if t.Day() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) continue } @@ -148,7 +148,7 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string if !nextYearGTE(t, end) { break } else if t.Month() != 1 { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) continue } @@ -164,16 +164,16 @@ func ViewsByTimeRange(name string, start, end time.Time, q TimeQuantum) []string // Walk back down from largest units to smallest units. for t.Before(end) { if hasYear && nextYearGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'Y')) + results = append(results, viewByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) } else if hasMonth && nextMonthGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'M')) + results = append(results, viewByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) } else if hasDay && nextDayGTE(t, end) { - results = append(results, ViewByTimeUnit(name, t, 'D')) + results = append(results, viewByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) } else if hasHour { - results = append(results, ViewByTimeUnit(name, t, 'H')) + results = append(results, viewByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) } else { break diff --git a/time_test.go b/time_internal_test.go similarity index 61% rename from time_test.go rename to time_internal_test.go index 233a2ae1a..2920685fc 100644 --- a/time_test.go +++ b/time_internal_test.go @@ -12,28 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "reflect" "testing" "time" - - "github.com/pilosa/pilosa" ) // Ensure string can be parsed into time quantum. func TestParseTimeQuantum(t *testing.T) { t.Run("OK", func(t *testing.T) { - if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + if q, err := ParseTimeQuantum("YMDH"); err != nil { t.Fatalf("unexpected error: %s", err) - } else if q != pilosa.TimeQuantum("YMDH") { + } else if q != TimeQuantum("YMDH") { t.Fatalf("unexpected quantum: %#v", q) } }) t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { - if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + if _, err := ParseTimeQuantum("BADQUANTUM"); err != ErrInvalidTimeQuantum { t.Fatalf("unexpected error: %s", err) } }) @@ -44,125 +42,125 @@ func TestViewByTimeUnit(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("Y", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'Y'); s != "F_2000" { + if s := viewByTimeUnit("F", ts, 'Y'); s != "F_2000" { t.Fatalf("unexpected name: %s", s) } }) t.Run("M", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'M'); s != "F_200001" { + if s := viewByTimeUnit("F", ts, 'M'); s != "F_200001" { t.Fatalf("unexpected name: %s", s) } }) t.Run("D", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'D'); s != "F_20000102" { + if s := viewByTimeUnit("F", ts, 'D'); s != "F_20000102" { t.Fatalf("unexpected name: %s", s) } }) t.Run("H", func(t *testing.T) { - if s := pilosa.ViewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + if s := viewByTimeUnit("F", ts, 'H'); s != "F_2000010203" { t.Fatalf("unexpected name: %s", s) } }) } -// Ensure all applicable frame names can be generated when mutating a time bit. +// Ensure all applicable field names can be generated when mutating a time bit. func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("YMDH")) + a := viewsByTime("F", ts, mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { t.Fatalf("unexpected names: %+v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTime("F", ts, MustParseTimeQuantum("D")) + a := viewsByTime("F", ts, mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000102"}) { t.Fatalf("unexpected names: %+v", a) } }) } -// Ensure sets of frames can be returned for a given time range. +// Ensure sets of fields can be returned for a given time range. func TestViewsByTimeRange(t *testing.T) { t.Run("Y", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2002-01-01 00:00"), mustParseTimeQuantum("Y")) if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YM", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + a := viewsByTimeRange("F", mustParseTime("2000-11-01 00:00"), mustParseTime("2003-03-01 00:00"), mustParseTimeQuantum("YM")) if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 00:00"), mustParseTime("2003-03-02 00:00"), mustParseTimeQuantum("YMD")) if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("YMDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-28 22:00"), mustParseTime("2002-03-01 03:00"), mustParseTimeQuantum("YMDH")) if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("M", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-03-01 00:00"), mustParseTimeQuantum("M")) if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MD", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 00:00"), mustParseTime("2002-02-03 00:00"), mustParseTimeQuantum("MD")) if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("MDH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + a := viewsByTimeRange("F", mustParseTime("2000-11-29 22:00"), mustParseTime("2002-03-02 03:00"), mustParseTimeQuantum("MDH")) if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("D", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-04 00:00"), mustParseTimeQuantum("D")) if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("DH", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 22:00"), mustParseTime("2000-03-01 02:00"), mustParseTimeQuantum("DH")) if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) t.Run("H", func(t *testing.T) { - a := pilosa.ViewsByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + a := viewsByTimeRange("F", mustParseTime("2000-01-01 00:00"), mustParseTime("2000-01-01 02:00"), mustParseTimeQuantum("H")) if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { - t.Fatalf("unexpected frames: %#v", a) + t.Fatalf("unexpected fields: %#v", a) } }) } -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" +// defaultTimeLayout is the time layout used by the tests. +const defaultTimeLayout = "2006-01-02 15:04" -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) +// mustParseTime parses value using DefaultTimeLayout. Panic on error. +func mustParseTime(value string) time.Time { + v, err := time.Parse(defaultTimeLayout, value) if err != nil { panic(err) } return v } -// MustParseTimeQuantum parses v into a time quantum. Panic on error. -func MustParseTimeQuantum(v string) pilosa.TimeQuantum { - q, err := pilosa.ParseTimeQuantum(v) +// mustParseTimeQuantum parses v into a time quantum. Panic on error. +func mustParseTimeQuantum(v string) TimeQuantum { + q, err := ParseTimeQuantum(v) if err != nil { panic(err) } diff --git a/translate.go b/translate.go new file mode 100644 index 000000000..398055fdb --- /dev/null +++ b/translate.go @@ -0,0 +1,1006 @@ +package pilosa + +import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/cespare/xxhash" +) + +const ( + LogEntryTypeInsertColumn = 1 + LogEntryTypeInsertRow = 2 +) + +const ( + DefaultMapSize = 10 * (1 << 30) // 10GB + + DefaultReplicationRetryInterval = 1 * time.Second +) + +const ( + ReplicationBufferSize = 65536 +) + +var ( + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: operation not supported, translate store read only") +) + +// TranslateStore is the storage for translation string-to-uint64 values. +type TranslateStore interface { + TranslateColumnsToUint64(index string, values []string) ([]uint64, error) + TranslateColumnToString(index string, values uint64) (string, error) + + TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) + TranslateRowToString(index, frame string, values uint64) (string, error) + + // Returns a reader from the given offset of the raw data file. + // The returned reader must be closed by the caller when done. + Reader(ctx context.Context, off int64) (io.ReadCloser, error) +} + +// Ensure type implements interface. +var _ TranslateStore = &TranslateFile{} + +// TranslateFile is an on-disk storage engine for translating string-to-uint64 values. +type TranslateFile struct { + mu sync.RWMutex + data []byte + file *os.File + w *bufio.Writer + n int64 + writeNotify chan struct{} + + once sync.Once + wg sync.WaitGroup + closing chan struct{} + + cols map[string]*index + rows map[frameKey]*index + + Path string + MapSize int + + // If non-nil, data is streamed from a primary and this is a read-only store. + PrimaryTranslateStore TranslateStore + + // Delay after attempting to connect to a primary that the store will retry. + ReplicationRetryInterval time.Duration +} + +// NewTranslateFile returns a new instance of TranslateFile. +func NewTranslateFile() *TranslateFile { + return &TranslateFile{ + writeNotify: make(chan struct{}), + closing: make(chan struct{}), + cols: make(map[string]*index), + rows: make(map[frameKey]*index), + + MapSize: DefaultMapSize, + + ReplicationRetryInterval: DefaultReplicationRetryInterval, + } +} + +func (s *TranslateFile) Open() (err error) { + // Open writer & buffered writer. + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { + return err + } else if s.file, err = os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666); err != nil { + return err + } + s.w = bufio.NewWriter(s.file) + + // Memory map data file. + if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.MapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + return err + } + + // Replay the log. + if err := s.replayEntries(); err != nil { + return err + } + + // Stream from primary, if available. + if s.PrimaryTranslateStore != nil { + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorReplication() }() + } + + return nil +} + +func (s *TranslateFile) Close() (err error) { + s.once.Do(func() { + close(s.closing) + + if s.file != nil { + if e := s.file.Close(); e != nil && err == nil { + err = e + } + } + if s.data != nil { + if e := syscall.Munmap(s.data); e != nil && err == nil { + err = e + } + } + }) + s.wg.Wait() + return err +} + +// Closing returns a channel that is closed when the store is closed. +func (s *TranslateFile) Closing() <-chan struct{} { + return s.closing +} + +// Size returns the number of bytes in use in the data file. +func (s *TranslateFile) Size() int64 { + s.mu.RLock() + n := s.n + s.mu.RUnlock() + return n +} + +// IsReadOnly returns true if this store is being replicated from a primary store. +func (s *TranslateFile) IsReadOnly() bool { + return s.PrimaryTranslateStore != nil +} + +// WriteNotify returns a channel that is closed when a new entry is written. +func (s *TranslateFile) WriteNotify() <-chan struct{} { + s.mu.RLock() + ch := s.writeNotify + s.mu.RUnlock() + return ch +} + +func (s *TranslateFile) appendEntry(entry *LogEntry) error { + offset := s.n + + // Append entry to the end of the WAL. + n, err := entry.WriteTo(s.w) + if err != nil { + return err + } else if err := s.w.Flush(); err != nil { + return err + } + + // Move position forward. + s.n += n + + // Apply the entry to the current state. + if err := s.applyEntry(entry, offset); err != nil { + return err + } else if err := s.file.Sync(); err != nil { + return err + } + + // Notify others of write update. + close(s.writeNotify) + s.writeNotify = make(chan struct{}) + + return nil +} + +func (s *TranslateFile) applyEntry(entry *LogEntry, offset int64) error { + // Move offset to the start of the id/key pairs. + offset += entry.HeaderSize() + + var idx *index + switch entry.Type { + case LogEntryTypeInsertColumn: + idx = s.col(string(entry.Index)) + + case LogEntryTypeInsertRow: + idx = s.row(string(entry.Index), string(entry.Frame)) + + default: + return fmt.Errorf("enterprise.TranslateFile.applyEntry(): unknown log entry type: 0x%20x", entry.Type) + } + + // Insert id/key pairs into index. + for i, id := range entry.IDs { + key := entry.Keys[i] + + // Determine key offset based on ID size. + sz := int64(UvarintSize(id)) + idx.insert(id, offset+sz) + + // Move sequence forward. + if id > idx.seq { + idx.seq = id + } + + // Move offset forward. + offset += sz + int64(UvarintSize(uint64(len(key)))) + int64(len(key)) + } + + return nil +} + +func (s *TranslateFile) replayEntries() error { + // Build a reader from the memory-map data. + fi, err := os.Stat(s.Path) + if err != nil { + return err + } + r := bytes.NewReader(s.data[:fi.Size()]) + + // Iterate over each entry and reapply. + for { + offset := s.n + + var entry LogEntry + if n, err := entry.ReadFrom(r); err == io.EOF { + return nil + } else if err != nil { + return err + } else { + s.n += n + } + + if err := s.applyEntry(&entry, offset); err != nil { + return err + } + } +} + +// monitorReplication is executed in a separate goroutine and continually streams +// from the primary store until this store is closed. +func (s *TranslateFile) monitorReplication() { + // Create context that will cancel on close. + ctx, cancel := context.WithCancel(context.Background()) + go func() { <-s.closing; cancel() }() + + // Keep attempting to replicate until the store closes. + for { + if err := s.replicate(ctx); err != nil { + log.Printf("pilosa: replication error: %s", err) + } + + select { + case <-s.closing: + return + case <-time.After(s.ReplicationRetryInterval): + log.Printf("pilosa: reconnecting to primary replica") + } + } +} + +func (s *TranslateFile) replicate(ctx context.Context) error { + off := s.Size() + + // Connect to remote primary. + log.Printf("pilosa: replicating from offset %d", off) + rc, err := s.PrimaryTranslateStore.Reader(ctx, off) + if err != nil { + return err + } + defer rc.Close() + + // Wrap in bufferred I/O so it implements io.ByteReader. + bufr := bufio.NewReader(rc) + + // Continually read new entries from primary and append to local store. + for { + // Read next available entry. + var entry LogEntry + if _, err := entry.ReadFrom(bufr); err == io.EOF { + return nil + } else if err != nil { + return err + } + + // Write to local store. + if err := s.appendEntry(&entry); err != nil { + return err + } + } +} + +func (s *TranslateFile) col(index string) *index { + idx := s.cols[index] + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + return idx +} + +func (s *TranslateFile) row(index, frame string) *index { + idx := s.rows[frameKey{index, frame}] + if idx == nil { + idx = newIndex(s.data) + s.rows[frameKey{index, frame}] = idx + } + return idx +} + +// TranslateColumnsToUint64 converts values to a uint64 id. +// If value does not have an associated id then one is created. +func (s *TranslateFile) TranslateColumnsToUint64(index string, values []string) ([]uint64, error) { + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.cols[index] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create index map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.cols[index] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertColumn, + Index: []byte(index), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + + ret[i] = v + + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +// TranslateColumnToString converts a uint64 id to its associated string value. +// If the id is not associated with a string value then a blank string is returned. +func (s *TranslateFile) TranslateColumnToString(index string, value uint64) (string, error) { + s.mu.RLock() + if idx := s.cols[index]; idx != nil { + if ret, ok := idx.keyByID(value); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +func (s *TranslateFile) TranslateRowsToUint64(index, frame string, values []string) ([]uint64, error) { + key := frameKey{index, frame} + + ret := make([]uint64, len(values)) + + // Read value under read lock. + s.mu.RLock() + if idx := s.rows[key]; idx != nil { + var writeRequired bool + for i := range values { + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + } + ret[i] = v + } + if !writeRequired { + s.mu.RUnlock() + return ret, nil + } + } + s.mu.RUnlock() + + // Return error if not all values could be translated and this store is read-only. + if s.IsReadOnly() { + return ret, ErrTranslateStoreReadOnly + } + + // If any values not found then recheck and then add under a write lock. + s.mu.Lock() + defer s.mu.Unlock() + + // Recheck if value was created between the read lock and write lock. + idx := s.rows[key] + if idx != nil { + var writeRequired bool + for i := range values { + if ret[i] != 0 { + continue + } + v, ok := idx.idByKey([]byte(values[i])) + if !ok { + writeRequired = true + continue + } + ret[i] = v + } + if !writeRequired { + return ret, nil + } + } + + // Create map if it doesn't exists. + if idx == nil { + idx = newIndex(s.data) + s.rows[key] = idx + } + + // Append new identifiers to log. + entry := &LogEntry{ + Type: LogEntryTypeInsertRow, + Index: []byte(index), + Frame: []byte(frame), + IDs: make([]uint64, 0, len(values)), + Keys: make([][]byte, 0, len(values)), + } + check := make(map[string]uint64) + for i := range values { + if ret[i] != 0 { + continue + } + + v, found := check[values[i]] + if !found { + idx.seq++ + v = idx.seq + check[values[i]] = v + } + ret[i] = v + entry.IDs = append(entry.IDs, v) + entry.Keys = append(entry.Keys, []byte(values[i])) + } + + // Write entry. + if err := s.appendEntry(entry); err != nil { + return nil, err + } + + return ret, nil +} + +func (s *TranslateFile) TranslateRowToString(index, frame string, id uint64) (string, error) { + s.mu.RLock() + if idx := s.rows[frameKey{index, frame}]; idx != nil { + if ret, ok := idx.keyByID(id); ok { + s.mu.RUnlock() + return string(ret), nil + } + } + s.mu.RUnlock() + return "", nil +} + +// Reader returns a reader that streams the underlying data file. +func (s *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser, error) { + rc := NewTranslateFileReader(ctx, s, offset) + if err := rc.Open(); err != nil { + return nil, err + } + return rc, nil +} + +type LogEntry struct { + Type uint8 + Index []byte + Frame []byte + + IDs []uint64 + Keys [][]byte + + // Length of the entry, in bytes. + // This is only populated after ReadFrom() or WriteTo(). + Length uint64 +} + +// HeaderSize returns the number of bytes required for size, type, index, frame, & pair count. +func (e *LogEntry) HeaderSize() int64 { + sz := UvarintSize(e.Length) + // total entry length + 1 + // type + UvarintSize(uint64(len(e.Index))) + len(e.Index) + // Index length and data + UvarintSize(uint64(len(e.Frame))) + len(e.Frame) + // Frame length and data + UvarintSize(uint64(len(e.IDs))) // ID/Key pair count + return int64(sz) +} + +// ReadFrom deserializes a LogEntry from r. r must be a ByteReader. +func (e *LogEntry) ReadFrom(r io.Reader) (_ int64, err error) { + br := r.(io.ByteReader) + + // Read the entry length. + if e.Length, err = binary.ReadUvarint(br); err != nil { + return int64(UvarintSize(e.Length)), err + } + + // Slurp entire entry and replace reader. + buf := make([]byte, e.Length) + n, err := io.ReadFull(r, buf) + n64 := int64(n + UvarintSize(e.Length)) + if err != nil { + return n64, err + } + bufr := bytes.NewReader(buf) + br, r = bufr, bufr + + // Read the entry type. + if err := binary.Read(r, binary.BigEndian, &e.Type); err != nil { + return n64, err + } + + // Read index name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Index = nil + } else { + e.Index = make([]byte, sz) + if _, err := io.ReadFull(r, e.Index); err != nil { + return n64, err + } + } + + // Read frame name. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz == 0 { + e.Frame = nil + } else { + e.Frame = make([]byte, sz) + if _, err := io.ReadFull(r, e.Frame); err != nil { + return n64, err + } + } + + // Read key count. + if n, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if n == 0 { + e.IDs, e.Keys = nil, nil + } else { + e.IDs, e.Keys = make([]uint64, n), make([][]byte, n) + } + + // Read each id/key pairs. + for i := range e.Keys { + // Read identifier. + if e.IDs[i], err = binary.ReadUvarint(br); err != nil { + return n64, err + } + + // Read key. + if sz, err := binary.ReadUvarint(br); err != nil { + return n64, err + } else if sz > 0 { + e.Keys[i] = make([]byte, sz) + if _, err := io.ReadFull(r, e.Keys[i]); err != nil { + return n64, err + } + } + } + return n64, nil +} + +// WriteTo serializes a LogEntry to w. +func (e *LogEntry) WriteTo(w io.Writer) (_ int64, err error) { + var buf bytes.Buffer + b := make([]byte, binary.MaxVarintLen64) + + // Write the entry type. + if err := binary.Write(&buf, binary.BigEndian, e.Type); err != nil { + return 0, err + } + + // Write the index name. + sz := binary.PutUvarint(b, uint64(len(e.Index))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Index); err != nil { + return 0, err + } + + // Write frame name. + sz = binary.PutUvarint(b, uint64(len(e.Frame))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Frame); err != nil { + return 0, err + } + + // Write key count. + sz = binary.PutUvarint(b, uint64(len(e.IDs))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write each id/key pairs. + for i := range e.Keys { + // Write identifier. + sz = binary.PutUvarint(b, e.IDs[i]) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } + + // Write key. + sz = binary.PutUvarint(b, uint64(len(e.Keys[i]))) + if _, err := buf.Write(b[:sz]); err != nil { + return 0, err + } else if _, err := buf.Write(e.Keys[i]); err != nil { + return 0, err + } + } + + // Write buffer size. + e.Length = uint64(buf.Len()) + sz = binary.PutUvarint(b, e.Length) + if n, err := w.Write(b[:sz]); err != nil { + return int64(n), err + } + + // Write buffer. + n, err := buf.WriteTo(w) + return int64(sz) + n, err +} + +// ValidLogEntriesLen returns the maximum length of p that contains valid entries. +func ValidLogEntriesLen(p []byte) (n int) { + r := bytes.NewReader(p) + for { + if sz, err := binary.ReadUvarint(r); err != nil { + return n + } else if off, err := r.Seek(int64(sz), io.SeekCurrent); err != nil { + return n + } else if off > int64(len(p)) { + return n + } else { + n = int(off) + } + } +} + +type frameKey struct { + index string + frame string +} + +const defaultLoadFactor = 90 + +// index represents a two-way index between IDs and keys. +type index struct { + seq uint64 // autoincrement sequence + data []byte // memory-mapped file containing key data + + // RHH hashmap for id-to-offset mapping. + // This is required so we don't need to store key data on the heap. + // https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf + elems []elem // id/offset key pairs + n uint64 // number of inuse elements + mask uint64 // mask applied for modulus + threshold uint64 // threshold when capacity doubles + loadFactor int // factor used to calculate threshold + + // Builtin hashmap for offset-to-id mapping. + offsetsByID map[uint64]int64 +} + +func newIndex(data []byte) *index { + idx := &index{ + data: data, + offsetsByID: make(map[uint64]int64), + + loadFactor: defaultLoadFactor, + } + idx.alloc(pow2(uint64(256))) + return idx +} + +// keyByID returns the key for a given ID, if it exists. +func (idx *index) keyByID(id uint64) ([]byte, bool) { + offset, ok := idx.offsetsByID[id] + if !ok { + return nil, false + } + return idx.lookupKey(offset), true +} + +// idByKey returns the ID for a given key, if it exists. +func (idx *index) idByKey(key []byte) (uint64, bool) { + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + if e := &idx.elems[pos]; e.hash == 0 { + return 0, false + } else if dist > idx.dist(e.hash, pos) { + return 0, false + } else if e.hash == hash && bytes.Equal(idx.lookupKey(e.offset), key) { + return e.id, true + } + + pos = (pos + 1) & idx.mask + dist++ + } +} + +// insert adds the id/offset pair to the index. +// This function will resize the map if it crosses the threshold. +func (idx *index) insert(id uint64, offset int64) { + idx.n++ + + // Add to reverse lookup. + idx.offsetsByID[id] = offset + + // Grow the map if we've run out of slots. + if idx.n > idx.threshold { + elems, capacity := idx.elems, uint64(len(idx.elems)) + idx.alloc(uint64(len(idx.elems) * 2)) + + for i := uint64(0); i < capacity; i++ { + e := &elems[i] + if e.hash == 0 { + continue + } + idx.insertIDbyOffset(e.offset, e.id) + } + } + + // If the key was overwritten then decrement the size. + if overwritten := idx.insertIDbyOffset(offset, id); overwritten { + idx.n-- + } +} + +// insertIDbyOffset writes to the RHH id-by-offset map. +func (idx *index) insertIDbyOffset(offset int64, id uint64) (overwritten bool) { + key := idx.lookupKey(offset) + hash := hashKey(key) + pos := hash & idx.mask + + var dist uint64 + for { + e := &idx.elems[pos] + + // Exit if a matching or empty slot exists. + if e.hash == 0 { + e.hash, e.offset, e.id = hash, offset, id + return false + } else if bytes.Equal(idx.lookupKey(e.offset), key) { + e.hash, e.offset, e.id = hash, offset, id + return true + } + + // Swap if current element has a lower probe distance. + d := idx.dist(e.hash, pos) + if d < dist { + hash, e.hash = e.hash, hash + offset, e.offset = e.offset, offset + id, e.id = e.id, id + dist = d + } + + // Move position forward. + pos = (pos + 1) & idx.mask + dist++ + } +} + +// lookupKey returns the key at the given offset in the memory-mapped file. +func (idx *index) lookupKey(offset int64) []byte { + data := idx.data[offset:] + n, sz := binary.Uvarint(data) + if sz == 0 { + return nil + } + return data[sz : sz+int(n)] +} + +func (idx *index) alloc(capacity uint64) { + idx.elems = make([]elem, capacity) + idx.threshold = (capacity * uint64(idx.loadFactor)) / 100 + idx.mask = uint64(capacity - 1) +} + +func (idx *index) dist(hash, i uint64) uint64 { + return (i + uint64(len(idx.elems)) - (hash & idx.mask)) & idx.mask +} + +type elem struct { + offset int64 + id uint64 + hash uint64 +} + +func (e *elem) reset() { + e.offset = 0 + e.id = 0 + e.hash = 0 +} + +func hashKey(key []byte) uint64 { + h := xxhash.Sum64(key) + if h == 0 { + h = 1 + } + return h +} + +func pow2(v uint64) uint64 { + for i := uint64(2); i < 1<<62; i *= 2 { + if i >= v { + return i + } + } + panic("unreachable") +} + +// TranslateFileReader implements a reader that continuously streams data from a store. +type TranslateFileReader struct { + ctx context.Context + store *TranslateFile + file *os.File + offset int64 + notify <-chan struct{} + + once sync.Once + closing chan struct{} +} + +// NewTranslateFileReader returns a new instance of TranslateFileReader. +func NewTranslateFileReader(ctx context.Context, store *TranslateFile, offset int64) *TranslateFileReader { + return &TranslateFileReader{ + ctx: ctx, + store: store, + offset: offset, + notify: store.WriteNotify(), + closing: make(chan struct{}), + } +} + +// Open initializes the reader. +func (r *TranslateFileReader) Open() (err error) { + if r.file, err = os.Open(r.store.Path); err != nil { + return err + } + return nil +} + +// Close closes the underlying file reader. +func (r *TranslateFileReader) Close() error { + r.once.Do(func() { close(r.closing) }) + + if r.file != nil { + return r.file.Close() + } + return nil +} + +// Read reads the next section of the available data to p. This should always +// read from the start of an entry and read n bytes to the end of another entry. +func (r *TranslateFileReader) Read(p []byte) (n int, err error) { + for { + // Obtain notification channel before we check for new data. + notify := r.store.WriteNotify() + + // Exit if we can read one or more valid entries or we receive an error. + if n, err = r.read(p); n > 0 || err != nil { + return n, err + } + + // Wait for new data or close. + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + case <-r.closing: + return 0, ErrTranslateStoreReaderClosed + case <-r.store.Closing(): + return 0, ErrTranslateStoreClosed + case <-notify: + continue + } + } +} + +// read writes the bytes for zero or more valid entries to p. +func (r *TranslateFileReader) read(p []byte) (n int, err error) { + sz := r.store.Size() + + // Exit if there is no new data. + if sz < r.offset { + return 0, fmt.Errorf("pilosa: translate store reader past file size: sz=%d off=%d", sz, r.offset) + } else if sz == r.offset { + return 0, nil + } + + // Shorten buffer to maximum read size. + if max := sz - r.offset; int64(len(p)) > max { + p = p[:max] + } + + // Read data from file at offset. + // Limit the number of bytes read to only whole entries. + n, err = r.file.ReadAt(p, r.offset) + n = ValidLogEntriesLen(p[:n]) + r.offset += int64(n) + return n, err +} + +// Copied & modified from encoding/binary. +func UvarintSize(x uint64) (i int) { + for x >= 0x80 { + x >>= 7 + i++ + } + return i + 1 +} + +func hexdump(b []byte) { os.Stderr.Write([]byte(hex.Dump(b))) } diff --git a/translate_test.go b/translate_test.go new file mode 100644 index 000000000..c73853d69 --- /dev/null +++ b/translate_test.go @@ -0,0 +1,565 @@ +package pilosa_test + +import ( + "bufio" + "context" + "fmt" + "io/ioutil" + "math/rand" + "os" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/pilosa/pilosa" +) + +func TestTranslateFile_TranslateColumn(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateColumn_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateColumnsToUint64("IDX0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateColumnToString("IDX0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_TranslateRow(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FRAME0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + // Reopen the store. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + + // Translation on a different frame restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } +} + +func TestTranslateFile_TranslateRow_Large(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate key/values. + for i := 0; i < 1000000; i += 1000 { + keys := make([]string, 1000) + for j := 0; j < 1000; j++ { + keys[j] = strconv.Itoa(i + j + 1) + } + + ids, err := s.TranslateRowsToUint64("IDX0", "FRAME0", keys) + if err != nil { + t.Fatal(err) + } + + for j, id := range ids { + if exp := uint64(i + j + 1); id != exp { + t.Fatalf("unexpected id: got=%d, exp=%d", id, exp) + } + } + } + + // Verify values can be returned. + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } + + // Reopen and re-verify. + if err := s.Reopen(); err != nil { + t.Fatal(err) + } + for i := 0; i < 1000000; i++ { + exp := strconv.Itoa(i + 1) + if key, err := s.TranslateRowToString("IDX0", "FRAME0", uint64(i+1)); err != nil { + t.Fatal(err) + } else if key != exp { + t.Fatalf("unexpected key: got=%q, exp=%q", key, exp) + } + } +} + +func TestTranslateFile_Reader(t *testing.T) { + t.Run("NoOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + rc, err := s.Reader(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // Read first entry. Should read 'entry length' (13) plus uvarint(size) (1) = 14b. + var entry pilosa.LogEntry + if n, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if n != 14 { + t.Fatalf("unexpected n: %d", n) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{1}, + Keys: [][]byte{[]byte("foo")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Read second entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + + // Write new entry. + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"xyz"}); err != nil { + t.Fatal(err) + } + + // Read new entry. + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertColumn, + Index: []byte("IDX0"), + IDs: []uint64{2}, + Keys: [][]byte{[]byte("xyz")}, + Length: 13, + }); diff != "" { + t.Fatal(diff) + } + + // Close reader and ensure it returns EOF. + if err := rc.Close(); err != nil { + t.Fatal(err) + } else if _, err := entry.ReadFrom(brc); err != pilosa.ErrTranslateStoreReaderClosed { + t.Fatalf("unexpected error: %s", err) + } + }) + + t.Run("WithOffset", func(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + if _, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := s.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Start offset after the first entry. + rc, err := s.Reader(context.Background(), 14) + if err != nil { + t.Fatal(err) + } + brc := bufio.NewReader(rc) + defer rc.Close() + + // This should be the second entry. + var entry pilosa.LogEntry + if _, err := entry.ReadFrom(brc); err != nil { + t.Fatal(err) + } else if diff := cmp.Diff(entry, pilosa.LogEntry{ + Type: pilosa.LogEntryTypeInsertRow, + Index: []byte("IDX0"), + Frame: []byte("FRAME0"), + IDs: []uint64{1, 2}, + Keys: [][]byte{[]byte("bar"), []byte("baz")}, + Length: 24, + }); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestTranslateFile_PrimaryTranslateStore(t *testing.T) { + // Create a primary store that accepts writes. + primary := MustOpenTranslateFile() + defer primary.MustClose() + + // Create a replica that accepts writes from primary. + replica := NewTranslateFile() + replica.PrimaryTranslateStore = primary + if err := replica.Open(); err != nil { + t.Fatal(err) + } + defer replica.MustClose() + + // Write to the primary. + if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateRowsToUint64("IDX0", "FRAME0", []string{"bar", "baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until writes appear. + if err := retryFor(2*time.Second, func() error { + // Verify that replica have received writes. + if value, err := replica.TranslateColumnToString("IDX0", 1); err != nil { + return err + } else if value != "foo" { + return fmt.Errorf("unexpected column 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 1); err != nil { + return err + } else if value != "bar" { + return fmt.Errorf("unexpected row 1 value: %s", value) + } + + if value, err := replica.TranslateRowToString("IDX0", "FRAME0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected row 2 value: %s", value) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect primary store & write more values. + if err := primary.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 2); err != nil { + return err + } else if value != "baz" { + return fmt.Errorf("unexpected column 2 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // Disconnect replica store & write more values. + if err := replica.Reopen(); err != nil { + t.Fatal(err) + } else if _, err := primary.TranslateColumnsToUint64("IDX0", []string{"foobar"}); err != nil { + t.Fatal(err) + } + + // Attempt to read replica until write appear. + if err := retryFor(2*time.Second, func() error { + if value, err := replica.TranslateColumnToString("IDX0", 3); err != nil { + return err + } else if value != "foobar" { + return fmt.Errorf("unexpected column 3 value: %s", value) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + keySets := make([][]string, b.N/batchSize) + for i := range keySets { + keySets[i] = make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySets[i][j] = fmt.Sprintf("%08d%08d", jv, i) + } + } + + b.ResetTimer() + + for _, keySet := range keySets { + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkTranslateFile_TranslateColumnToString(b *testing.B) { + const batchSize = 1000 + + s := MustOpenTranslateFile() + defer s.MustClose() + + // Generate keys before benchmark begins + for i := 0; i < b.N; i += batchSize { + keySet := make([]string, batchSize) + for j, jv := range rand.New(rand.NewSource(0)).Perm(batchSize) { + keySet[j] = fmt.Sprintf("%08d%08d", jv, i) + } + if _, err := s.TranslateColumnsToUint64("IDX0", keySet); err != nil { + b.Fatal(err) + } + } + + // Generate random key access. + perm := rand.New(rand.NewSource(0)).Perm(b.N) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := s.TranslateColumnToString("IDX0", uint64(perm[i])); err != nil { + b.Fatal(err) + } + } +} + +type TranslateFile struct { + *pilosa.TranslateFile +} + +func NewTranslateFile() *TranslateFile { + f, err := ioutil.TempFile("", "") + if err != nil { + panic(err) + } + f.Close() + + s := &TranslateFile{TranslateFile: pilosa.NewTranslateFile()} + s.Path = f.Name() + return s +} + +func MustOpenTranslateFile() *TranslateFile { + s := NewTranslateFile() + if err := s.Open(); err != nil { + panic(err) + } + return s +} + +func (s *TranslateFile) Close() error { + defer os.Remove(s.Path) + return s.TranslateFile.Close() +} + +func (s *TranslateFile) MustClose() { + if err := s.Close(); err != nil { + panic(err) + } +} + +// Reopen closes the store and opens a new instance of it for the same path. +func (s *TranslateFile) Reopen() error { + prev := s.TranslateFile + if err := s.TranslateFile.Close(); err != nil { + return err + } + + s.TranslateFile = pilosa.NewTranslateFile() + s.Path = prev.Path + s.PrimaryTranslateStore = prev.PrimaryTranslateStore + if err := s.Open(); err != nil { + return err + } + return nil +} + +// retryFor executes fn every 100ms until d time passes or until fn return nil. +func retryFor(d time.Duration, fn func() error) (err error) { + timer, ticker := time.NewTimer(d), time.NewTicker(100*time.Millisecond) + defer timer.Stop() + defer ticker.Stop() + + for { + if err = fn(); err == nil { + return nil + } + + select { + case <-timer.C: + return err + case <-ticker.C: + } + } +} diff --git a/utils_test.go b/utils_internal_test.go similarity index 85% rename from utils_test.go rename to utils_internal_test.go index 1732522d4..4055ef566 100644 --- a/utils_test.go +++ b/utils_internal_test.go @@ -104,33 +104,33 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } -func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { +func (t *ClusterCluster) CreateField(index, field string, opt FieldOptions) error { for _, c := range t.Clusters { idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) if err != nil { return err } - if _, err := idx.CreateFrame(frame, opt); err != nil { + if _, err := idx.CreateField(field, opt); err != nil { return err } } return nil } -func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { +func (t *ClusterCluster) SetBit(index, field, view string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / SliceWidth - nodes := c0.SliceNodes(index, slice) + nodes := c0.sliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) if c == nil { continue } - f := c.Holder.Frame(index, frame) + f := c.Holder.Field(index, field) if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + return fmt.Errorf("index/field does not exist: %s/%s", index, field) } _, err := f.SetBit(view, rowID, colID, x) if err != nil { @@ -141,30 +141,6 @@ func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, return nil } -func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { - // Determine which node should receive the SetFieldValue. - c0 := t.Clusters[0] // use the first node's cluster to determine slice location. - slice := columnID / SliceWidth - nodes := c0.SliceNodes(index, slice) - - for _, node := range nodes { - c := t.clusterByID(node.ID) - if c == nil { - continue - } - f := c.Holder.Frame(index, frame) - if f == nil { - return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) - } - _, err := f.SetFieldValue(columnID, name, value) - if err != nil { - return err - } - } - - return nil -} - func (t *ClusterCluster) clusterByID(id string) *Cluster { for _, c := range t.Clusters { if c.Node.ID == id { @@ -260,7 +236,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) // add nodes if saveTopology { for _, n := range t.common.Nodes { - c.AddNode(n) + c.addNode(n) } } @@ -297,13 +273,13 @@ func (t *ClusterCluster) SetState(state string) { // Open opens all clusters in the test cluster. func (t *ClusterCluster) Open() error { for _, c := range t.Clusters { - if err := c.Open(); err != nil { + if err := c.open(); err != nil { return err } if err := c.Holder.Open(); err != nil { return err } - if err := c.SetNodeState(NodeStateReady); err != nil { + if err := c.setNodeState(NodeStateReady); err != nil { return err } } @@ -312,7 +288,7 @@ func (t *ClusterCluster) Open() error { if len(t.Clusters) == 0 { return nil } - t.Clusters[0].ListenForJoins() + t.Clusters[0].listenForJoins() return nil } @@ -320,7 +296,7 @@ func (t *ClusterCluster) Open() error { // Close closes all clusters in the test cluster. func (t *ClusterCluster) Close() error { for _, c := range t.Clusters { - err := c.Close() + err := c.close() if err != nil { return err } @@ -334,7 +310,7 @@ func (t *ClusterCluster) SendSync(pb proto.Message) error { case *internal.ClusterStatus: // Apply the send message to all nodes (except the coordinator). for _, c := range t.Clusters { - c.MergeClusterStatus(obj) + c.mergeClusterStatus(obj) } t.mu.RLock() if obj.State == ClusterStateNormal && t.resizing { @@ -361,7 +337,7 @@ func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { } case *internal.ResizeInstructionComplete: coord := t.clusterByID(to.ID) - go coord.MarkResizeInstructionComplete(obj) + go coord.markResizeInstructionComplete(obj) } return nil } @@ -380,7 +356,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi if err := func() error { // figure out which node it was meant for, then call the operation on that cluster - // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Field, src.View, src.Slice, srcURI) instrNode := DecodeNode(instr.Node) destCluster := t.clusterByID(instrNode.ID) @@ -393,11 +369,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstructi srcNode := DecodeNode(src.Node) srcCluster := t.clusterByID(srcNode.ID) - srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) - destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Field, src.View, src.Slice) if destFragment == nil { // Create fragment on destination if it doesn't exist. - f := destCluster.Holder.Frame(src.Index, src.Frame) + f := destCluster.Holder.Field(src.Index, src.Field) v := f.View(src.View) var err error destFragment, err = v.CreateFragmentIfNotExists(src.Slice) diff --git a/view.go b/view.go index f90bf64e9..428fc6f54 100644 --- a/view.go +++ b/view.go @@ -31,26 +31,26 @@ import ( const ( ViewStandard = "standard" - ViewFieldPrefix = "field_" + viewBSIGroupPrefix = "bsig_" ) -// IsValidView returns true if name is valid. -func IsValidView(name string) bool { +// isValidView returns true if name is valid. +func isValidView(name string) bool { return name == ViewStandard } -// View represents a container for frame data. +// View represents a container for field data. type View struct { mu sync.RWMutex path string index string - frame string + field string name string cacheSize uint32 // Fragments by slice. - cacheType string // passed in by frame + cacheType string // passed in by field fragments map[uint64]*Fragment // maxSlice maintains this view's max slice in order to @@ -65,15 +65,15 @@ type View struct { } // NewView returns a new instance of View. -func NewView(path, index, frame, name string, cacheSize uint32) *View { +func NewView(path, index, field, name string, cacheSize uint32) *View { return &View{ path: path, index: index, - frame: frame, + field: field, name: name, cacheSize: cacheSize, - cacheType: DefaultCacheType, + cacheType: defaultCacheType, fragments: make(map[uint64]*Fragment), broadcaster: NopBroadcaster, @@ -82,23 +82,11 @@ func NewView(path, index, frame, name string, cacheSize uint32) *View { } } -// Name returns the name the view was initialized with. -func (v *View) Name() string { return v.name } - -// Index returns the index name the view was initialized with. -func (v *View) Index() string { return v.index } - -// Frame returns the frame name the view was initialized with. -func (v *View) Frame() string { return v.frame } - -// Path returns the path the view was initialized with. -func (v *View) Path() string { return v.path } - -// Open opens and initializes the view. -func (v *View) Open() error { +// open opens and initializes the view. +func (v *View) open() error { // Never keep a cache for field views. - if strings.HasPrefix(v.name, ViewFieldPrefix) { + if strings.HasPrefix(v.name, viewBSIGroupPrefix) { v.cacheType = CacheTypeNone } @@ -116,7 +104,7 @@ func (v *View) Open() error { return nil }(); err != nil { - v.Close() + v.close() return err } @@ -149,19 +137,19 @@ func (v *View) openFragments() error { continue } - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { - return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err) + return fmt.Errorf("open fragment: slice=%d, err=%s", frag.slice, err) } frag.RowAttrStore = v.RowAttrStore - v.fragments[frag.Slice()] = frag + v.fragments[frag.slice] = frag } return nil } -// Close closes the view and its fragments. -func (v *View) Close() error { +// close closes the view and its fragments. +func (v *View) close() error { v.mu.Lock() defer v.mu.Unlock() @@ -176,8 +164,8 @@ func (v *View) Close() error { return nil } -// MaxSlice returns the max slice in the view. -func (v *View) MaxSlice() uint64 { +// calculateMaxSlice returns the max slice in the view. +func (v *View) calculateMaxSlice() uint64 { v.mu.RLock() defer v.mu.RUnlock() @@ -191,8 +179,8 @@ func (v *View) MaxSlice() uint64 { return max } -// FragmentPath returns the path to a fragment in the view. -func (v *View) FragmentPath(slice uint64) string { +// fragmentPath returns the path to a fragment in the view. +func (v *View) fragmentPath(slice uint64) string { return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10)) } @@ -205,8 +193,8 @@ func (v *View) Fragment(slice uint64) *Fragment { func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] } -// Fragments returns a list of all fragments in the view. -func (v *View) Fragments() []*Fragment { +// allFragments returns a list of all fragments in the view. +func (v *View) allFragments() []*Fragment { v.mu.Lock() defer v.mu.Unlock() @@ -217,9 +205,9 @@ func (v *View) Fragments() []*Fragment { return other } -// RecalculateCaches recalculates the cache on every fragment in the view. -func (v *View) RecalculateCaches() { - for _, fragment := range v.Fragments() { +// recalculateCaches recalculates the cache on every fragment in the view. +func (v *View) recalculateCaches() { + for _, fragment := range v.allFragments() { fragment.RecalculateCache() } } @@ -238,7 +226,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } // Initialize and open fragment. - frag := v.newFragment(v.FragmentPath(slice), slice) + frag := v.newFragment(v.fragmentPath(slice), slice) if err := frag.Open(); err != nil { return nil, errors.Wrap(err, "opening fragment") } @@ -249,13 +237,13 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { v.maxSlice = slice // Send the create slice message to all nodes. - err := v.broadcaster.SendAsync( + err := v.broadcaster.SendSync( &internal.CreateSliceMessage{ Index: v.index, Slice: slice, }) if err != nil { - return nil, errors.Wrap(err, "sending message") + return nil, errors.Wrap(err, "sending createslice message") } } @@ -265,7 +253,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { } func (v *View) newFragment(path string, slice uint64) *Fragment { - frag := NewFragment(path, v.index, v.frame, v.name, slice) + frag := NewFragment(path, v.index, v.field, v.name, slice) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize frag.Logger = v.Logger @@ -273,15 +261,15 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { return frag } -// DeleteFragment removes the fragment from the view. -func (v *View) DeleteFragment(slice uint64) error { +// deleteFragment removes the fragment from the view. +func (v *View) deleteFragment(slice uint64) error { fragment := v.fragments[slice] if fragment == nil { return ErrFragmentNotFound } - v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice) + v.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, slice) // Close data files before deletion. if err := fragment.Close(); err != nil { @@ -289,12 +277,12 @@ func (v *View) DeleteFragment(slice uint64) error { } // Delete fragment file. - if err := os.Remove(fragment.Path()); err != nil { + if err := os.Remove(fragment.path); err != nil { return errors.Wrap(err, "deleting fragment file") } // Delete fragment cache file. - if err := os.Remove(fragment.CachePath()); err != nil { + if err := os.Remove(fragment.cachePath()); err != nil { v.Logger.Printf("no cache file to delete for slice %d", slice) } @@ -303,50 +291,64 @@ func (v *View) DeleteFragment(slice uint64) error { return nil } -// SetBit sets a bit within the view. -func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { +// row returns a row for a slice of the view. +func (v *View) row(rowID uint64) *Row { + row := NewRow() + for _, frag := range v.allFragments() { + fr := frag.row(rowID) + if fr == nil { + continue + } + row.Merge(fr) + } + return row + +} + +// setBit sets a bit within the view. +func (v *View) setBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetBit(rowID, columnID) + return frag.setBit(rowID, columnID) } -// ClearBit clears a bit within the view. -func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { +// clearBit clears a bit within the view. +func (v *View) clearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.ClearBit(rowID, columnID) + return frag.clearBit(rowID, columnID) } -// FieldValue uses a column of bits to read a multi-bit value. -func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// value uses a column of bits to read a multi-bit value. +func (v *View) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return value, exists, err } - return frag.FieldValue(columnID, bitDepth) + return frag.value(columnID, bitDepth) } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// setValue uses a column of bits to set a multi-bit value. +func (v *View) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetFieldValue(columnID, bitDepth, value) + return frag.setValue(columnID, bitDepth, value) } -// FieldSum returns the sum & count of a field. -func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - for _, f := range v.Fragments() { - fsum, fcount, err := f.FieldSum(filter, bitDepth) +// sum returns the sum & count of a field. +func (v *View) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) { + for _, f := range v.allFragments() { + fsum, fcount, err := f.sum(filter, bitDepth) if err != nil { return sum, count, err } @@ -356,11 +358,11 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro return sum, count, nil } -// FieldMin returns the min and count of a field. -func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { +// min returns the min and count of a field. +func (v *View) min(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool - for _, f := range v.Fragments() { - fmin, fcount, err := f.FieldMin(filter, bitDepth) + for _, f := range v.allFragments() { + fmin, fcount, err := f.min(filter, bitDepth) if err != nil { return min, count, err } @@ -384,10 +386,10 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro return min, count, nil } -// FieldMax returns the max and count of a field. -func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { - for _, f := range v.Fragments() { - fmax, fcount, err := f.FieldMax(filter, bitDepth) +// max returns the max and count of a field. +func (v *View) max(filter *Row, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.allFragments() { + fmax, fcount, err := f.max(filter, bitDepth) if err != nil { return max, count, err } @@ -399,11 +401,11 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro return max, count, nil } -// FieldRange returns rows with a field value encoding matching the predicate. -func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +// rangeOp returns rows with a field value encoding matching the predicate. +func (v *View) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.FieldRange(op, bitDepth, predicate) + for _, frag := range v.allFragments() { + other, err := frag.rangeOp(op, bitDepth, predicate) if err != nil { return nil, err } @@ -412,12 +414,12 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, return r, nil } -// FieldRangeBetween returns bitmaps with a field value encoding matching any +// rangeBetween returns bitmaps with a field value encoding matching any // value between predicateMin and predicateMax. -func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (v *View) rangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() - for _, frag := range v.Fragments() { - other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) + for _, frag := range v.allFragments() { + other, err := frag.rangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } diff --git a/view_internal_test.go b/view_internal_test.go new file mode 100644 index 000000000..48df030db --- /dev/null +++ b/view_internal_test.go @@ -0,0 +1,68 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "io/ioutil" + "testing" +) + +// mustOpenView returns a new instance of View with a temporary path. +func mustOpenView(index, field, name string) *View { + path, err := ioutil.TempDir("", "pilosa-view-") + if err != nil { + panic(err) + } + + v := NewView(path, index, field, name, defaultCacheSize) + if err := v.open(); err != nil { + panic(err) + } + v.RowAttrStore = newMemAttrStore() + return v +} + +// Ensure view can open and retrieve a fragment. +func TestView_DeleteFragment(t *testing.T) { + v := mustOpenView("i", "f", "v") + defer v.close() + + slice := uint64(9) + + // Create fragment. + fragment, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == nil { + t.Fatal("expected fragment") + } + + err = v.deleteFragment(slice) + if err != nil { + t.Fatal(err) + } + + if v.Fragment(slice) != nil { + t.Fatal("fragment still exists in view") + } + + // Recreate fragment with same slice, verify that the old fragment was not reused. + fragment2, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + t.Fatal(err) + } else if fragment == fragment2 { + t.Fatal("failed to create new fragment") + } +} diff --git a/view_test.go b/view_test.go deleted file mode 100644 index 21834113d..000000000 --- a/view_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -// View is a test wrapper for pilosa.View. -type View struct { - *pilosa.View - RowAttrStore pilosa.AttrStore -} - -// NewView returns a new instance of View with a temporary path. -func NewView(index, frame, name string) *View { - path, err := ioutil.TempDir("", "pilosa-view-") - if err != nil { - panic(err) - } - - v := &View{ - View: pilosa.NewView(path, index, frame, name, pilosa.DefaultCacheSize), - RowAttrStore: test.MustOpenAttrStore(), - } - v.View.RowAttrStore = v.RowAttrStore - return v -} - -// MustOpenView creates and opens an view at a temporary path. Panic on error. -func MustOpenView(index, frame, name string) *View { - v := NewView(index, frame, name) - if err := v.Open(); err != nil { - panic(err) - } - return v -} - -// Close closes the view and removes all underlying data. -func (v *View) Close() error { - defer os.Remove(v.Path()) - defer v.RowAttrStore.Close() - return v.View.Close() -} - -// Reopen closes the view and reopens it as a new instance. -func (v *View) Reopen() error { - path := v.Path() - if err := v.View.Close(); err != nil { - return err - } - - v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.RowAttrStore = v.RowAttrStore - return v.Open() -} - -// MustSetBits sets columns on a row. Panic on error. -// This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.SetBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// MustClearColumns clears columns on a row. Panic on error. -func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { - for _, columnID := range columnIDs { - if _, err := v.ClearBit(rowID, columnID); err != nil { - panic(err) - } - } -} - -// Ensure view can open and retrieve a fragment. -func TestView_DeleteFragment(t *testing.T) { - v := MustOpenView("i", "f", "v") - defer v.Close() - - slice := uint64(9) - - // Create fragment. - fragment, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == nil { - t.Fatal("expected fragment") - } - - err = v.DeleteFragment(slice) - if err != nil { - t.Fatal(err) - } - - if v.Fragment(slice) != nil { - t.Fatal("fragment still exists in view") - } - - // Recreate fragment with same slice, verify that the old fragment was not reused. - fragment2, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - t.Fatal(err) - } else if fragment == fragment2 { - t.Fatal("failed to create new fragment") - } -} diff --git a/webui/assets/chevron-down.png b/webui/assets/chevron-down.png deleted file mode 100644 index 3312489a2..000000000 Binary files a/webui/assets/chevron-down.png and /dev/null differ diff --git a/webui/assets/main.js b/webui/assets/main.js deleted file mode 100644 index 265f51997..000000000 --- a/webui/assets/main.js +++ /dev/null @@ -1,594 +0,0 @@ -class REPL { - constructor(input, output, button, completer) { - this.input = input - this.output = output - this.button = button - this.completer = completer - this.history = [] - this.history_index = 0 - this.history_buffer = '' - this.result_number = 0 - } - bind_events() { - var repl = this - var keys = { - TAB: 9, - ENTER: 13, - UP_ARROW: 38, - DOWN_ARROW: 40 - } - - this.input.addEventListener("keydown", function(e) { - if (e.keyCode == keys.UP_ARROW) { - e.preventDefault() - if (repl.input.value.substring(0, repl.input.selectionStart).indexOf('\n') == '-1') { - if (repl.history_index == 0) { - return - } else { - if (repl.history_index == repl.history.length) { - repl.history_buffer = repl.input.value - } - repl.history_index-- - repl.input.value = repl.history[repl.history_index] - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.DOWN_ARROW) { - e.preventDefault() - if (repl.input.value.substring(repl.input.selectionEnd, repl.input.length).indexOf('\n') == '-1') { - if (repl.history_index == repl.history.length) { - return - } else { - repl.history_index++ - if (repl.history_index == repl.history.length) { - repl.input.value = repl.history_buffer - } else { - repl.input.value = repl.history[repl.history_index] - } - repl.input.setSelectionRange(repl.input.value.length, repl.input.value.length) - } - } - } - if (e.keyCode == keys.ENTER && !e.shiftKey) { - e.preventDefault() - repl.submit(); - } - if (e.keyCode == keys.TAB) { - e.preventDefault() - repl.completer.complete() - } - }) - repl.button.onclick = function() { - repl.submit(); - }; - } - - submit() { - this.history_buffer = '' - this.history_index = this.history.length - this.history[this.history_index] = this.input.value - this.history_index++ - this.process_query(this.input.value) - this.input.value = "" - } - - process_query(query) { - var xhr = new XMLHttpRequest(); - var url, data, request, command_name; - var e = document.getElementById("index-dropdown"); - var indexname = e.options[e.selectedIndex].text; - var repl = this; - if (query.startsWith(":")) { - var parsed_query = parse_query(query, indexname); - if (Object.keys(parsed_query).length === 0) { - repl.create_single_output({ - "input": query, - "output": "invalid query", - "status": 400, - "indexname": indexname, - }); - return; - } else { - // set selectedIndex from dropdown list - if (parsed_query.command === "use") { - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === parsed_query.command_name) { - e.selectedIndex = i; - break; - } - } - return; - } - url = parsed_query.url; - data = parsed_query.data; - request = parsed_query.request; - command_name = parsed_query.command_name; - } - } - else { - url = '/index/' + indexname + '/query' - request = "POST" - data = query - } - xhr.open(request, url); - xhr.setRequestHeader('Content-Type', 'application/text'); - - var start_time = new Date().getTime(); - xhr.onload = function () { - var end_time = new Date().getTime(); - repl.result_number++ - repl.create_single_output({ - "input": query, - "output": xhr.responseText, - "status": xhr.status, - "indexname": indexname, - "querytime_ms": end_time - start_time, - }); - }; - - xhr.send(data); - // Remove index from dropdown with delete index command - if (request === 'DELETE' && url === '/index/' + command_name){ - for (var i = 0; i < e.options.length; i++) { - if (e.options[i].text === command_name) { - e.remove(i); - break; - } - } - } - } - - create_single_output(res) { - var node = document.createElement("div"); - node.classList.add('output'); - var output_string = res['output'] - var result_class = "result-output" - var getting_started_errors = [ - 'index not found', - 'frame not found', - ] - var output_json; - if (isJSON(output_string)) { - output_json = JSON.parse(output_string) - } - // handle output formatting - if (res["status"] != 200) { - result_class = "result-error"; - if (output_json) { - if ("error" in output_json) { - if (getting_started_errors.indexOf(output_json['error']) >= 0) { - output_string += `
-
- Just getting started? Try this:
- :create index test
- :use test
- :create frame foo
- SetBit(rowID=0, columnID=0, frame=foo) # Use PQL to set a bit - ` - } - } - } - } - - - var markup =` -
-
-
-
-
Input
-        - Source: ${res.indexname} -
-
- ${res.input} -
-
-
-
-
output
-        - ${res.querytime_ms} ms -
-
- ${output_string} -
-
Expand
 - -
-
-
-
- - Export raw .csv -
-
-
- ` - node.innerHTML = markup; - this.output.insertBefore(node, this.output.firstChild); - - // Expand when overflow - var element = this.output.firstChild.getElementsByClassName(result_class)[0]; - var expand = this.output.firstChild.getElementsByClassName("expand")[0]; - if (element.clientHeight < element.scrollHeight) { - expand.style.display = 'block'; - } else { - expand.style.display = 'none'; - } - expand.onclick = function () { - element.style.height = element.scrollHeight + "px"; - expand.style.display = 'none'; - return false; - }; - } - - populate_index_dropdown() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/schema') - var select = document.getElementById('index-dropdown') - - xhr.onload = function() { - var schema = JSON.parse(xhr.responseText) - for(var i=0; i 0) { - select.value = 1; - } - } - xhr.send(null) - } - -} - -function populate_version() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/version') - var node = document.getElementById('server-version') - - xhr.onload = function() { - var version = JSON.parse(xhr.responseText)['version'] - var version_major_minor = /v?(\d+\.\d+).*/.exec(version)[1] - var doc_link = document.getElementById('nav-documentation') - doc_link.onclick = function() { - window.open('https://www.pilosa.com/docs/v' + version_major_minor + '/introduction/') - } - node.innerHTML = "Pilosa v" + version - } - xhr.send(null) -} - -function handle_nav_click(e) { - // e.id = "nav-xxx" - name = e.id.substring(4) - set_active_pane_by_name(name) - window.location.hash = name -} - -function set_active_pane_by_name(name) { - // toggle the nav buttons - document.getElementsByClassName("nav-active")[0].classList.remove("nav-active") - document.getElementById("nav-" + name).classList.add("nav-active") - - // toggle the main interface content divs - document.getElementsByClassName("interface-active")[0].classList.remove("interface-active") - document.getElementById('interface-' + name).classList.add("interface-active") - - // hack hack - switch(name) { - case "cluster": - update_cluster_status() - break - case "documentation": - open_external_docs() - break - } -} - - -function update_cluster_status() { - var xhr = new XMLHttpRequest(); - xhr.open('GET', '/status') - xhr.onload = function() { - var status = JSON.parse(xhr.responseText) - render_status(status) - } - xhr.send(null) - - var xhrSchema = new XMLHttpRequest(); - xhrSchema.open('GET', '/schema') - xhrSchema.onload = function() { - var schema = JSON.parse(xhrSchema.responseText) - render_schema(schema) - } - xhrSchema.send(null) -} - -function render_status(status) { - // render node table - var nodes_div = document.getElementById("status-nodes") - while (nodes_div.firstChild) { - nodes_div.removeChild(nodes_div.firstChild); - } - - var nodes = status["nodes"] - table = document.createElement("table") - tbody = document.createElement("tbody") - table.appendChild(tbody) - var caption = document.createElement("caption") - caption.innerHTML = "(" + nodes.length + ")" - table.appendChild(caption) - - var header = document.createElement('tr') - markup = `Host - ID - Coordinator` - header.innerHTML = markup - tbody.appendChild(header) - for(var n=0; n${nodes[n]["uri"]["host"]}:${nodes[n]["uri"]["port"]} - ${nodes[n]["id"]} - ${nodes[n]["isCoordinator"]}` - row.innerHTML = markup - tbody.appendChild(row) - } - nodes_div.appendChild(table) -} - -function render_schema(schema) { - // render index tables - var indexes_div = document.getElementById("status-indexes") - while (indexes_div.firstChild) { - indexes_div.removeChild(indexes_div.firstChild); - } - - var indexes = schema["indexes"] // TODO currently comes from only node 0 - for(var n=0; nName - Cache Type - Cache Size` - header.innerHTML = markup - tbody.appendChild(header) - - var frames = indexes[n]["frames"] - if(frames) { - for(var m=0; m${frames[m]["name"]} - ${frames[m]["options"]["cacheType"]} - ${frames[m]["options"]["cacheSize"]}` - tbody.appendChild(row) - } - } - indexes_div.appendChild(table) - } - - // render slice tables - // TODO enable when Slices element is present in status response - /* - var slices_div = document.getElementById("status-slices") - data = "" - for(var n=0; n" - } - } - slices_div.innerHTML = data - */ - -} - -function open_external_docs() { - window.open("https://www.pilosa.com/docs"); -} - -function check_anchor_uri() { - var pane_names = {"console": 0, "cluster": 0, "documentation": 0} - var anchor = window.location.hash.substr(1); - if(anchor in pane_names) { - set_active_pane_by_name(anchor) - } -} - -Date.prototype.today = function () { - return this.getFullYear() +"/"+ (((this.getMonth()+1) < 10)?"0":"") + (this.getMonth()+1) +"/"+ ((this.getDate() < 10)?"0":"") + this.getDate(); -} - -Date.prototype.timeNow = function () { - return ((this.getHours() < 10)?"0":"") + this.getHours() +":"+ ((this.getMinutes() < 10)?"0":"") + this.getMinutes() +":"+ ((this.getSeconds() < 10)?"0":"") + this.getSeconds(); -} - -populate_version() - - -class Autocompleter { - constructor(input, output) { - this.input = input - this.output = output - this.keyword_map = this.static_keywords - this.init_dynamic_keywords() - } - - get static_keywords() { - return { - // keyword: length of substring that comes after cursor - "SetBit()": 1, - "ClearBit()": 1, - "SetRowAttrs()": 1, - "SetColumnAttrs()": 1, - "Bitmap()": 1, - "Union()": 1, - "Intersect()": 1, - "Difference()": 1, - "Count()": 1, - "Range()": 1, - "TopN()": 1, - "frame=": 0, - } - } - - complete() { - var completer = this - // extract word fragment ending at cursor. a word fragment: - // - starts with last nonalpha character before cursor (or beginning of string) - // - ends at cursor - var word_start = completer.input.selectionEnd-1 - while(word_start>0) { - var c = completer.input.value.charCodeAt(word_start) - if(!((c>64 && c<91) || (c>96 && c<123))) { - word_start++ - break - } - word_start-- - } - var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd) - - // check for keyword match and insert if exactly one match - var matches = [] - for(var keyword in this.keyword_map) { - if(keyword.startsWith(input_word)){ - matches.push(keyword) - } - } - if(matches.length > 1) { - // completer.output.innerHTML = whatever - } - - if(matches.length == 1) { - // completer.output.innerHTML = "" - var cursor_pos = completer.input.selectionEnd - var completion = matches[0].substring(input_word.length) - var before = completer.input.value.substring(0, cursor_pos) - var after = completer.input.value.substring(cursor_pos) - completer.input.value = before + completion + after - var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]] - completer.input.setSelectionRange(new_pos, new_pos) - } - } - - init_dynamic_keywords() { - // hit /schema, parse indexes, frames, add to list - } - - add_keyword() { - // call when index or frame created in webui - } - - remove_keyword() { - // call when index or frame deleted in webui - // issue: if e.g. multiple indexes have same frame, removing one removes all. - // solution: maintain count. requires more elaborate representation of keywords. - } -} - -var input = document.getElementById('query') -var output = document.getElementById('outputs') -var button = document.getElementById('query-btn') -var autocomplete_output = document.getElementById('autocomplete-container') - -autocompleter = new Autocompleter(input, autocomplete_output) -repl = new REPL(input, output, button, autocompleter) -repl.populate_index_dropdown() -repl.bind_events() - -input.focus() - -check_anchor_uri() - -function isJSON(str) { - try { - JSON.parse(str) - } catch (e) { - return false - } - return true -} - -function parse_query(query, indexname) { - var keys = query.replace(/\s+/g, " ").split(" "); - var command = keys[0]; - var command_type = keys[1]; - var command_name = keys[2]; - var option_str = keys.slice(3, keys.length) - var options = parse_options(option_str); - if (command !== ":use") { - if (!command_name){ - return {} - } - } - - var parsed_query = {}; - parsed_query["command"] = command.substr(1, command.length); - parsed_query["command_name"] = command_name; - switch (command) { - case ":create": - parsed_query["request"] = "POST"; - if(Object.keys(options).length === 0) { - parsed_query["data"] = ""; - } else { - var opts = {"options":{}}; - for (var o in options) { - opts.options[o] = options[o] - } - parsed_query["data"] = JSON.stringify(opts); - } - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - break - } - break; - case ":delete": - parsed_query["request"] = "DELETE"; - switch (command_type){ - case "index": - parsed_query["url"] = '/index/' + command_name; - parsed_query["data"] = ""; - break; - case "frame": - parsed_query["url"] = '/index/' + indexname + '/frame/' + command_name; - parsed_query["data"] = ""; - break; - } - break; - case ":use": - parsed_query["command_name"] = keys[1]; - break; - default: - return {} - } - return parsed_query; -} - -function parse_options(option_str) { - var int_keys = ["cacheSize"]; - var options = {}; - for (var i = 0; i < option_str.length; i++) { - var parts = option_str[i].split('='); - if (int_keys.indexOf(parts[0]) !== -1 ){ - options[parts[0]] = Number(parts[1]) - } else { - options[parts[0]] = parts[1] - } - } - return options; -} diff --git a/webui/assets/nav-cluster-active.svg b/webui/assets/nav-cluster-active.svg deleted file mode 100644 index 871d6f2af..000000000 --- a/webui/assets/nav-cluster-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 diff --git a/webui/assets/nav-cluster.svg b/webui/assets/nav-cluster.svg deleted file mode 100644 index baf6310ea..000000000 --- a/webui/assets/nav-cluster.svg +++ /dev/null @@ -1 +0,0 @@ -nav_cluster_1 \ No newline at end of file diff --git a/webui/assets/nav-console-active.svg b/webui/assets/nav-console-active.svg deleted file mode 100644 index 2263c2a69..000000000 --- a/webui/assets/nav-console-active.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console \ No newline at end of file diff --git a/webui/assets/nav-console.svg b/webui/assets/nav-console.svg deleted file mode 100644 index e9d2a9569..000000000 --- a/webui/assets/nav-console.svg +++ /dev/null @@ -1 +0,0 @@ -nav_console diff --git a/webui/assets/nav-documentation-active.svg b/webui/assets/nav-documentation-active.svg deleted file mode 100644 index b6e994980..000000000 --- a/webui/assets/nav-documentation-active.svg +++ /dev/null @@ -1 +0,0 @@ -documentation diff --git a/webui/assets/nav-documentation.svg b/webui/assets/nav-documentation.svg deleted file mode 100644 index 12cd2108c..000000000 --- a/webui/assets/nav-documentation.svg +++ /dev/null @@ -1 +0,0 @@ -documentation \ No newline at end of file diff --git a/webui/assets/nav_item1.svg b/webui/assets/nav_item1.svg deleted file mode 100644 index d6d7f779e..000000000 --- a/webui/assets/nav_item1.svg +++ /dev/null @@ -1 +0,0 @@ -nav_item1 \ No newline at end of file diff --git a/webui/assets/style.css b/webui/assets/style.css deleted file mode 100644 index 2cf65d086..000000000 --- a/webui/assets/style.css +++ /dev/null @@ -1,336 +0,0 @@ -*{ - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} -body{ - font-family: sans-serif; - background-color: #fbfcfd; - margin: 0; - color: #102445; -} -h2{ - margin-bottom: 30px; -} - -h5{ - text-transform: uppercase; - letter-spacing: 2px; - line-height: 1.21; - margin: 0; -} -a{ - line-height: 1.38; - letter-spacing: 0.2px; - text-decoration: none; - color: #102445; -} - - -a:hover{ - color: #1db598; -} - -textarea{ - width: 100%; - margin-bottom: 10px; - border-radius: 2px; - background-color: #fbfcfd; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - outline: none; - padding: 30px; -} - - -select{ - /*-webkit-appearance: none; - -moz-appearance: none; - appearance: none; - background: url("img/chevron-down.png") no-repeat calc(100% - 10px) !important;*/ - border-radius: 3px; - background-color: #fbfcfd; - width: 187px; - height: 50px; - border: solid 1.5px #e4eff4; - font-size: 18px; - font-weight: bold; - line-height: 1.39; - letter-spacing: 0.2px; - color: #102445; - padding: 10.5px; - -} - -button{ - width: 165px; - height: 50px; - border-radius: 3px; - background-color: #1db598; - outline: none; - border: none; - font-size: 16px; - color: white; -} - -em{ - font-style: normal; - opacity: 0.5; - font-size: 14px; - font-weight: 500; - letter-spacing: 0.2px; - color: #102445; -} - -.header{ - height: 92px; - display: flex; - align-items: center; - justify-content: space-between; - width: 90%; - margin: auto; -} - -.container{ - display: flex; - height:100%; - min-height: 100vh; -} -.nav{ - color: white; - display: flex; - flex-direction: column; - width: 150px; - background: #3c5f8d; -} - -.nav-item{ - height:150px; - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; - border-bottom: 3px solid #2a4871; - cursor: pointer; -} - -.nav-active{ - background: #f2f7f9; - font-weight: bold; - color: #1db598; -} - -.nav-item > .nav-image { - display: flex; -} - -.nav-item > .nav-image-active { - display: none; -} - -.nav-active > .nav-image { - display: none; -} - -.nav-active > .nav-image-active { - display: flex; -} - - -.interface{ - display: none; - flex: 1; - flex-direction: column; - align-items: center; - background: #f2f7f9; -} - -.interface-active{ - display: flex; -} - -.query{ - margin-bottom: 30px; -} -.query, -.output-container, -.status-container{ - width: 75%; -} - -.output{ - margin-bottom: 30px; -} - -.input-controls{ - display: flex; - justify-content: flex-end; -} - -.tabs{ - display: flex; - background: #eaf2f6; -} -.active-tab{ - background: white; - font-weight: bold; - color: #1db598; - -} - -.tab{ - height:60px; - width: 100px; - border-top-right-radius: 5px; - display: flex; - align-items: center; - justify-content: center; - visibility: visible; - cursor: pointer; - -} - -.pane{ - background: white; - padding: 30px; - display: none; -} - -.active{ - display: block; -} - -.result-io-header{ - display: flex; - align-items: center; - margin-bottom: 15px; -} - -.result-input, -.result-output, -.result-error{ - height: 60px; - border-radius: 2px; - background-color: #fafafa; - border: solid 1.5px #e4eff4; - font-family: monospace; - font-size: 16px; - line-height: 1.5; - letter-spacing: 1.1px; - color: #102445; - padding: 15px; - margin-bottom: 15px; - word-break: break-all; - overflow-wrap: break-word; - overflow:hidden; -} - - -.result-output{ - background-color: #edf9f7; - border-left: solid 4px #1db598; -} - -.result-error{ - background-color: #fbf1f0; - border-left: solid 4px #fa3035; - color: #fa3035; -} - -.raw{ - height: 253px; - display: flex; - align-items: center; - justify-content: center; -} - - -.result-table > table { - border-left: solid 4px #1db598; -} - -table{ - border: solid 0.5px #e0e0e0; - width: 100%; - margin-bottom: 30px; - /*color:#3c5f8d;*/ -} -caption{ - text-align:left; - font-size: 16px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - text-align: left; -} -th{ - font-size: 14px; - font-weight: bold; - line-height: 1.21; - letter-spacing: 2px; - color: #102445; - text-transform: uppercase; - text-align: left; - padding: 21px 30px; - background-color: white; -} -tr{ - border: solid 0.5px #e0e0e0; - background-color: white; -} -tr:nth-child(even) { - background-color: #f2f7f9; -} -td{ - padding: 21px 30px; -} - -.expand { - text-align: center; -} - -.query h2 { - display: inline-block; -} - -.query-tooltip { - position: relative; - display: inline; - color: #000; - margin-left: 5px; -} - -.query-tooltip:hover { - color: #000; -} - -.query-tooltip-content { - background-color: rgb(250, 250, 250); - border: solid 1.5px #e4eff4; - color: #102445; - border-radius: 2px; - padding: 15px; - margin-bottom: 15px; - - position: absolute; - left: 80px; - top: -30px; - z-index: 1; -} - -.query-tooltip-container { - position: relative; - visibility: hidden; -} - -.query-tooltip:hover+.query-tooltip-container{ - visibility: visible; -} - -.code{ - font-family: monospace; -} - diff --git a/webui/index.html b/webui/index.html deleted file mode 100644 index f6bc369de..000000000 --- a/webui/index.html +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - - - Pilosa WebUI - - - -
- -
-
-
-
- - - -
-
- -
-

Query

- ? -
-
-
PQL
-
- SetBit(frame=foo, row=0, col=0)
- ClearBit(frame=foo, row=0, col=0)
- SetRowAttrs(frame=foo, row=0, color="blue")
- SetColumnAttrs(frame=foo, col=0, shape="circle")
- SetFieldValue(frame=foo, col=0, age=30)
- Bitmap(frame=foo, row=0)
- Range(frame=foo, row=0, start="2010-01", end="2017-03")
- Count(<BITMAP_CALL>)
- TopN([BITMAP_CALL], frame=foo, n=20)
- Union([BITMAP_CALL, ...])
- Intersect(<BITMAP_CALL>, [BITMAP_CALL, ...])
- Difference(<BITMAP_CALL>, <BITMAP_CALL>)
- Xor([BITMAP_CALL, ...])
- Min([BITMAP_CALL], frame=foo, field=age)
- Max([BITMAP_CALL], frame=foo, field=age)
- Sum([BITMAP_CALL], frame=foo, field=age) -
-
-
Special commands
-
- :create index test
- :use test
- :create frame foo
- :delete index test
- :delete frame foo -
-
- <tab>: autocomplete
- <up>/<down>: history
-
-
- -
-
- -     - -
-
-
- -
-

Output

-
- -
-
- -
- -
-
-

Nodes

-
-
-
-
-

Indexes

-
-
-
-
- -
- -
- -
- docs! -
- -
- - -